From 725c4b804a294b75dbf8cd00e4b42a2e4f54cf57 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Thu, 20 Feb 2020 09:02:05 -0500 Subject: [PATCH] import --- .clang-format | 40 + .gitignore | 3 + 3rd/catch/catch2/catch.hpp | 13359 +++++++++ 3rd/cpp-httplib/httplib.h | 4026 +++ 3rd/datasketches/datasketches/MurmurHash3.h | 178 + .../datasketches/cpc/compression_data.hpp | 6022 ++++ .../datasketches/cpc/counter_of_zeros.hpp | 105 + .../datasketches/cpc/cpc_common.hpp | 63 + .../datasketches/cpc/cpc_compressor.hpp | 147 + .../datasketches/cpc/cpc_compressor_impl.hpp | 743 + .../datasketches/cpc/cpc_confidence.hpp | 167 + .../datasketches/cpc/cpc_sketch.hpp | 184 + .../datasketches/cpc/cpc_sketch_impl.hpp | 793 + .../datasketches/cpc/cpc_union.hpp | 83 + .../datasketches/cpc/cpc_union_impl.hpp | 332 + .../datasketches/cpc/cpc_util.hpp | 137 + .../datasketches/cpc/icon_estimator.hpp | 274 + .../datasketches/cpc/inv_pow_2_tab.hpp | 107 + .../datasketches/cpc/kxp_byte_lookup.hpp | 81 + .../datasketches/cpc/u32_table.hpp | 84 + .../datasketches/cpc/u32_table_impl.hpp | 266 + .../datasketches/fi/frequent_items_sketch.hpp | 500 + .../fi/reverse_purge_hash_map.hpp | 425 + .../datasketches/kll/kll_helper.hpp | 150 + .../datasketches/kll/kll_helper_impl.hpp | 319 + .../kll/kll_quantile_calculator.hpp | 60 + .../kll/kll_quantile_calculator_impl.hpp | 191 + .../datasketches/kll/kll_sketch.hpp | 354 + .../datasketches/kll/kll_sketch_impl.hpp | 977 + 3rd/datasketches/datasketches/serde.hpp | 134 + 3rd/docopt/docopt.cpp | 685 + 3rd/docopt/docopt.h | 94 + 3rd/docopt/docopt_private.h | 674 + 3rd/docopt/docopt_util.h | 120 + 3rd/docopt/docopt_value.h | 340 + 3rd/json/json.hpp | 22815 ++++++++++++++++ CMakeLists.txt | 73 + Dockerfile | 50 + README.md | 33 + cmd/pktvisor/pktvisor.go | 655 + entry.sh | 11 + src/dns.h | 119 + src/main.cpp | 417 + src/metrics.cpp | 649 + src/metrics.h | 276 + src/pktvisor.h | 10 + src/querypairmgr.cpp | 38 + src/querypairmgr.h | 46 + src/tcpsession.cpp | 159 + src/tcpsession.h | 102 + src/timer.h | 92 + src/utils.cpp | 107 + src/utils.h | 22 + src/version.h | 4 + tests/fixtures/dns_ipv4_tcp.pcap | Bin 0 -> 198032 bytes tests/fixtures/dns_ipv4_udp.pcap | Bin 0 -> 15584 bytes tests/fixtures/dns_ipv6_tcp.pcap | Bin 0 -> 206504 bytes tests/fixtures/dns_ipv6_udp.pcap | Bin 0 -> 18004 bytes tests/main.cpp | 17 + tests/test_parse_pcap.cpp | 235 + tests/test_sketches.cpp | 100 + tests/test_utils.cpp | 74 + 62 files changed, 58321 insertions(+) create mode 100644 .clang-format create mode 100644 .gitignore create mode 100644 3rd/catch/catch2/catch.hpp create mode 100644 3rd/cpp-httplib/httplib.h create mode 100644 3rd/datasketches/datasketches/MurmurHash3.h create mode 100644 3rd/datasketches/datasketches/cpc/compression_data.hpp create mode 100644 3rd/datasketches/datasketches/cpc/counter_of_zeros.hpp create mode 100644 3rd/datasketches/datasketches/cpc/cpc_common.hpp create mode 100644 3rd/datasketches/datasketches/cpc/cpc_compressor.hpp create mode 100644 3rd/datasketches/datasketches/cpc/cpc_compressor_impl.hpp create mode 100644 3rd/datasketches/datasketches/cpc/cpc_confidence.hpp create mode 100644 3rd/datasketches/datasketches/cpc/cpc_sketch.hpp create mode 100644 3rd/datasketches/datasketches/cpc/cpc_sketch_impl.hpp create mode 100644 3rd/datasketches/datasketches/cpc/cpc_union.hpp create mode 100644 3rd/datasketches/datasketches/cpc/cpc_union_impl.hpp create mode 100644 3rd/datasketches/datasketches/cpc/cpc_util.hpp create mode 100644 3rd/datasketches/datasketches/cpc/icon_estimator.hpp create mode 100644 3rd/datasketches/datasketches/cpc/inv_pow_2_tab.hpp create mode 100644 3rd/datasketches/datasketches/cpc/kxp_byte_lookup.hpp create mode 100644 3rd/datasketches/datasketches/cpc/u32_table.hpp create mode 100644 3rd/datasketches/datasketches/cpc/u32_table_impl.hpp create mode 100644 3rd/datasketches/datasketches/fi/frequent_items_sketch.hpp create mode 100644 3rd/datasketches/datasketches/fi/reverse_purge_hash_map.hpp create mode 100644 3rd/datasketches/datasketches/kll/kll_helper.hpp create mode 100644 3rd/datasketches/datasketches/kll/kll_helper_impl.hpp create mode 100644 3rd/datasketches/datasketches/kll/kll_quantile_calculator.hpp create mode 100644 3rd/datasketches/datasketches/kll/kll_quantile_calculator_impl.hpp create mode 100644 3rd/datasketches/datasketches/kll/kll_sketch.hpp create mode 100644 3rd/datasketches/datasketches/kll/kll_sketch_impl.hpp create mode 100644 3rd/datasketches/datasketches/serde.hpp create mode 100644 3rd/docopt/docopt.cpp create mode 100644 3rd/docopt/docopt.h create mode 100644 3rd/docopt/docopt_private.h create mode 100644 3rd/docopt/docopt_util.h create mode 100644 3rd/docopt/docopt_value.h create mode 100644 3rd/json/json.hpp create mode 100644 CMakeLists.txt create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 cmd/pktvisor/pktvisor.go create mode 100755 entry.sh create mode 100644 src/dns.h create mode 100644 src/main.cpp create mode 100644 src/metrics.cpp create mode 100644 src/metrics.h create mode 100644 src/pktvisor.h create mode 100644 src/querypairmgr.cpp create mode 100644 src/querypairmgr.h create mode 100644 src/tcpsession.cpp create mode 100644 src/tcpsession.h create mode 100644 src/timer.h create mode 100644 src/utils.cpp create mode 100644 src/utils.h create mode 100644 src/version.h create mode 100644 tests/fixtures/dns_ipv4_tcp.pcap create mode 100644 tests/fixtures/dns_ipv4_udp.pcap create mode 100644 tests/fixtures/dns_ipv6_tcp.pcap create mode 100644 tests/fixtures/dns_ipv6_udp.pcap create mode 100644 tests/main.cpp create mode 100644 tests/test_parse_pcap.cpp create mode 100644 tests/test_sketches.cpp create mode 100644 tests/test_utils.cpp diff --git a/.clang-format b/.clang-format new file mode 100644 index 000000000..785584f93 --- /dev/null +++ b/.clang-format @@ -0,0 +1,40 @@ +--- +Language: Cpp + +# defaults (if not documented) can be found here: +# https://github.com/llvm-mirror/clang/blob/master/lib/Format/Format.cpp +BasedOnStyle: WebKit + +# make it look more Linuxy +AllowShortFunctionsOnASingleLine: None +PointerAlignment: Right + +# custom brace breaks, but defaults based on "Linux" style. +BreakBeforeBraces: 'Custom' +BraceWrapping: { + AfterClass: 'true' # linux default + AfterFunction: 'true' # linux default + AfterNamespace: 'false' # non-linux + # everything else is same default as linux style. + AfterControlStatement: 'false' + AfterEnum: 'false' + AfterObjCDeclaration: 'false' + AfterStruct: 'false' + AfterUnion: 'false' + BeforeCatch: 'false' + BeforeElse: 'false' + IndentBraces: 'false' +} + +AlignTrailingComments: true + +# had to re-set this after changing BraceWrapping. WebKit default is 0. +ColumnLimit: 0 + +# don't inner (nested) namespaces. +NamespaceIndentation: None + +# modernize +Standard: Cpp11 +Cpp11BracedListStyle: true +SpaceBeforeCpp11BracedList: false diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..8f5f082f3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.idea/ +cmake-build-*/ +cmd/.idea/ diff --git a/3rd/catch/catch2/catch.hpp b/3rd/catch/catch2/catch.hpp new file mode 100644 index 000000000..bdc2f74a9 --- /dev/null +++ b/3rd/catch/catch2/catch.hpp @@ -0,0 +1,13359 @@ +/* + * Catch v2.3.0 + * Generated: 2018-07-23 10:09:14.936841 + * ---------------------------------------------------------- + * This file has been merged from multiple headers. Please don't edit it directly + * Copyright (c) 2018 Two Blue Cubes Ltd. All rights reserved. + * + * Distributed under the Boost Software License, Version 1.0. (See accompanying + * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + */ +#ifndef TWOBLUECUBES_SINGLE_INCLUDE_CATCH_HPP_INCLUDED +#define TWOBLUECUBES_SINGLE_INCLUDE_CATCH_HPP_INCLUDED +// start catch.hpp + + +#define CATCH_VERSION_MAJOR 2 +#define CATCH_VERSION_MINOR 3 +#define CATCH_VERSION_PATCH 0 + +#ifdef __clang__ +# pragma clang system_header +#elif defined __GNUC__ +# pragma GCC system_header +#endif + +// start catch_suppress_warnings.h + +#ifdef __clang__ +# ifdef __ICC // icpc defines the __clang__ macro +# pragma warning(push) +# pragma warning(disable: 161 1682) +# else // __ICC +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wpadded" +# pragma clang diagnostic ignored "-Wswitch-enum" +# pragma clang diagnostic ignored "-Wcovered-switch-default" +# endif +#elif defined __GNUC__ + // GCC likes to warn on REQUIREs, and we cannot suppress them + // locally because g++'s support for _Pragma is lacking in older, + // still supported, versions +# pragma GCC diagnostic ignored "-Wparentheses" +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic ignored "-Wpadded" +#endif +// end catch_suppress_warnings.h +#if defined(CATCH_CONFIG_MAIN) || defined(CATCH_CONFIG_RUNNER) +# define CATCH_IMPL +# define CATCH_CONFIG_ALL_PARTS +#endif + +// In the impl file, we want to have access to all parts of the headers +// Can also be used to sanely support PCHs +#if defined(CATCH_CONFIG_ALL_PARTS) +# define CATCH_CONFIG_EXTERNAL_INTERFACES +# if defined(CATCH_CONFIG_DISABLE_MATCHERS) +# undef CATCH_CONFIG_DISABLE_MATCHERS +# endif +# if !defined(CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER) +# define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER +# endif +#endif + +#if !defined(CATCH_CONFIG_IMPL_ONLY) +// start catch_platform.h + +#ifdef __APPLE__ +# include +# if TARGET_OS_OSX == 1 +# define CATCH_PLATFORM_MAC +# elif TARGET_OS_IPHONE == 1 +# define CATCH_PLATFORM_IPHONE +# endif + +#elif defined(linux) || defined(__linux) || defined(__linux__) +# define CATCH_PLATFORM_LINUX + +#elif defined(WIN32) || defined(__WIN32__) || defined(_WIN32) || defined(_MSC_VER) || defined(__MINGW32__) +# define CATCH_PLATFORM_WINDOWS +#endif + +// end catch_platform.h + +#ifdef CATCH_IMPL +# ifndef CLARA_CONFIG_MAIN +# define CLARA_CONFIG_MAIN_NOT_DEFINED +# define CLARA_CONFIG_MAIN +# endif +#endif + +// start catch_user_interfaces.h + +namespace Catch { + unsigned int rngSeed(); +} + +// end catch_user_interfaces.h +// start catch_tag_alias_autoregistrar.h + +// start catch_common.h + +// start catch_compiler_capabilities.h + +// Detect a number of compiler features - by compiler +// The following features are defined: +// +// CATCH_CONFIG_COUNTER : is the __COUNTER__ macro supported? +// CATCH_CONFIG_WINDOWS_SEH : is Windows SEH supported? +// CATCH_CONFIG_POSIX_SIGNALS : are POSIX signals supported? +// **************** +// Note to maintainers: if new toggles are added please document them +// in configuration.md, too +// **************** + +// In general each macro has a _NO_ form +// (e.g. CATCH_CONFIG_NO_POSIX_SIGNALS) which disables the feature. +// Many features, at point of detection, define an _INTERNAL_ macro, so they +// can be combined, en-mass, with the _NO_ forms later. + +#ifdef __cplusplus + +# if __cplusplus >= 201402L +# define CATCH_CPP14_OR_GREATER +# endif + +# if __cplusplus >= 201703L +# define CATCH_CPP17_OR_GREATER +# endif + +#endif + +#if defined(CATCH_CPP17_OR_GREATER) +# define CATCH_INTERNAL_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS +#endif + +#ifdef __clang__ + +# define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + _Pragma( "clang diagnostic push" ) \ + _Pragma( "clang diagnostic ignored \"-Wexit-time-destructors\"" ) \ + _Pragma( "clang diagnostic ignored \"-Wglobal-constructors\"") +# define CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS \ + _Pragma( "clang diagnostic pop" ) + +# define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS \ + _Pragma( "clang diagnostic push" ) \ + _Pragma( "clang diagnostic ignored \"-Wparentheses\"" ) +# define CATCH_INTERNAL_UNSUPPRESS_PARENTHESES_WARNINGS \ + _Pragma( "clang diagnostic pop" ) + +# define CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS \ + _Pragma( "clang diagnostic push" ) \ + _Pragma( "clang diagnostic ignored \"-Wunused-variable\"" ) +# define CATCH_INTERNAL_UNSUPPRESS_UNUSED_WARNINGS \ + _Pragma( "clang diagnostic pop" ) + +#endif // __clang__ + +//////////////////////////////////////////////////////////////////////////////// +// Assume that non-Windows platforms support posix signals by default +#if !defined(CATCH_PLATFORM_WINDOWS) + #define CATCH_INTERNAL_CONFIG_POSIX_SIGNALS +#endif + +//////////////////////////////////////////////////////////////////////////////// +// We know some environments not to support full POSIX signals +#if defined(__CYGWIN__) || defined(__QNX__) || defined(__EMSCRIPTEN__) || defined(__DJGPP__) + #define CATCH_INTERNAL_CONFIG_NO_POSIX_SIGNALS +#endif + +#ifdef __OS400__ +# define CATCH_INTERNAL_CONFIG_NO_POSIX_SIGNALS +# define CATCH_CONFIG_COLOUR_NONE +#endif + +//////////////////////////////////////////////////////////////////////////////// +// Android somehow still does not support std::to_string +#if defined(__ANDROID__) +# define CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING +#endif + +//////////////////////////////////////////////////////////////////////////////// +// Not all Windows environments support SEH properly +#if defined(__MINGW32__) +# define CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH +#endif + +//////////////////////////////////////////////////////////////////////////////// +// PS4 +#if defined(__ORBIS__) +# define CATCH_INTERNAL_CONFIG_NO_NEW_CAPTURE +#endif + +//////////////////////////////////////////////////////////////////////////////// +// Cygwin +#ifdef __CYGWIN__ + +// Required for some versions of Cygwin to declare gettimeofday +// see: http://stackoverflow.com/questions/36901803/gettimeofday-not-declared-in-this-scope-cygwin +# define _BSD_SOURCE + +#endif // __CYGWIN__ + +//////////////////////////////////////////////////////////////////////////////// +// Visual C++ +#ifdef _MSC_VER + +# if _MSC_VER >= 1900 // Visual Studio 2015 or newer +# define CATCH_INTERNAL_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS +# endif + +// Universal Windows platform does not support SEH +// Or console colours (or console at all...) +# if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP) +# define CATCH_CONFIG_COLOUR_NONE +# else +# define CATCH_INTERNAL_CONFIG_WINDOWS_SEH +# endif + +#endif // _MSC_VER + +//////////////////////////////////////////////////////////////////////////////// + +// DJGPP +#ifdef __DJGPP__ +# define CATCH_INTERNAL_CONFIG_NO_WCHAR +#endif // __DJGPP__ + +//////////////////////////////////////////////////////////////////////////////// + +// Use of __COUNTER__ is suppressed during code analysis in +// CLion/AppCode 2017.2.x and former, because __COUNTER__ is not properly +// handled by it. +// Otherwise all supported compilers support COUNTER macro, +// but user still might want to turn it off +#if ( !defined(__JETBRAINS_IDE__) || __JETBRAINS_IDE__ >= 20170300L ) + #define CATCH_INTERNAL_CONFIG_COUNTER +#endif + +#if defined(CATCH_INTERNAL_CONFIG_COUNTER) && !defined(CATCH_CONFIG_NO_COUNTER) && !defined(CATCH_CONFIG_COUNTER) +# define CATCH_CONFIG_COUNTER +#endif +#if defined(CATCH_INTERNAL_CONFIG_WINDOWS_SEH) && !defined(CATCH_CONFIG_NO_WINDOWS_SEH) && !defined(CATCH_CONFIG_WINDOWS_SEH) && !defined(CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH) +# define CATCH_CONFIG_WINDOWS_SEH +#endif +// This is set by default, because we assume that unix compilers are posix-signal-compatible by default. +#if defined(CATCH_INTERNAL_CONFIG_POSIX_SIGNALS) && !defined(CATCH_INTERNAL_CONFIG_NO_POSIX_SIGNALS) && !defined(CATCH_CONFIG_NO_POSIX_SIGNALS) && !defined(CATCH_CONFIG_POSIX_SIGNALS) +# define CATCH_CONFIG_POSIX_SIGNALS +#endif +// This is set by default, because we assume that compilers with no wchar_t support are just rare exceptions. +#if !defined(CATCH_INTERNAL_CONFIG_NO_WCHAR) && !defined(CATCH_CONFIG_NO_WCHAR) && !defined(CATCH_CONFIG_WCHAR) +# define CATCH_CONFIG_WCHAR +#endif + +#if !defined(CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING) && !defined(CATCH_CONFIG_NO_CPP11_TO_STRING) && !defined(CATCH_CONFIG_CPP11_TO_STRING) +# define CATCH_CONFIG_CPP11_TO_STRING +#endif + +#if defined(CATCH_INTERNAL_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS) && !defined(CATCH_CONFIG_NO_CPP17_UNCAUGHT_EXCEPTIONS) && !defined(CATCH_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS) +# define CATCH_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS +#endif + +#if defined(CATCH_CONFIG_EXPERIMENTAL_REDIRECT) +# define CATCH_INTERNAL_CONFIG_NEW_CAPTURE +#endif + +#if defined(CATCH_INTERNAL_CONFIG_NEW_CAPTURE) && !defined(CATCH_INTERNAL_CONFIG_NO_NEW_CAPTURE) && !defined(CATCH_CONFIG_NO_NEW_CAPTURE) && !defined(CATCH_CONFIG_NEW_CAPTURE) +# define CATCH_CONFIG_NEW_CAPTURE +#endif + +#if !defined(CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS +# define CATCH_INTERNAL_UNSUPPRESS_PARENTHESES_WARNINGS +#endif +#if !defined(CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS +# define CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS +#endif +#if !defined(CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS +# define CATCH_INTERNAL_UNSUPPRESS_UNUSED_WARNINGS +#endif + +// end catch_compiler_capabilities.h +#define INTERNAL_CATCH_UNIQUE_NAME_LINE2( name, line ) name##line +#define INTERNAL_CATCH_UNIQUE_NAME_LINE( name, line ) INTERNAL_CATCH_UNIQUE_NAME_LINE2( name, line ) +#ifdef CATCH_CONFIG_COUNTER +# define INTERNAL_CATCH_UNIQUE_NAME( name ) INTERNAL_CATCH_UNIQUE_NAME_LINE( name, __COUNTER__ ) +#else +# define INTERNAL_CATCH_UNIQUE_NAME( name ) INTERNAL_CATCH_UNIQUE_NAME_LINE( name, __LINE__ ) +#endif + +#include +#include +#include + +namespace Catch { + + struct CaseSensitive { enum Choice { + Yes, + No + }; }; + + class NonCopyable { + NonCopyable( NonCopyable const& ) = delete; + NonCopyable( NonCopyable && ) = delete; + NonCopyable& operator = ( NonCopyable const& ) = delete; + NonCopyable& operator = ( NonCopyable && ) = delete; + + protected: + NonCopyable(); + virtual ~NonCopyable(); + }; + + struct SourceLineInfo { + + SourceLineInfo() = delete; + SourceLineInfo( char const* _file, std::size_t _line ) noexcept + : file( _file ), + line( _line ) + {} + + SourceLineInfo( SourceLineInfo const& other ) = default; + SourceLineInfo( SourceLineInfo && ) = default; + SourceLineInfo& operator = ( SourceLineInfo const& ) = default; + SourceLineInfo& operator = ( SourceLineInfo && ) = default; + + bool empty() const noexcept; + bool operator == ( SourceLineInfo const& other ) const noexcept; + bool operator < ( SourceLineInfo const& other ) const noexcept; + + char const* file; + std::size_t line; + }; + + std::ostream& operator << ( std::ostream& os, SourceLineInfo const& info ); + + // Use this in variadic streaming macros to allow + // >> +StreamEndStop + // as well as + // >> stuff +StreamEndStop + struct StreamEndStop { + std::string operator+() const; + }; + template + T const& operator + ( T const& value, StreamEndStop ) { + return value; + } +} + +#define CATCH_INTERNAL_LINEINFO \ + ::Catch::SourceLineInfo( __FILE__, static_cast( __LINE__ ) ) + +// end catch_common.h +namespace Catch { + + struct RegistrarForTagAliases { + RegistrarForTagAliases( char const* alias, char const* tag, SourceLineInfo const& lineInfo ); + }; + +} // end namespace Catch + +#define CATCH_REGISTER_TAG_ALIAS( alias, spec ) \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + namespace{ Catch::RegistrarForTagAliases INTERNAL_CATCH_UNIQUE_NAME( AutoRegisterTagAlias )( alias, spec, CATCH_INTERNAL_LINEINFO ); } \ + CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS + +// end catch_tag_alias_autoregistrar.h +// start catch_test_registry.h + +// start catch_interfaces_testcase.h + +#include +#include + +namespace Catch { + + class TestSpec; + + struct ITestInvoker { + virtual void invoke () const = 0; + virtual ~ITestInvoker(); + }; + + using ITestCasePtr = std::shared_ptr; + + class TestCase; + struct IConfig; + + struct ITestCaseRegistry { + virtual ~ITestCaseRegistry(); + virtual std::vector const& getAllTests() const = 0; + virtual std::vector const& getAllTestsSorted( IConfig const& config ) const = 0; + }; + + bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ); + std::vector filterTests( std::vector const& testCases, TestSpec const& testSpec, IConfig const& config ); + std::vector const& getAllTestCasesSorted( IConfig const& config ); + +} + +// end catch_interfaces_testcase.h +// start catch_stringref.h + +#include +#include +#include + +namespace Catch { + + class StringData; + + /// A non-owning string class (similar to the forthcoming std::string_view) + /// Note that, because a StringRef may be a substring of another string, + /// it may not be null terminated. c_str() must return a null terminated + /// string, however, and so the StringRef will internally take ownership + /// (taking a copy), if necessary. In theory this ownership is not externally + /// visible - but it does mean (substring) StringRefs should not be shared between + /// threads. + class StringRef { + public: + using size_type = std::size_t; + + private: + friend struct StringRefTestAccess; + + char const* m_start; + size_type m_size; + + char* m_data = nullptr; + + void takeOwnership(); + + static constexpr char const* const s_empty = ""; + + public: // construction/ assignment + StringRef() noexcept + : StringRef( s_empty, 0 ) + {} + + StringRef( StringRef const& other ) noexcept + : m_start( other.m_start ), + m_size( other.m_size ) + {} + + StringRef( StringRef&& other ) noexcept + : m_start( other.m_start ), + m_size( other.m_size ), + m_data( other.m_data ) + { + other.m_data = nullptr; + } + + StringRef( char const* rawChars ) noexcept; + + StringRef( char const* rawChars, size_type size ) noexcept + : m_start( rawChars ), + m_size( size ) + {} + + StringRef( std::string const& stdString ) noexcept + : m_start( stdString.c_str() ), + m_size( stdString.size() ) + {} + + ~StringRef() noexcept { + delete[] m_data; + } + + auto operator = ( StringRef const &other ) noexcept -> StringRef& { + delete[] m_data; + m_data = nullptr; + m_start = other.m_start; + m_size = other.m_size; + return *this; + } + + operator std::string() const; + + void swap( StringRef& other ) noexcept; + + public: // operators + auto operator == ( StringRef const& other ) const noexcept -> bool; + auto operator != ( StringRef const& other ) const noexcept -> bool; + + auto operator[] ( size_type index ) const noexcept -> char; + + public: // named queries + auto empty() const noexcept -> bool { + return m_size == 0; + } + auto size() const noexcept -> size_type { + return m_size; + } + + auto numberOfCharacters() const noexcept -> size_type; + auto c_str() const -> char const*; + + public: // substrings and searches + auto substr( size_type start, size_type size ) const noexcept -> StringRef; + + // Returns the current start pointer. + // Note that the pointer can change when if the StringRef is a substring + auto currentData() const noexcept -> char const*; + + private: // ownership queries - may not be consistent between calls + auto isOwned() const noexcept -> bool; + auto isSubstring() const noexcept -> bool; + }; + + auto operator + ( StringRef const& lhs, StringRef const& rhs ) -> std::string; + auto operator + ( StringRef const& lhs, char const* rhs ) -> std::string; + auto operator + ( char const* lhs, StringRef const& rhs ) -> std::string; + + auto operator += ( std::string& lhs, StringRef const& sr ) -> std::string&; + auto operator << ( std::ostream& os, StringRef const& sr ) -> std::ostream&; + + inline auto operator "" _sr( char const* rawChars, std::size_t size ) noexcept -> StringRef { + return StringRef( rawChars, size ); + } + +} // namespace Catch + +// end catch_stringref.h +namespace Catch { + +template +class TestInvokerAsMethod : public ITestInvoker { + void (C::*m_testAsMethod)(); +public: + TestInvokerAsMethod( void (C::*testAsMethod)() ) noexcept : m_testAsMethod( testAsMethod ) {} + + void invoke() const override { + C obj; + (obj.*m_testAsMethod)(); + } +}; + +auto makeTestInvoker( void(*testAsFunction)() ) noexcept -> ITestInvoker*; + +template +auto makeTestInvoker( void (C::*testAsMethod)() ) noexcept -> ITestInvoker* { + return new(std::nothrow) TestInvokerAsMethod( testAsMethod ); +} + +struct NameAndTags { + NameAndTags( StringRef const& name_ = StringRef(), StringRef const& tags_ = StringRef() ) noexcept; + StringRef name; + StringRef tags; +}; + +struct AutoReg : NonCopyable { + AutoReg( ITestInvoker* invoker, SourceLineInfo const& lineInfo, StringRef const& classOrMethod, NameAndTags const& nameAndTags ) noexcept; + ~AutoReg(); +}; + +} // end namespace Catch + +#define INTERNAL_CATCH_EXPAND1(param) INTERNAL_CATCH_EXPAND2(param) +#define INTERNAL_CATCH_EXPAND2(...) INTERNAL_CATCH_NO## __VA_ARGS__ +#define INTERNAL_CATCH_DEF(...) INTERNAL_CATCH_DEF __VA_ARGS__ +#define INTERNAL_CATCH_NOINTERNAL_CATCH_DEF + +#if defined(CATCH_CONFIG_DISABLE) + #define INTERNAL_CATCH_TESTCASE_NO_REGISTRATION( TestName, ... ) \ + static void TestName() + #define INTERNAL_CATCH_TESTCASE_METHOD_NO_REGISTRATION( TestName, ClassName, ... ) \ + namespace{ \ + struct TestName : INTERNAL_CATCH_EXPAND1(INTERNAL_CATCH_DEF ClassName) { \ + void test(); \ + }; \ + } \ + void TestName::test() + +#endif + + /////////////////////////////////////////////////////////////////////////////// + #define INTERNAL_CATCH_TESTCASE2( TestName, ... ) \ + static void TestName(); \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + namespace{ Catch::AutoReg INTERNAL_CATCH_UNIQUE_NAME( autoRegistrar )( Catch::makeTestInvoker( &TestName ), CATCH_INTERNAL_LINEINFO, Catch::StringRef(), Catch::NameAndTags{ __VA_ARGS__ } ); } /* NOLINT */ \ + CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS \ + static void TestName() + #define INTERNAL_CATCH_TESTCASE( ... ) \ + INTERNAL_CATCH_TESTCASE2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ), __VA_ARGS__ ) + + /////////////////////////////////////////////////////////////////////////////// + #define INTERNAL_CATCH_METHOD_AS_TEST_CASE( QualifiedMethod, ... ) \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + namespace{ Catch::AutoReg INTERNAL_CATCH_UNIQUE_NAME( autoRegistrar )( Catch::makeTestInvoker( &QualifiedMethod ), CATCH_INTERNAL_LINEINFO, "&" #QualifiedMethod, Catch::NameAndTags{ __VA_ARGS__ } ); } /* NOLINT */ \ + CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS + + /////////////////////////////////////////////////////////////////////////////// + #define INTERNAL_CATCH_TEST_CASE_METHOD2( TestName, ClassName, ... )\ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + namespace{ \ + struct TestName : INTERNAL_CATCH_EXPAND1(INTERNAL_CATCH_DEF ClassName) { \ + void test(); \ + }; \ + Catch::AutoReg INTERNAL_CATCH_UNIQUE_NAME( autoRegistrar ) ( Catch::makeTestInvoker( &TestName::test ), CATCH_INTERNAL_LINEINFO, #ClassName, Catch::NameAndTags{ __VA_ARGS__ } ); /* NOLINT */ \ + } \ + CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS \ + void TestName::test() + #define INTERNAL_CATCH_TEST_CASE_METHOD( ClassName, ... ) \ + INTERNAL_CATCH_TEST_CASE_METHOD2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ), ClassName, __VA_ARGS__ ) + + /////////////////////////////////////////////////////////////////////////////// + #define INTERNAL_CATCH_REGISTER_TESTCASE( Function, ... ) \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + Catch::AutoReg INTERNAL_CATCH_UNIQUE_NAME( autoRegistrar )( Catch::makeTestInvoker( Function ), CATCH_INTERNAL_LINEINFO, Catch::StringRef(), Catch::NameAndTags{ __VA_ARGS__ } ); /* NOLINT */ \ + CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS + +// end catch_test_registry.h +// start catch_capture.hpp + +// start catch_assertionhandler.h + +// start catch_assertioninfo.h + +// start catch_result_type.h + +namespace Catch { + + // ResultWas::OfType enum + struct ResultWas { enum OfType { + Unknown = -1, + Ok = 0, + Info = 1, + Warning = 2, + + FailureBit = 0x10, + + ExpressionFailed = FailureBit | 1, + ExplicitFailure = FailureBit | 2, + + Exception = 0x100 | FailureBit, + + ThrewException = Exception | 1, + DidntThrowException = Exception | 2, + + FatalErrorCondition = 0x200 | FailureBit + + }; }; + + bool isOk( ResultWas::OfType resultType ); + bool isJustInfo( int flags ); + + // ResultDisposition::Flags enum + struct ResultDisposition { enum Flags { + Normal = 0x01, + + ContinueOnFailure = 0x02, // Failures fail test, but execution continues + FalseTest = 0x04, // Prefix expression with ! + SuppressFail = 0x08 // Failures are reported but do not fail the test + }; }; + + ResultDisposition::Flags operator | ( ResultDisposition::Flags lhs, ResultDisposition::Flags rhs ); + + bool shouldContinueOnFailure( int flags ); + inline bool isFalseTest( int flags ) { return ( flags & ResultDisposition::FalseTest ) != 0; } + bool shouldSuppressFailure( int flags ); + +} // end namespace Catch + +// end catch_result_type.h +namespace Catch { + + struct AssertionInfo + { + StringRef macroName; + SourceLineInfo lineInfo; + StringRef capturedExpression; + ResultDisposition::Flags resultDisposition; + + // We want to delete this constructor but a compiler bug in 4.8 means + // the struct is then treated as non-aggregate + //AssertionInfo() = delete; + }; + +} // end namespace Catch + +// end catch_assertioninfo.h +// start catch_decomposer.h + +// start catch_tostring.h + +#include +#include +#include +#include +// start catch_stream.h + +#include +#include +#include + +namespace Catch { + + std::ostream& cout(); + std::ostream& cerr(); + std::ostream& clog(); + + class StringRef; + + struct IStream { + virtual ~IStream(); + virtual std::ostream& stream() const = 0; + }; + + auto makeStream( StringRef const &filename ) -> IStream const*; + + class ReusableStringStream { + std::size_t m_index; + std::ostream* m_oss; + public: + ReusableStringStream(); + ~ReusableStringStream(); + + auto str() const -> std::string; + + template + auto operator << ( T const& value ) -> ReusableStringStream& { + *m_oss << value; + return *this; + } + auto get() -> std::ostream& { return *m_oss; } + + static void cleanup(); + }; +} + +// end catch_stream.h + +#ifdef __OBJC__ +// start catch_objc_arc.hpp + +#import + +#ifdef __has_feature +#define CATCH_ARC_ENABLED __has_feature(objc_arc) +#else +#define CATCH_ARC_ENABLED 0 +#endif + +void arcSafeRelease( NSObject* obj ); +id performOptionalSelector( id obj, SEL sel ); + +#if !CATCH_ARC_ENABLED +inline void arcSafeRelease( NSObject* obj ) { + [obj release]; +} +inline id performOptionalSelector( id obj, SEL sel ) { + if( [obj respondsToSelector: sel] ) + return [obj performSelector: sel]; + return nil; +} +#define CATCH_UNSAFE_UNRETAINED +#define CATCH_ARC_STRONG +#else +inline void arcSafeRelease( NSObject* ){} +inline id performOptionalSelector( id obj, SEL sel ) { +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Warc-performSelector-leaks" +#endif + if( [obj respondsToSelector: sel] ) + return [obj performSelector: sel]; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + return nil; +} +#define CATCH_UNSAFE_UNRETAINED __unsafe_unretained +#define CATCH_ARC_STRONG __strong +#endif + +// end catch_objc_arc.hpp +#endif + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable:4180) // We attempt to stream a function (address) by const&, which MSVC complains about but is harmless +#endif + +// We need a dummy global operator<< so we can bring it into Catch namespace later +struct Catch_global_namespace_dummy {}; +std::ostream& operator<<(std::ostream&, Catch_global_namespace_dummy); + +namespace Catch { + // Bring in operator<< from global namespace into Catch namespace + using ::operator<<; + + namespace Detail { + + extern const std::string unprintableString; + + std::string rawMemoryToString( const void *object, std::size_t size ); + + template + std::string rawMemoryToString( const T& object ) { + return rawMemoryToString( &object, sizeof(object) ); + } + + template + class IsStreamInsertable { + template + static auto test(int) + -> decltype(std::declval() << std::declval(), std::true_type()); + + template + static auto test(...)->std::false_type; + + public: + static const bool value = decltype(test(0))::value; + }; + + template + std::string convertUnknownEnumToString( E e ); + + template + typename std::enable_if< + !std::is_enum::value && !std::is_base_of::value, + std::string>::type convertUnstreamable( T const& ) { + return Detail::unprintableString; + } + template + typename std::enable_if< + !std::is_enum::value && std::is_base_of::value, + std::string>::type convertUnstreamable(T const& ex) { + return ex.what(); + } + + template + typename std::enable_if< + std::is_enum::value + , std::string>::type convertUnstreamable( T const& value ) { + return convertUnknownEnumToString( value ); + } + +#if defined(_MANAGED) + //! Convert a CLR string to a utf8 std::string + template + std::string clrReferenceToString( T^ ref ) { + if (ref == nullptr) + return std::string("null"); + auto bytes = System::Text::Encoding::UTF8->GetBytes(ref->ToString()); + cli::pin_ptr p = &bytes[0]; + return std::string(reinterpret_cast(p), bytes->Length); + } +#endif + + } // namespace Detail + + // If we decide for C++14, change these to enable_if_ts + template + struct StringMaker { + template + static + typename std::enable_if<::Catch::Detail::IsStreamInsertable::value, std::string>::type + convert(const Fake& value) { + ReusableStringStream rss; + // NB: call using the function-like syntax to avoid ambiguity with + // user-defined templated operator<< under clang. + rss.operator<<(value); + return rss.str(); + } + + template + static + typename std::enable_if::value, std::string>::type + convert( const Fake& value ) { +#if !defined(CATCH_CONFIG_FALLBACK_STRINGIFIER) + return Detail::convertUnstreamable(value); +#else + return CATCH_CONFIG_FALLBACK_STRINGIFIER(value); +#endif + } + }; + + namespace Detail { + + // This function dispatches all stringification requests inside of Catch. + // Should be preferably called fully qualified, like ::Catch::Detail::stringify + template + std::string stringify(const T& e) { + return ::Catch::StringMaker::type>::type>::convert(e); + } + + template + std::string convertUnknownEnumToString( E e ) { + return ::Catch::Detail::stringify(static_cast::type>(e)); + } + +#if defined(_MANAGED) + template + std::string stringify( T^ e ) { + return ::Catch::StringMaker::convert(e); + } +#endif + + } // namespace Detail + + // Some predefined specializations + + template<> + struct StringMaker { + static std::string convert(const std::string& str); + }; +#ifdef CATCH_CONFIG_WCHAR + template<> + struct StringMaker { + static std::string convert(const std::wstring& wstr); + }; +#endif + + template<> + struct StringMaker { + static std::string convert(char const * str); + }; + template<> + struct StringMaker { + static std::string convert(char * str); + }; + +#ifdef CATCH_CONFIG_WCHAR + template<> + struct StringMaker { + static std::string convert(wchar_t const * str); + }; + template<> + struct StringMaker { + static std::string convert(wchar_t * str); + }; +#endif + + // TBD: Should we use `strnlen` to ensure that we don't go out of the buffer, + // while keeping string semantics? + template + struct StringMaker { + static std::string convert(char const* str) { + return ::Catch::Detail::stringify(std::string{ str }); + } + }; + template + struct StringMaker { + static std::string convert(signed char const* str) { + return ::Catch::Detail::stringify(std::string{ reinterpret_cast(str) }); + } + }; + template + struct StringMaker { + static std::string convert(unsigned char const* str) { + return ::Catch::Detail::stringify(std::string{ reinterpret_cast(str) }); + } + }; + + template<> + struct StringMaker { + static std::string convert(int value); + }; + template<> + struct StringMaker { + static std::string convert(long value); + }; + template<> + struct StringMaker { + static std::string convert(long long value); + }; + template<> + struct StringMaker { + static std::string convert(unsigned int value); + }; + template<> + struct StringMaker { + static std::string convert(unsigned long value); + }; + template<> + struct StringMaker { + static std::string convert(unsigned long long value); + }; + + template<> + struct StringMaker { + static std::string convert(bool b); + }; + + template<> + struct StringMaker { + static std::string convert(char c); + }; + template<> + struct StringMaker { + static std::string convert(signed char c); + }; + template<> + struct StringMaker { + static std::string convert(unsigned char c); + }; + + template<> + struct StringMaker { + static std::string convert(std::nullptr_t); + }; + + template<> + struct StringMaker { + static std::string convert(float value); + }; + template<> + struct StringMaker { + static std::string convert(double value); + }; + + template + struct StringMaker { + template + static std::string convert(U* p) { + if (p) { + return ::Catch::Detail::rawMemoryToString(p); + } else { + return "nullptr"; + } + } + }; + + template + struct StringMaker { + static std::string convert(R C::* p) { + if (p) { + return ::Catch::Detail::rawMemoryToString(p); + } else { + return "nullptr"; + } + } + }; + +#if defined(_MANAGED) + template + struct StringMaker { + static std::string convert( T^ ref ) { + return ::Catch::Detail::clrReferenceToString(ref); + } + }; +#endif + + namespace Detail { + template + std::string rangeToString(InputIterator first, InputIterator last) { + ReusableStringStream rss; + rss << "{ "; + if (first != last) { + rss << ::Catch::Detail::stringify(*first); + for (++first; first != last; ++first) + rss << ", " << ::Catch::Detail::stringify(*first); + } + rss << " }"; + return rss.str(); + } + } + +#ifdef __OBJC__ + template<> + struct StringMaker { + static std::string convert(NSString * nsstring) { + if (!nsstring) + return "nil"; + return std::string("@") + [nsstring UTF8String]; + } + }; + template<> + struct StringMaker { + static std::string convert(NSObject* nsObject) { + return ::Catch::Detail::stringify([nsObject description]); + } + + }; + namespace Detail { + inline std::string stringify( NSString* nsstring ) { + return StringMaker::convert( nsstring ); + } + + } // namespace Detail +#endif // __OBJC__ + +} // namespace Catch + +////////////////////////////////////////////////////// +// Separate std-lib types stringification, so it can be selectively enabled +// This means that we do not bring in + +#if defined(CATCH_CONFIG_ENABLE_ALL_STRINGMAKERS) +# define CATCH_CONFIG_ENABLE_PAIR_STRINGMAKER +# define CATCH_CONFIG_ENABLE_TUPLE_STRINGMAKER +# define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER +#endif + +// Separate std::pair specialization +#if defined(CATCH_CONFIG_ENABLE_PAIR_STRINGMAKER) +#include +namespace Catch { + template + struct StringMaker > { + static std::string convert(const std::pair& pair) { + ReusableStringStream rss; + rss << "{ " + << ::Catch::Detail::stringify(pair.first) + << ", " + << ::Catch::Detail::stringify(pair.second) + << " }"; + return rss.str(); + } + }; +} +#endif // CATCH_CONFIG_ENABLE_PAIR_STRINGMAKER + +// Separate std::tuple specialization +#if defined(CATCH_CONFIG_ENABLE_TUPLE_STRINGMAKER) +#include +namespace Catch { + namespace Detail { + template< + typename Tuple, + std::size_t N = 0, + bool = (N < std::tuple_size::value) + > + struct TupleElementPrinter { + static void print(const Tuple& tuple, std::ostream& os) { + os << (N ? ", " : " ") + << ::Catch::Detail::stringify(std::get(tuple)); + TupleElementPrinter::print(tuple, os); + } + }; + + template< + typename Tuple, + std::size_t N + > + struct TupleElementPrinter { + static void print(const Tuple&, std::ostream&) {} + }; + + } + + template + struct StringMaker> { + static std::string convert(const std::tuple& tuple) { + ReusableStringStream rss; + rss << '{'; + Detail::TupleElementPrinter>::print(tuple, rss.get()); + rss << " }"; + return rss.str(); + } + }; +} +#endif // CATCH_CONFIG_ENABLE_TUPLE_STRINGMAKER + +namespace Catch { + struct not_this_one {}; // Tag type for detecting which begin/ end are being selected + + // Import begin/ end from std here so they are considered alongside the fallback (...) overloads in this namespace + using std::begin; + using std::end; + + not_this_one begin( ... ); + not_this_one end( ... ); + + template + struct is_range { + static const bool value = + !std::is_same())), not_this_one>::value && + !std::is_same())), not_this_one>::value; + }; + +#if defined(_MANAGED) // Managed types are never ranges + template + struct is_range { + static const bool value = false; + }; +#endif + + template + std::string rangeToString( Range const& range ) { + return ::Catch::Detail::rangeToString( begin( range ), end( range ) ); + } + + // Handle vector specially + template + std::string rangeToString( std::vector const& v ) { + ReusableStringStream rss; + rss << "{ "; + bool first = true; + for( bool b : v ) { + if( first ) + first = false; + else + rss << ", "; + rss << ::Catch::Detail::stringify( b ); + } + rss << " }"; + return rss.str(); + } + + template + struct StringMaker::value && !::Catch::Detail::IsStreamInsertable::value>::type> { + static std::string convert( R const& range ) { + return rangeToString( range ); + } + }; + + template + struct StringMaker { + static std::string convert(T const(&arr)[SZ]) { + return rangeToString(arr); + } + }; + +} // namespace Catch + +// Separate std::chrono::duration specialization +#if defined(CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER) +#include +#include +#include + +namespace Catch { + +template +struct ratio_string { + static std::string symbol(); +}; + +template +std::string ratio_string::symbol() { + Catch::ReusableStringStream rss; + rss << '[' << Ratio::num << '/' + << Ratio::den << ']'; + return rss.str(); +} +template <> +struct ratio_string { + static std::string symbol(); +}; +template <> +struct ratio_string { + static std::string symbol(); +}; +template <> +struct ratio_string { + static std::string symbol(); +}; +template <> +struct ratio_string { + static std::string symbol(); +}; +template <> +struct ratio_string { + static std::string symbol(); +}; +template <> +struct ratio_string { + static std::string symbol(); +}; + + //////////// + // std::chrono::duration specializations + template + struct StringMaker> { + static std::string convert(std::chrono::duration const& duration) { + ReusableStringStream rss; + rss << duration.count() << ' ' << ratio_string::symbol() << 's'; + return rss.str(); + } + }; + template + struct StringMaker>> { + static std::string convert(std::chrono::duration> const& duration) { + ReusableStringStream rss; + rss << duration.count() << " s"; + return rss.str(); + } + }; + template + struct StringMaker>> { + static std::string convert(std::chrono::duration> const& duration) { + ReusableStringStream rss; + rss << duration.count() << " m"; + return rss.str(); + } + }; + template + struct StringMaker>> { + static std::string convert(std::chrono::duration> const& duration) { + ReusableStringStream rss; + rss << duration.count() << " h"; + return rss.str(); + } + }; + + //////////// + // std::chrono::time_point specialization + // Generic time_point cannot be specialized, only std::chrono::time_point + template + struct StringMaker> { + static std::string convert(std::chrono::time_point const& time_point) { + return ::Catch::Detail::stringify(time_point.time_since_epoch()) + " since epoch"; + } + }; + // std::chrono::time_point specialization + template + struct StringMaker> { + static std::string convert(std::chrono::time_point const& time_point) { + auto converted = std::chrono::system_clock::to_time_t(time_point); + +#ifdef _MSC_VER + std::tm timeInfo = {}; + gmtime_s(&timeInfo, &converted); +#else + std::tm* timeInfo = std::gmtime(&converted); +#endif + + auto const timeStampSize = sizeof("2017-01-16T17:06:45Z"); + char timeStamp[timeStampSize]; + const char * const fmt = "%Y-%m-%dT%H:%M:%SZ"; + +#ifdef _MSC_VER + std::strftime(timeStamp, timeStampSize, fmt, &timeInfo); +#else + std::strftime(timeStamp, timeStampSize, fmt, timeInfo); +#endif + return std::string(timeStamp); + } + }; +} +#endif // CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +// end catch_tostring.h +#include + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable:4389) // '==' : signed/unsigned mismatch +#pragma warning(disable:4018) // more "signed/unsigned mismatch" +#pragma warning(disable:4312) // Converting int to T* using reinterpret_cast (issue on x64 platform) +#pragma warning(disable:4180) // qualifier applied to function type has no meaning +#endif + +namespace Catch { + + struct ITransientExpression { + auto isBinaryExpression() const -> bool { return m_isBinaryExpression; } + auto getResult() const -> bool { return m_result; } + virtual void streamReconstructedExpression( std::ostream &os ) const = 0; + + ITransientExpression( bool isBinaryExpression, bool result ) + : m_isBinaryExpression( isBinaryExpression ), + m_result( result ) + {} + + // We don't actually need a virtual destructor, but many static analysers + // complain if it's not here :-( + virtual ~ITransientExpression(); + + bool m_isBinaryExpression; + bool m_result; + + }; + + void formatReconstructedExpression( std::ostream &os, std::string const& lhs, StringRef op, std::string const& rhs ); + + template + class BinaryExpr : public ITransientExpression { + LhsT m_lhs; + StringRef m_op; + RhsT m_rhs; + + void streamReconstructedExpression( std::ostream &os ) const override { + formatReconstructedExpression + ( os, Catch::Detail::stringify( m_lhs ), m_op, Catch::Detail::stringify( m_rhs ) ); + } + + public: + BinaryExpr( bool comparisonResult, LhsT lhs, StringRef op, RhsT rhs ) + : ITransientExpression{ true, comparisonResult }, + m_lhs( lhs ), + m_op( op ), + m_rhs( rhs ) + {} + }; + + template + class UnaryExpr : public ITransientExpression { + LhsT m_lhs; + + void streamReconstructedExpression( std::ostream &os ) const override { + os << Catch::Detail::stringify( m_lhs ); + } + + public: + explicit UnaryExpr( LhsT lhs ) + : ITransientExpression{ false, lhs ? true : false }, + m_lhs( lhs ) + {} + }; + + // Specialised comparison functions to handle equality comparisons between ints and pointers (NULL deduces as an int) + template + auto compareEqual( LhsT const& lhs, RhsT const& rhs ) -> bool { return static_cast(lhs == rhs); } + template + auto compareEqual( T* const& lhs, int rhs ) -> bool { return lhs == reinterpret_cast( rhs ); } + template + auto compareEqual( T* const& lhs, long rhs ) -> bool { return lhs == reinterpret_cast( rhs ); } + template + auto compareEqual( int lhs, T* const& rhs ) -> bool { return reinterpret_cast( lhs ) == rhs; } + template + auto compareEqual( long lhs, T* const& rhs ) -> bool { return reinterpret_cast( lhs ) == rhs; } + + template + auto compareNotEqual( LhsT const& lhs, RhsT&& rhs ) -> bool { return static_cast(lhs != rhs); } + template + auto compareNotEqual( T* const& lhs, int rhs ) -> bool { return lhs != reinterpret_cast( rhs ); } + template + auto compareNotEqual( T* const& lhs, long rhs ) -> bool { return lhs != reinterpret_cast( rhs ); } + template + auto compareNotEqual( int lhs, T* const& rhs ) -> bool { return reinterpret_cast( lhs ) != rhs; } + template + auto compareNotEqual( long lhs, T* const& rhs ) -> bool { return reinterpret_cast( lhs ) != rhs; } + + template + class ExprLhs { + LhsT m_lhs; + public: + explicit ExprLhs( LhsT lhs ) : m_lhs( lhs ) {} + + template + auto operator == ( RhsT const& rhs ) -> BinaryExpr const { + return { compareEqual( m_lhs, rhs ), m_lhs, "==", rhs }; + } + auto operator == ( bool rhs ) -> BinaryExpr const { + return { m_lhs == rhs, m_lhs, "==", rhs }; + } + + template + auto operator != ( RhsT const& rhs ) -> BinaryExpr const { + return { compareNotEqual( m_lhs, rhs ), m_lhs, "!=", rhs }; + } + auto operator != ( bool rhs ) -> BinaryExpr const { + return { m_lhs != rhs, m_lhs, "!=", rhs }; + } + + template + auto operator > ( RhsT const& rhs ) -> BinaryExpr const { + return { static_cast(m_lhs > rhs), m_lhs, ">", rhs }; + } + template + auto operator < ( RhsT const& rhs ) -> BinaryExpr const { + return { static_cast(m_lhs < rhs), m_lhs, "<", rhs }; + } + template + auto operator >= ( RhsT const& rhs ) -> BinaryExpr const { + return { static_cast(m_lhs >= rhs), m_lhs, ">=", rhs }; + } + template + auto operator <= ( RhsT const& rhs ) -> BinaryExpr const { + return { static_cast(m_lhs <= rhs), m_lhs, "<=", rhs }; + } + + auto makeUnaryExpr() const -> UnaryExpr { + return UnaryExpr{ m_lhs }; + } + }; + + void handleExpression( ITransientExpression const& expr ); + + template + void handleExpression( ExprLhs const& expr ) { + handleExpression( expr.makeUnaryExpr() ); + } + + struct Decomposer { + template + auto operator <= ( T const& lhs ) -> ExprLhs { + return ExprLhs{ lhs }; + } + + auto operator <=( bool value ) -> ExprLhs { + return ExprLhs{ value }; + } + }; + +} // end namespace Catch + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +// end catch_decomposer.h +// start catch_interfaces_capture.h + +#include + +namespace Catch { + + class AssertionResult; + struct AssertionInfo; + struct SectionInfo; + struct SectionEndInfo; + struct MessageInfo; + struct Counts; + struct BenchmarkInfo; + struct BenchmarkStats; + struct AssertionReaction; + + struct ITransientExpression; + + struct IResultCapture { + + virtual ~IResultCapture(); + + virtual bool sectionStarted( SectionInfo const& sectionInfo, + Counts& assertions ) = 0; + virtual void sectionEnded( SectionEndInfo const& endInfo ) = 0; + virtual void sectionEndedEarly( SectionEndInfo const& endInfo ) = 0; + + virtual void benchmarkStarting( BenchmarkInfo const& info ) = 0; + virtual void benchmarkEnded( BenchmarkStats const& stats ) = 0; + + virtual void pushScopedMessage( MessageInfo const& message ) = 0; + virtual void popScopedMessage( MessageInfo const& message ) = 0; + + virtual void handleFatalErrorCondition( StringRef message ) = 0; + + virtual void handleExpr + ( AssertionInfo const& info, + ITransientExpression const& expr, + AssertionReaction& reaction ) = 0; + virtual void handleMessage + ( AssertionInfo const& info, + ResultWas::OfType resultType, + StringRef const& message, + AssertionReaction& reaction ) = 0; + virtual void handleUnexpectedExceptionNotThrown + ( AssertionInfo const& info, + AssertionReaction& reaction ) = 0; + virtual void handleUnexpectedInflightException + ( AssertionInfo const& info, + std::string const& message, + AssertionReaction& reaction ) = 0; + virtual void handleIncomplete + ( AssertionInfo const& info ) = 0; + virtual void handleNonExpr + ( AssertionInfo const &info, + ResultWas::OfType resultType, + AssertionReaction &reaction ) = 0; + + virtual bool lastAssertionPassed() = 0; + virtual void assertionPassed() = 0; + + // Deprecated, do not use: + virtual std::string getCurrentTestName() const = 0; + virtual const AssertionResult* getLastResult() const = 0; + virtual void exceptionEarlyReported() = 0; + }; + + IResultCapture& getResultCapture(); +} + +// end catch_interfaces_capture.h +namespace Catch { + + struct TestFailureException{}; + struct AssertionResultData; + struct IResultCapture; + class RunContext; + + class LazyExpression { + friend class AssertionHandler; + friend struct AssertionStats; + friend class RunContext; + + ITransientExpression const* m_transientExpression = nullptr; + bool m_isNegated; + public: + LazyExpression( bool isNegated ); + LazyExpression( LazyExpression const& other ); + LazyExpression& operator = ( LazyExpression const& ) = delete; + + explicit operator bool() const; + + friend auto operator << ( std::ostream& os, LazyExpression const& lazyExpr ) -> std::ostream&; + }; + + struct AssertionReaction { + bool shouldDebugBreak = false; + bool shouldThrow = false; + }; + + class AssertionHandler { + AssertionInfo m_assertionInfo; + AssertionReaction m_reaction; + bool m_completed = false; + IResultCapture& m_resultCapture; + + public: + AssertionHandler + ( StringRef macroName, + SourceLineInfo const& lineInfo, + StringRef capturedExpression, + ResultDisposition::Flags resultDisposition ); + ~AssertionHandler() { + if ( !m_completed ) { + m_resultCapture.handleIncomplete( m_assertionInfo ); + } + } + + template + void handleExpr( ExprLhs const& expr ) { + handleExpr( expr.makeUnaryExpr() ); + } + void handleExpr( ITransientExpression const& expr ); + + void handleMessage(ResultWas::OfType resultType, StringRef const& message); + + void handleExceptionThrownAsExpected(); + void handleUnexpectedExceptionNotThrown(); + void handleExceptionNotThrownAsExpected(); + void handleThrowingCallSkipped(); + void handleUnexpectedInflightException(); + + void complete(); + void setCompleted(); + + // query + auto allowThrows() const -> bool; + }; + + void handleExceptionMatchExpr( AssertionHandler& handler, std::string const& str, StringRef matcherString ); + +} // namespace Catch + +// end catch_assertionhandler.h +// start catch_message.h + +#include + +namespace Catch { + + struct MessageInfo { + MessageInfo( std::string const& _macroName, + SourceLineInfo const& _lineInfo, + ResultWas::OfType _type ); + + std::string macroName; + std::string message; + SourceLineInfo lineInfo; + ResultWas::OfType type; + unsigned int sequence; + + bool operator == ( MessageInfo const& other ) const; + bool operator < ( MessageInfo const& other ) const; + private: + static unsigned int globalCount; + }; + + struct MessageStream { + + template + MessageStream& operator << ( T const& value ) { + m_stream << value; + return *this; + } + + ReusableStringStream m_stream; + }; + + struct MessageBuilder : MessageStream { + MessageBuilder( std::string const& macroName, + SourceLineInfo const& lineInfo, + ResultWas::OfType type ); + + template + MessageBuilder& operator << ( T const& value ) { + m_stream << value; + return *this; + } + + MessageInfo m_info; + }; + + class ScopedMessage { + public: + explicit ScopedMessage( MessageBuilder const& builder ); + ~ScopedMessage(); + + MessageInfo m_info; + }; + +} // end namespace Catch + +// end catch_message.h +#if !defined(CATCH_CONFIG_DISABLE) + +#if !defined(CATCH_CONFIG_DISABLE_STRINGIFICATION) + #define CATCH_INTERNAL_STRINGIFY(...) #__VA_ARGS__ +#else + #define CATCH_INTERNAL_STRINGIFY(...) "Disabled by CATCH_CONFIG_DISABLE_STRINGIFICATION" +#endif + +#if defined(CATCH_CONFIG_FAST_COMPILE) + +/////////////////////////////////////////////////////////////////////////////// +// Another way to speed-up compilation is to omit local try-catch for REQUIRE* +// macros. +#define INTERNAL_CATCH_TRY +#define INTERNAL_CATCH_CATCH( capturer ) + +#else // CATCH_CONFIG_FAST_COMPILE + +#define INTERNAL_CATCH_TRY try +#define INTERNAL_CATCH_CATCH( handler ) catch(...) { handler.handleUnexpectedInflightException(); } + +#endif + +#define INTERNAL_CATCH_REACT( handler ) handler.complete(); + +/////////////////////////////////////////////////////////////////////////////// +#define INTERNAL_CATCH_TEST( macroName, resultDisposition, ... ) \ + do { \ + Catch::AssertionHandler catchAssertionHandler( macroName, CATCH_INTERNAL_LINEINFO, CATCH_INTERNAL_STRINGIFY(__VA_ARGS__), resultDisposition ); \ + INTERNAL_CATCH_TRY { \ + CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS \ + catchAssertionHandler.handleExpr( Catch::Decomposer() <= __VA_ARGS__ ); \ + CATCH_INTERNAL_UNSUPPRESS_PARENTHESES_WARNINGS \ + } INTERNAL_CATCH_CATCH( catchAssertionHandler ) \ + INTERNAL_CATCH_REACT( catchAssertionHandler ) \ + } while( (void)0, false && static_cast( !!(__VA_ARGS__) ) ) // the expression here is never evaluated at runtime but it forces the compiler to give it a look + // The double negation silences MSVC's C4800 warning, the static_cast forces short-circuit evaluation if the type has overloaded &&. + +/////////////////////////////////////////////////////////////////////////////// +#define INTERNAL_CATCH_IF( macroName, resultDisposition, ... ) \ + INTERNAL_CATCH_TEST( macroName, resultDisposition, __VA_ARGS__ ); \ + if( Catch::getResultCapture().lastAssertionPassed() ) + +/////////////////////////////////////////////////////////////////////////////// +#define INTERNAL_CATCH_ELSE( macroName, resultDisposition, ... ) \ + INTERNAL_CATCH_TEST( macroName, resultDisposition, __VA_ARGS__ ); \ + if( !Catch::getResultCapture().lastAssertionPassed() ) + +/////////////////////////////////////////////////////////////////////////////// +#define INTERNAL_CATCH_NO_THROW( macroName, resultDisposition, ... ) \ + do { \ + Catch::AssertionHandler catchAssertionHandler( macroName, CATCH_INTERNAL_LINEINFO, CATCH_INTERNAL_STRINGIFY(__VA_ARGS__), resultDisposition ); \ + try { \ + static_cast(__VA_ARGS__); \ + catchAssertionHandler.handleExceptionNotThrownAsExpected(); \ + } \ + catch( ... ) { \ + catchAssertionHandler.handleUnexpectedInflightException(); \ + } \ + INTERNAL_CATCH_REACT( catchAssertionHandler ) \ + } while( false ) + +/////////////////////////////////////////////////////////////////////////////// +#define INTERNAL_CATCH_THROWS( macroName, resultDisposition, ... ) \ + do { \ + Catch::AssertionHandler catchAssertionHandler( macroName, CATCH_INTERNAL_LINEINFO, CATCH_INTERNAL_STRINGIFY(__VA_ARGS__), resultDisposition); \ + if( catchAssertionHandler.allowThrows() ) \ + try { \ + static_cast(__VA_ARGS__); \ + catchAssertionHandler.handleUnexpectedExceptionNotThrown(); \ + } \ + catch( ... ) { \ + catchAssertionHandler.handleExceptionThrownAsExpected(); \ + } \ + else \ + catchAssertionHandler.handleThrowingCallSkipped(); \ + INTERNAL_CATCH_REACT( catchAssertionHandler ) \ + } while( false ) + +/////////////////////////////////////////////////////////////////////////////// +#define INTERNAL_CATCH_THROWS_AS( macroName, exceptionType, resultDisposition, expr ) \ + do { \ + Catch::AssertionHandler catchAssertionHandler( macroName, CATCH_INTERNAL_LINEINFO, CATCH_INTERNAL_STRINGIFY(expr) ", " CATCH_INTERNAL_STRINGIFY(exceptionType), resultDisposition ); \ + if( catchAssertionHandler.allowThrows() ) \ + try { \ + static_cast(expr); \ + catchAssertionHandler.handleUnexpectedExceptionNotThrown(); \ + } \ + catch( exceptionType const& ) { \ + catchAssertionHandler.handleExceptionThrownAsExpected(); \ + } \ + catch( ... ) { \ + catchAssertionHandler.handleUnexpectedInflightException(); \ + } \ + else \ + catchAssertionHandler.handleThrowingCallSkipped(); \ + INTERNAL_CATCH_REACT( catchAssertionHandler ) \ + } while( false ) + +/////////////////////////////////////////////////////////////////////////////// +#define INTERNAL_CATCH_MSG( macroName, messageType, resultDisposition, ... ) \ + do { \ + Catch::AssertionHandler catchAssertionHandler( macroName, CATCH_INTERNAL_LINEINFO, Catch::StringRef(), resultDisposition ); \ + catchAssertionHandler.handleMessage( messageType, ( Catch::MessageStream() << __VA_ARGS__ + ::Catch::StreamEndStop() ).m_stream.str() ); \ + INTERNAL_CATCH_REACT( catchAssertionHandler ) \ + } while( false ) + +/////////////////////////////////////////////////////////////////////////////// +#define INTERNAL_CATCH_INFO( macroName, log ) \ + Catch::ScopedMessage INTERNAL_CATCH_UNIQUE_NAME( scopedMessage )( Catch::MessageBuilder( macroName, CATCH_INTERNAL_LINEINFO, Catch::ResultWas::Info ) << log ); + +/////////////////////////////////////////////////////////////////////////////// +// Although this is matcher-based, it can be used with just a string +#define INTERNAL_CATCH_THROWS_STR_MATCHES( macroName, resultDisposition, matcher, ... ) \ + do { \ + Catch::AssertionHandler catchAssertionHandler( macroName, CATCH_INTERNAL_LINEINFO, CATCH_INTERNAL_STRINGIFY(__VA_ARGS__) ", " CATCH_INTERNAL_STRINGIFY(matcher), resultDisposition ); \ + if( catchAssertionHandler.allowThrows() ) \ + try { \ + static_cast(__VA_ARGS__); \ + catchAssertionHandler.handleUnexpectedExceptionNotThrown(); \ + } \ + catch( ... ) { \ + Catch::handleExceptionMatchExpr( catchAssertionHandler, matcher, #matcher ); \ + } \ + else \ + catchAssertionHandler.handleThrowingCallSkipped(); \ + INTERNAL_CATCH_REACT( catchAssertionHandler ) \ + } while( false ) + +#endif // CATCH_CONFIG_DISABLE + +// end catch_capture.hpp +// start catch_section.h + +// start catch_section_info.h + +// start catch_totals.h + +#include + +namespace Catch { + + struct Counts { + Counts operator - ( Counts const& other ) const; + Counts& operator += ( Counts const& other ); + + std::size_t total() const; + bool allPassed() const; + bool allOk() const; + + std::size_t passed = 0; + std::size_t failed = 0; + std::size_t failedButOk = 0; + }; + + struct Totals { + + Totals operator - ( Totals const& other ) const; + Totals& operator += ( Totals const& other ); + + Totals delta( Totals const& prevTotals ) const; + + int error = 0; + Counts assertions; + Counts testCases; + }; +} + +// end catch_totals.h +#include + +namespace Catch { + + struct SectionInfo { + SectionInfo + ( SourceLineInfo const& _lineInfo, + std::string const& _name ); + + // Deprecated + SectionInfo + ( SourceLineInfo const& _lineInfo, + std::string const& _name, + std::string const& ) : SectionInfo( _lineInfo, _name ) {} + + std::string name; + std::string description; // !Deprecated: this will always be empty + SourceLineInfo lineInfo; + }; + + struct SectionEndInfo { + SectionInfo sectionInfo; + Counts prevAssertions; + double durationInSeconds; + }; + +} // end namespace Catch + +// end catch_section_info.h +// start catch_timer.h + +#include + +namespace Catch { + + auto getCurrentNanosecondsSinceEpoch() -> uint64_t; + auto getEstimatedClockResolution() -> uint64_t; + + class Timer { + uint64_t m_nanoseconds = 0; + public: + void start(); + auto getElapsedNanoseconds() const -> uint64_t; + auto getElapsedMicroseconds() const -> uint64_t; + auto getElapsedMilliseconds() const -> unsigned int; + auto getElapsedSeconds() const -> double; + }; + +} // namespace Catch + +// end catch_timer.h +#include + +namespace Catch { + + class Section : NonCopyable { + public: + Section( SectionInfo const& info ); + ~Section(); + + // This indicates whether the section should be executed or not + explicit operator bool() const; + + private: + SectionInfo m_info; + + std::string m_name; + Counts m_assertions; + bool m_sectionIncluded; + Timer m_timer; + }; + +} // end namespace Catch + +#define INTERNAL_CATCH_SECTION( ... ) \ + CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS \ + if( Catch::Section const& INTERNAL_CATCH_UNIQUE_NAME( catch_internal_Section ) = Catch::SectionInfo( CATCH_INTERNAL_LINEINFO, __VA_ARGS__ ) ) \ + CATCH_INTERNAL_UNSUPPRESS_UNUSED_WARNINGS + +#define INTERNAL_CATCH_DYNAMIC_SECTION( ... ) \ + CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS \ + if( Catch::Section const& INTERNAL_CATCH_UNIQUE_NAME( catch_internal_Section ) = Catch::SectionInfo( CATCH_INTERNAL_LINEINFO, (Catch::ReusableStringStream() << __VA_ARGS__).str() ) ) \ + CATCH_INTERNAL_UNSUPPRESS_UNUSED_WARNINGS + +// end catch_section.h +// start catch_benchmark.h + +#include +#include + +namespace Catch { + + class BenchmarkLooper { + + std::string m_name; + std::size_t m_count = 0; + std::size_t m_iterationsToRun = 1; + uint64_t m_resolution; + Timer m_timer; + + static auto getResolution() -> uint64_t; + public: + // Keep most of this inline as it's on the code path that is being timed + BenchmarkLooper( StringRef name ) + : m_name( name ), + m_resolution( getResolution() ) + { + reportStart(); + m_timer.start(); + } + + explicit operator bool() { + if( m_count < m_iterationsToRun ) + return true; + return needsMoreIterations(); + } + + void increment() { + ++m_count; + } + + void reportStart(); + auto needsMoreIterations() -> bool; + }; + +} // end namespace Catch + +#define BENCHMARK( name ) \ + for( Catch::BenchmarkLooper looper( name ); looper; looper.increment() ) + +// end catch_benchmark.h +// start catch_interfaces_exception.h + +// start catch_interfaces_registry_hub.h + +#include +#include + +namespace Catch { + + class TestCase; + struct ITestCaseRegistry; + struct IExceptionTranslatorRegistry; + struct IExceptionTranslator; + struct IReporterRegistry; + struct IReporterFactory; + struct ITagAliasRegistry; + class StartupExceptionRegistry; + + using IReporterFactoryPtr = std::shared_ptr; + + struct IRegistryHub { + virtual ~IRegistryHub(); + + virtual IReporterRegistry const& getReporterRegistry() const = 0; + virtual ITestCaseRegistry const& getTestCaseRegistry() const = 0; + virtual ITagAliasRegistry const& getTagAliasRegistry() const = 0; + + virtual IExceptionTranslatorRegistry& getExceptionTranslatorRegistry() = 0; + + virtual StartupExceptionRegistry const& getStartupExceptionRegistry() const = 0; + }; + + struct IMutableRegistryHub { + virtual ~IMutableRegistryHub(); + virtual void registerReporter( std::string const& name, IReporterFactoryPtr const& factory ) = 0; + virtual void registerListener( IReporterFactoryPtr const& factory ) = 0; + virtual void registerTest( TestCase const& testInfo ) = 0; + virtual void registerTranslator( const IExceptionTranslator* translator ) = 0; + virtual void registerTagAlias( std::string const& alias, std::string const& tag, SourceLineInfo const& lineInfo ) = 0; + virtual void registerStartupException() noexcept = 0; + }; + + IRegistryHub& getRegistryHub(); + IMutableRegistryHub& getMutableRegistryHub(); + void cleanUp(); + std::string translateActiveException(); + +} + +// end catch_interfaces_registry_hub.h +#if defined(CATCH_CONFIG_DISABLE) + #define INTERNAL_CATCH_TRANSLATE_EXCEPTION_NO_REG( translatorName, signature) \ + static std::string translatorName( signature ) +#endif + +#include +#include +#include + +namespace Catch { + using exceptionTranslateFunction = std::string(*)(); + + struct IExceptionTranslator; + using ExceptionTranslators = std::vector>; + + struct IExceptionTranslator { + virtual ~IExceptionTranslator(); + virtual std::string translate( ExceptionTranslators::const_iterator it, ExceptionTranslators::const_iterator itEnd ) const = 0; + }; + + struct IExceptionTranslatorRegistry { + virtual ~IExceptionTranslatorRegistry(); + + virtual std::string translateActiveException() const = 0; + }; + + class ExceptionTranslatorRegistrar { + template + class ExceptionTranslator : public IExceptionTranslator { + public: + + ExceptionTranslator( std::string(*translateFunction)( T& ) ) + : m_translateFunction( translateFunction ) + {} + + std::string translate( ExceptionTranslators::const_iterator it, ExceptionTranslators::const_iterator itEnd ) const override { + try { + if( it == itEnd ) + std::rethrow_exception(std::current_exception()); + else + return (*it)->translate( it+1, itEnd ); + } + catch( T& ex ) { + return m_translateFunction( ex ); + } + } + + protected: + std::string(*m_translateFunction)( T& ); + }; + + public: + template + ExceptionTranslatorRegistrar( std::string(*translateFunction)( T& ) ) { + getMutableRegistryHub().registerTranslator + ( new ExceptionTranslator( translateFunction ) ); + } + }; +} + +/////////////////////////////////////////////////////////////////////////////// +#define INTERNAL_CATCH_TRANSLATE_EXCEPTION2( translatorName, signature ) \ + static std::string translatorName( signature ); \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + namespace{ Catch::ExceptionTranslatorRegistrar INTERNAL_CATCH_UNIQUE_NAME( catch_internal_ExceptionRegistrar )( &translatorName ); } \ + CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS \ + static std::string translatorName( signature ) + +#define INTERNAL_CATCH_TRANSLATE_EXCEPTION( signature ) INTERNAL_CATCH_TRANSLATE_EXCEPTION2( INTERNAL_CATCH_UNIQUE_NAME( catch_internal_ExceptionTranslator ), signature ) + +// end catch_interfaces_exception.h +// start catch_approx.h + +#include +#include + +namespace Catch { +namespace Detail { + + class Approx { + private: + bool equalityComparisonImpl(double other) const; + + public: + explicit Approx ( double value ); + + static Approx custom(); + + Approx operator-() const; + + template ::value>::type> + Approx operator()( T const& value ) { + Approx approx( static_cast(value) ); + approx.epsilon( m_epsilon ); + approx.margin( m_margin ); + approx.scale( m_scale ); + return approx; + } + + template ::value>::type> + explicit Approx( T const& value ): Approx(static_cast(value)) + {} + + template ::value>::type> + friend bool operator == ( const T& lhs, Approx const& rhs ) { + auto lhs_v = static_cast(lhs); + return rhs.equalityComparisonImpl(lhs_v); + } + + template ::value>::type> + friend bool operator == ( Approx const& lhs, const T& rhs ) { + return operator==( rhs, lhs ); + } + + template ::value>::type> + friend bool operator != ( T const& lhs, Approx const& rhs ) { + return !operator==( lhs, rhs ); + } + + template ::value>::type> + friend bool operator != ( Approx const& lhs, T const& rhs ) { + return !operator==( rhs, lhs ); + } + + template ::value>::type> + friend bool operator <= ( T const& lhs, Approx const& rhs ) { + return static_cast(lhs) < rhs.m_value || lhs == rhs; + } + + template ::value>::type> + friend bool operator <= ( Approx const& lhs, T const& rhs ) { + return lhs.m_value < static_cast(rhs) || lhs == rhs; + } + + template ::value>::type> + friend bool operator >= ( T const& lhs, Approx const& rhs ) { + return static_cast(lhs) > rhs.m_value || lhs == rhs; + } + + template ::value>::type> + friend bool operator >= ( Approx const& lhs, T const& rhs ) { + return lhs.m_value > static_cast(rhs) || lhs == rhs; + } + + template ::value>::type> + Approx& epsilon( T const& newEpsilon ) { + double epsilonAsDouble = static_cast(newEpsilon); + if( epsilonAsDouble < 0 || epsilonAsDouble > 1.0 ) { + throw std::domain_error + ( "Invalid Approx::epsilon: " + + Catch::Detail::stringify( epsilonAsDouble ) + + ", Approx::epsilon has to be between 0 and 1" ); + } + m_epsilon = epsilonAsDouble; + return *this; + } + + template ::value>::type> + Approx& margin( T const& newMargin ) { + double marginAsDouble = static_cast(newMargin); + if( marginAsDouble < 0 ) { + throw std::domain_error + ( "Invalid Approx::margin: " + + Catch::Detail::stringify( marginAsDouble ) + + ", Approx::Margin has to be non-negative." ); + + } + m_margin = marginAsDouble; + return *this; + } + + template ::value>::type> + Approx& scale( T const& newScale ) { + m_scale = static_cast(newScale); + return *this; + } + + std::string toString() const; + + private: + double m_epsilon; + double m_margin; + double m_scale; + double m_value; + }; +} // end namespace Detail + +namespace literals { + Detail::Approx operator "" _a(long double val); + Detail::Approx operator "" _a(unsigned long long val); +} // end namespace literals + +template<> +struct StringMaker { + static std::string convert(Catch::Detail::Approx const& value); +}; + +} // end namespace Catch + +// end catch_approx.h +// start catch_string_manip.h + +#include +#include + +namespace Catch { + + bool startsWith( std::string const& s, std::string const& prefix ); + bool startsWith( std::string const& s, char prefix ); + bool endsWith( std::string const& s, std::string const& suffix ); + bool endsWith( std::string const& s, char suffix ); + bool contains( std::string const& s, std::string const& infix ); + void toLowerInPlace( std::string& s ); + std::string toLower( std::string const& s ); + std::string trim( std::string const& str ); + bool replaceInPlace( std::string& str, std::string const& replaceThis, std::string const& withThis ); + + struct pluralise { + pluralise( std::size_t count, std::string const& label ); + + friend std::ostream& operator << ( std::ostream& os, pluralise const& pluraliser ); + + std::size_t m_count; + std::string m_label; + }; +} + +// end catch_string_manip.h +#ifndef CATCH_CONFIG_DISABLE_MATCHERS +// start catch_capture_matchers.h + +// start catch_matchers.h + +#include +#include + +namespace Catch { +namespace Matchers { + namespace Impl { + + template struct MatchAllOf; + template struct MatchAnyOf; + template struct MatchNotOf; + + class MatcherUntypedBase { + public: + MatcherUntypedBase() = default; + MatcherUntypedBase ( MatcherUntypedBase const& ) = default; + MatcherUntypedBase& operator = ( MatcherUntypedBase const& ) = delete; + std::string toString() const; + + protected: + virtual ~MatcherUntypedBase(); + virtual std::string describe() const = 0; + mutable std::string m_cachedToString; + }; + + template + struct MatcherMethod { + virtual bool match( ObjectT const& arg ) const = 0; + }; + template + struct MatcherMethod { + virtual bool match( PtrT* arg ) const = 0; + }; + + template + struct MatcherBase : MatcherUntypedBase, MatcherMethod { + + MatchAllOf operator && ( MatcherBase const& other ) const; + MatchAnyOf operator || ( MatcherBase const& other ) const; + MatchNotOf operator ! () const; + }; + + template + struct MatchAllOf : MatcherBase { + bool match( ArgT const& arg ) const override { + for( auto matcher : m_matchers ) { + if (!matcher->match(arg)) + return false; + } + return true; + } + std::string describe() const override { + std::string description; + description.reserve( 4 + m_matchers.size()*32 ); + description += "( "; + bool first = true; + for( auto matcher : m_matchers ) { + if( first ) + first = false; + else + description += " and "; + description += matcher->toString(); + } + description += " )"; + return description; + } + + MatchAllOf& operator && ( MatcherBase const& other ) { + m_matchers.push_back( &other ); + return *this; + } + + std::vector const*> m_matchers; + }; + template + struct MatchAnyOf : MatcherBase { + + bool match( ArgT const& arg ) const override { + for( auto matcher : m_matchers ) { + if (matcher->match(arg)) + return true; + } + return false; + } + std::string describe() const override { + std::string description; + description.reserve( 4 + m_matchers.size()*32 ); + description += "( "; + bool first = true; + for( auto matcher : m_matchers ) { + if( first ) + first = false; + else + description += " or "; + description += matcher->toString(); + } + description += " )"; + return description; + } + + MatchAnyOf& operator || ( MatcherBase const& other ) { + m_matchers.push_back( &other ); + return *this; + } + + std::vector const*> m_matchers; + }; + + template + struct MatchNotOf : MatcherBase { + + MatchNotOf( MatcherBase const& underlyingMatcher ) : m_underlyingMatcher( underlyingMatcher ) {} + + bool match( ArgT const& arg ) const override { + return !m_underlyingMatcher.match( arg ); + } + + std::string describe() const override { + return "not " + m_underlyingMatcher.toString(); + } + MatcherBase const& m_underlyingMatcher; + }; + + template + MatchAllOf MatcherBase::operator && ( MatcherBase const& other ) const { + return MatchAllOf() && *this && other; + } + template + MatchAnyOf MatcherBase::operator || ( MatcherBase const& other ) const { + return MatchAnyOf() || *this || other; + } + template + MatchNotOf MatcherBase::operator ! () const { + return MatchNotOf( *this ); + } + + } // namespace Impl + +} // namespace Matchers + +using namespace Matchers; +using Matchers::Impl::MatcherBase; + +} // namespace Catch + +// end catch_matchers.h +// start catch_matchers_floating.h + +#include +#include + +namespace Catch { +namespace Matchers { + + namespace Floating { + + enum class FloatingPointKind : uint8_t; + + struct WithinAbsMatcher : MatcherBase { + WithinAbsMatcher(double target, double margin); + bool match(double const& matchee) const override; + std::string describe() const override; + private: + double m_target; + double m_margin; + }; + + struct WithinUlpsMatcher : MatcherBase { + WithinUlpsMatcher(double target, int ulps, FloatingPointKind baseType); + bool match(double const& matchee) const override; + std::string describe() const override; + private: + double m_target; + int m_ulps; + FloatingPointKind m_type; + }; + + } // namespace Floating + + // The following functions create the actual matcher objects. + // This allows the types to be inferred + Floating::WithinUlpsMatcher WithinULP(double target, int maxUlpDiff); + Floating::WithinUlpsMatcher WithinULP(float target, int maxUlpDiff); + Floating::WithinAbsMatcher WithinAbs(double target, double margin); + +} // namespace Matchers +} // namespace Catch + +// end catch_matchers_floating.h +// start catch_matchers_generic.hpp + +#include +#include + +namespace Catch { +namespace Matchers { +namespace Generic { + +namespace Detail { + std::string finalizeDescription(const std::string& desc); +} + +template +class PredicateMatcher : public MatcherBase { + std::function m_predicate; + std::string m_description; +public: + + PredicateMatcher(std::function const& elem, std::string const& descr) + :m_predicate(std::move(elem)), + m_description(Detail::finalizeDescription(descr)) + {} + + bool match( T const& item ) const override { + return m_predicate(item); + } + + std::string describe() const override { + return m_description; + } +}; + +} // namespace Generic + + // The following functions create the actual matcher objects. + // The user has to explicitly specify type to the function, because + // infering std::function is hard (but possible) and + // requires a lot of TMP. + template + Generic::PredicateMatcher Predicate(std::function const& predicate, std::string const& description = "") { + return Generic::PredicateMatcher(predicate, description); + } + +} // namespace Matchers +} // namespace Catch + +// end catch_matchers_generic.hpp +// start catch_matchers_string.h + +#include + +namespace Catch { +namespace Matchers { + + namespace StdString { + + struct CasedString + { + CasedString( std::string const& str, CaseSensitive::Choice caseSensitivity ); + std::string adjustString( std::string const& str ) const; + std::string caseSensitivitySuffix() const; + + CaseSensitive::Choice m_caseSensitivity; + std::string m_str; + }; + + struct StringMatcherBase : MatcherBase { + StringMatcherBase( std::string const& operation, CasedString const& comparator ); + std::string describe() const override; + + CasedString m_comparator; + std::string m_operation; + }; + + struct EqualsMatcher : StringMatcherBase { + EqualsMatcher( CasedString const& comparator ); + bool match( std::string const& source ) const override; + }; + struct ContainsMatcher : StringMatcherBase { + ContainsMatcher( CasedString const& comparator ); + bool match( std::string const& source ) const override; + }; + struct StartsWithMatcher : StringMatcherBase { + StartsWithMatcher( CasedString const& comparator ); + bool match( std::string const& source ) const override; + }; + struct EndsWithMatcher : StringMatcherBase { + EndsWithMatcher( CasedString const& comparator ); + bool match( std::string const& source ) const override; + }; + + struct RegexMatcher : MatcherBase { + RegexMatcher( std::string regex, CaseSensitive::Choice caseSensitivity ); + bool match( std::string const& matchee ) const override; + std::string describe() const override; + + private: + std::string m_regex; + CaseSensitive::Choice m_caseSensitivity; + }; + + } // namespace StdString + + // The following functions create the actual matcher objects. + // This allows the types to be inferred + + StdString::EqualsMatcher Equals( std::string const& str, CaseSensitive::Choice caseSensitivity = CaseSensitive::Yes ); + StdString::ContainsMatcher Contains( std::string const& str, CaseSensitive::Choice caseSensitivity = CaseSensitive::Yes ); + StdString::EndsWithMatcher EndsWith( std::string const& str, CaseSensitive::Choice caseSensitivity = CaseSensitive::Yes ); + StdString::StartsWithMatcher StartsWith( std::string const& str, CaseSensitive::Choice caseSensitivity = CaseSensitive::Yes ); + StdString::RegexMatcher Matches( std::string const& regex, CaseSensitive::Choice caseSensitivity = CaseSensitive::Yes ); + +} // namespace Matchers +} // namespace Catch + +// end catch_matchers_string.h +// start catch_matchers_vector.h + +#include + +namespace Catch { +namespace Matchers { + + namespace Vector { + namespace Detail { + template + size_t count(InputIterator first, InputIterator last, T const& item) { + size_t cnt = 0; + for (; first != last; ++first) { + if (*first == item) { + ++cnt; + } + } + return cnt; + } + template + bool contains(InputIterator first, InputIterator last, T const& item) { + for (; first != last; ++first) { + if (*first == item) { + return true; + } + } + return false; + } + } + + template + struct ContainsElementMatcher : MatcherBase> { + + ContainsElementMatcher(T const &comparator) : m_comparator( comparator) {} + + bool match(std::vector const &v) const override { + for (auto const& el : v) { + if (el == m_comparator) { + return true; + } + } + return false; + } + + std::string describe() const override { + return "Contains: " + ::Catch::Detail::stringify( m_comparator ); + } + + T const& m_comparator; + }; + + template + struct ContainsMatcher : MatcherBase> { + + ContainsMatcher(std::vector const &comparator) : m_comparator( comparator ) {} + + bool match(std::vector const &v) const override { + // !TBD: see note in EqualsMatcher + if (m_comparator.size() > v.size()) + return false; + for (auto const& comparator : m_comparator) { + auto present = false; + for (const auto& el : v) { + if (el == comparator) { + present = true; + break; + } + } + if (!present) { + return false; + } + } + return true; + } + std::string describe() const override { + return "Contains: " + ::Catch::Detail::stringify( m_comparator ); + } + + std::vector const& m_comparator; + }; + + template + struct EqualsMatcher : MatcherBase> { + + EqualsMatcher(std::vector const &comparator) : m_comparator( comparator ) {} + + bool match(std::vector const &v) const override { + // !TBD: This currently works if all elements can be compared using != + // - a more general approach would be via a compare template that defaults + // to using !=. but could be specialised for, e.g. std::vector etc + // - then just call that directly + if (m_comparator.size() != v.size()) + return false; + for (std::size_t i = 0; i < v.size(); ++i) + if (m_comparator[i] != v[i]) + return false; + return true; + } + std::string describe() const override { + return "Equals: " + ::Catch::Detail::stringify( m_comparator ); + } + std::vector const& m_comparator; + }; + + template + struct UnorderedEqualsMatcher : MatcherBase> { + UnorderedEqualsMatcher(std::vector const& target) : m_target(target) {} + bool match(std::vector const& vec) const override { + // Note: This is a reimplementation of std::is_permutation, + // because I don't want to include inside the common path + if (m_target.size() != vec.size()) { + return false; + } + auto lfirst = m_target.begin(), llast = m_target.end(); + auto rfirst = vec.begin(), rlast = vec.end(); + // Cut common prefix to optimize checking of permuted parts + while (lfirst != llast && *lfirst != *rfirst) { + ++lfirst; ++rfirst; + } + if (lfirst == llast) { + return true; + } + + for (auto mid = lfirst; mid != llast; ++mid) { + // Skip already counted items + if (Detail::contains(lfirst, mid, *mid)) { + continue; + } + size_t num_vec = Detail::count(rfirst, rlast, *mid); + if (num_vec == 0 || Detail::count(lfirst, llast, *mid) != num_vec) { + return false; + } + } + + return true; + } + + std::string describe() const override { + return "UnorderedEquals: " + ::Catch::Detail::stringify(m_target); + } + private: + std::vector const& m_target; + }; + + } // namespace Vector + + // The following functions create the actual matcher objects. + // This allows the types to be inferred + + template + Vector::ContainsMatcher Contains( std::vector const& comparator ) { + return Vector::ContainsMatcher( comparator ); + } + + template + Vector::ContainsElementMatcher VectorContains( T const& comparator ) { + return Vector::ContainsElementMatcher( comparator ); + } + + template + Vector::EqualsMatcher Equals( std::vector const& comparator ) { + return Vector::EqualsMatcher( comparator ); + } + + template + Vector::UnorderedEqualsMatcher UnorderedEquals(std::vector const& target) { + return Vector::UnorderedEqualsMatcher(target); + } + +} // namespace Matchers +} // namespace Catch + +// end catch_matchers_vector.h +namespace Catch { + + template + class MatchExpr : public ITransientExpression { + ArgT const& m_arg; + MatcherT m_matcher; + StringRef m_matcherString; + public: + MatchExpr( ArgT const& arg, MatcherT const& matcher, StringRef matcherString ) + : ITransientExpression{ true, matcher.match( arg ) }, + m_arg( arg ), + m_matcher( matcher ), + m_matcherString( matcherString ) + {} + + void streamReconstructedExpression( std::ostream &os ) const override { + auto matcherAsString = m_matcher.toString(); + os << Catch::Detail::stringify( m_arg ) << ' '; + if( matcherAsString == Detail::unprintableString ) + os << m_matcherString; + else + os << matcherAsString; + } + }; + + using StringMatcher = Matchers::Impl::MatcherBase; + + void handleExceptionMatchExpr( AssertionHandler& handler, StringMatcher const& matcher, StringRef matcherString ); + + template + auto makeMatchExpr( ArgT const& arg, MatcherT const& matcher, StringRef matcherString ) -> MatchExpr { + return MatchExpr( arg, matcher, matcherString ); + } + +} // namespace Catch + +/////////////////////////////////////////////////////////////////////////////// +#define INTERNAL_CHECK_THAT( macroName, matcher, resultDisposition, arg ) \ + do { \ + Catch::AssertionHandler catchAssertionHandler( macroName, CATCH_INTERNAL_LINEINFO, CATCH_INTERNAL_STRINGIFY(arg) ", " CATCH_INTERNAL_STRINGIFY(matcher), resultDisposition ); \ + INTERNAL_CATCH_TRY { \ + catchAssertionHandler.handleExpr( Catch::makeMatchExpr( arg, matcher, #matcher ) ); \ + } INTERNAL_CATCH_CATCH( catchAssertionHandler ) \ + INTERNAL_CATCH_REACT( catchAssertionHandler ) \ + } while( false ) + +/////////////////////////////////////////////////////////////////////////////// +#define INTERNAL_CATCH_THROWS_MATCHES( macroName, exceptionType, resultDisposition, matcher, ... ) \ + do { \ + Catch::AssertionHandler catchAssertionHandler( macroName, CATCH_INTERNAL_LINEINFO, CATCH_INTERNAL_STRINGIFY(__VA_ARGS__) ", " CATCH_INTERNAL_STRINGIFY(exceptionType) ", " CATCH_INTERNAL_STRINGIFY(matcher), resultDisposition ); \ + if( catchAssertionHandler.allowThrows() ) \ + try { \ + static_cast(__VA_ARGS__ ); \ + catchAssertionHandler.handleUnexpectedExceptionNotThrown(); \ + } \ + catch( exceptionType const& ex ) { \ + catchAssertionHandler.handleExpr( Catch::makeMatchExpr( ex, matcher, #matcher ) ); \ + } \ + catch( ... ) { \ + catchAssertionHandler.handleUnexpectedInflightException(); \ + } \ + else \ + catchAssertionHandler.handleThrowingCallSkipped(); \ + INTERNAL_CATCH_REACT( catchAssertionHandler ) \ + } while( false ) + +// end catch_capture_matchers.h +#endif + +// These files are included here so the single_include script doesn't put them +// in the conditionally compiled sections +// start catch_test_case_info.h + +#include +#include +#include + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wpadded" +#endif + +namespace Catch { + + struct ITestInvoker; + + struct TestCaseInfo { + enum SpecialProperties{ + None = 0, + IsHidden = 1 << 1, + ShouldFail = 1 << 2, + MayFail = 1 << 3, + Throws = 1 << 4, + NonPortable = 1 << 5, + Benchmark = 1 << 6 + }; + + TestCaseInfo( std::string const& _name, + std::string const& _className, + std::string const& _description, + std::vector const& _tags, + SourceLineInfo const& _lineInfo ); + + friend void setTags( TestCaseInfo& testCaseInfo, std::vector tags ); + + bool isHidden() const; + bool throws() const; + bool okToFail() const; + bool expectedToFail() const; + + std::string tagsAsString() const; + + std::string name; + std::string className; + std::string description; + std::vector tags; + std::vector lcaseTags; + SourceLineInfo lineInfo; + SpecialProperties properties; + }; + + class TestCase : public TestCaseInfo { + public: + + TestCase( ITestInvoker* testCase, TestCaseInfo&& info ); + + TestCase withName( std::string const& _newName ) const; + + void invoke() const; + + TestCaseInfo const& getTestCaseInfo() const; + + bool operator == ( TestCase const& other ) const; + bool operator < ( TestCase const& other ) const; + + private: + std::shared_ptr test; + }; + + TestCase makeTestCase( ITestInvoker* testCase, + std::string const& className, + NameAndTags const& nameAndTags, + SourceLineInfo const& lineInfo ); +} + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + +// end catch_test_case_info.h +// start catch_interfaces_runner.h + +namespace Catch { + + struct IRunner { + virtual ~IRunner(); + virtual bool aborting() const = 0; + }; +} + +// end catch_interfaces_runner.h + +#ifdef __OBJC__ +// start catch_objc.hpp + +#import + +#include + +// NB. Any general catch headers included here must be included +// in catch.hpp first to make sure they are included by the single +// header for non obj-usage + +/////////////////////////////////////////////////////////////////////////////// +// This protocol is really only here for (self) documenting purposes, since +// all its methods are optional. +@protocol OcFixture + +@optional + +-(void) setUp; +-(void) tearDown; + +@end + +namespace Catch { + + class OcMethod : public ITestInvoker { + + public: + OcMethod( Class cls, SEL sel ) : m_cls( cls ), m_sel( sel ) {} + + virtual void invoke() const { + id obj = [[m_cls alloc] init]; + + performOptionalSelector( obj, @selector(setUp) ); + performOptionalSelector( obj, m_sel ); + performOptionalSelector( obj, @selector(tearDown) ); + + arcSafeRelease( obj ); + } + private: + virtual ~OcMethod() {} + + Class m_cls; + SEL m_sel; + }; + + namespace Detail{ + + inline std::string getAnnotation( Class cls, + std::string const& annotationName, + std::string const& testCaseName ) { + NSString* selStr = [[NSString alloc] initWithFormat:@"Catch_%s_%s", annotationName.c_str(), testCaseName.c_str()]; + SEL sel = NSSelectorFromString( selStr ); + arcSafeRelease( selStr ); + id value = performOptionalSelector( cls, sel ); + if( value ) + return [(NSString*)value UTF8String]; + return ""; + } + } + + inline std::size_t registerTestMethods() { + std::size_t noTestMethods = 0; + int noClasses = objc_getClassList( nullptr, 0 ); + + Class* classes = (CATCH_UNSAFE_UNRETAINED Class *)malloc( sizeof(Class) * noClasses); + objc_getClassList( classes, noClasses ); + + for( int c = 0; c < noClasses; c++ ) { + Class cls = classes[c]; + { + u_int count; + Method* methods = class_copyMethodList( cls, &count ); + for( u_int m = 0; m < count ; m++ ) { + SEL selector = method_getName(methods[m]); + std::string methodName = sel_getName(selector); + if( startsWith( methodName, "Catch_TestCase_" ) ) { + std::string testCaseName = methodName.substr( 15 ); + std::string name = Detail::getAnnotation( cls, "Name", testCaseName ); + std::string desc = Detail::getAnnotation( cls, "Description", testCaseName ); + const char* className = class_getName( cls ); + + getMutableRegistryHub().registerTest( makeTestCase( new OcMethod( cls, selector ), className, NameAndTags( name.c_str(), desc.c_str() ), SourceLineInfo("",0) ) ); + noTestMethods++; + } + } + free(methods); + } + } + return noTestMethods; + } + +#if !defined(CATCH_CONFIG_DISABLE_MATCHERS) + + namespace Matchers { + namespace Impl { + namespace NSStringMatchers { + + struct StringHolder : MatcherBase{ + StringHolder( NSString* substr ) : m_substr( [substr copy] ){} + StringHolder( StringHolder const& other ) : m_substr( [other.m_substr copy] ){} + StringHolder() { + arcSafeRelease( m_substr ); + } + + bool match( NSString* arg ) const override { + return false; + } + + NSString* CATCH_ARC_STRONG m_substr; + }; + + struct Equals : StringHolder { + Equals( NSString* substr ) : StringHolder( substr ){} + + bool match( NSString* str ) const override { + return (str != nil || m_substr == nil ) && + [str isEqualToString:m_substr]; + } + + std::string describe() const override { + return "equals string: " + Catch::Detail::stringify( m_substr ); + } + }; + + struct Contains : StringHolder { + Contains( NSString* substr ) : StringHolder( substr ){} + + bool match( NSString* str ) const { + return (str != nil || m_substr == nil ) && + [str rangeOfString:m_substr].location != NSNotFound; + } + + std::string describe() const override { + return "contains string: " + Catch::Detail::stringify( m_substr ); + } + }; + + struct StartsWith : StringHolder { + StartsWith( NSString* substr ) : StringHolder( substr ){} + + bool match( NSString* str ) const override { + return (str != nil || m_substr == nil ) && + [str rangeOfString:m_substr].location == 0; + } + + std::string describe() const override { + return "starts with: " + Catch::Detail::stringify( m_substr ); + } + }; + struct EndsWith : StringHolder { + EndsWith( NSString* substr ) : StringHolder( substr ){} + + bool match( NSString* str ) const override { + return (str != nil || m_substr == nil ) && + [str rangeOfString:m_substr].location == [str length] - [m_substr length]; + } + + std::string describe() const override { + return "ends with: " + Catch::Detail::stringify( m_substr ); + } + }; + + } // namespace NSStringMatchers + } // namespace Impl + + inline Impl::NSStringMatchers::Equals + Equals( NSString* substr ){ return Impl::NSStringMatchers::Equals( substr ); } + + inline Impl::NSStringMatchers::Contains + Contains( NSString* substr ){ return Impl::NSStringMatchers::Contains( substr ); } + + inline Impl::NSStringMatchers::StartsWith + StartsWith( NSString* substr ){ return Impl::NSStringMatchers::StartsWith( substr ); } + + inline Impl::NSStringMatchers::EndsWith + EndsWith( NSString* substr ){ return Impl::NSStringMatchers::EndsWith( substr ); } + + } // namespace Matchers + + using namespace Matchers; + +#endif // CATCH_CONFIG_DISABLE_MATCHERS + +} // namespace Catch + +/////////////////////////////////////////////////////////////////////////////// +#define OC_MAKE_UNIQUE_NAME( root, uniqueSuffix ) root##uniqueSuffix +#define OC_TEST_CASE2( name, desc, uniqueSuffix ) \ ++(NSString*) OC_MAKE_UNIQUE_NAME( Catch_Name_test_, uniqueSuffix ) \ +{ \ +return @ name; \ +} \ ++(NSString*) OC_MAKE_UNIQUE_NAME( Catch_Description_test_, uniqueSuffix ) \ +{ \ +return @ desc; \ +} \ +-(void) OC_MAKE_UNIQUE_NAME( Catch_TestCase_test_, uniqueSuffix ) + +#define OC_TEST_CASE( name, desc ) OC_TEST_CASE2( name, desc, __LINE__ ) + +// end catch_objc.hpp +#endif + +#ifdef CATCH_CONFIG_EXTERNAL_INTERFACES +// start catch_external_interfaces.h + +// start catch_reporter_bases.hpp + +// start catch_interfaces_reporter.h + +// start catch_config.hpp + +// start catch_test_spec_parser.h + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wpadded" +#endif + +// start catch_test_spec.h + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wpadded" +#endif + +// start catch_wildcard_pattern.h + +namespace Catch +{ + class WildcardPattern { + enum WildcardPosition { + NoWildcard = 0, + WildcardAtStart = 1, + WildcardAtEnd = 2, + WildcardAtBothEnds = WildcardAtStart | WildcardAtEnd + }; + + public: + + WildcardPattern( std::string const& pattern, CaseSensitive::Choice caseSensitivity ); + virtual ~WildcardPattern() = default; + virtual bool matches( std::string const& str ) const; + + private: + std::string adjustCase( std::string const& str ) const; + CaseSensitive::Choice m_caseSensitivity; + WildcardPosition m_wildcard = NoWildcard; + std::string m_pattern; + }; +} + +// end catch_wildcard_pattern.h +#include +#include +#include + +namespace Catch { + + class TestSpec { + struct Pattern { + virtual ~Pattern(); + virtual bool matches( TestCaseInfo const& testCase ) const = 0; + }; + using PatternPtr = std::shared_ptr; + + class NamePattern : public Pattern { + public: + NamePattern( std::string const& name ); + virtual ~NamePattern(); + virtual bool matches( TestCaseInfo const& testCase ) const override; + private: + WildcardPattern m_wildcardPattern; + }; + + class TagPattern : public Pattern { + public: + TagPattern( std::string const& tag ); + virtual ~TagPattern(); + virtual bool matches( TestCaseInfo const& testCase ) const override; + private: + std::string m_tag; + }; + + class ExcludedPattern : public Pattern { + public: + ExcludedPattern( PatternPtr const& underlyingPattern ); + virtual ~ExcludedPattern(); + virtual bool matches( TestCaseInfo const& testCase ) const override; + private: + PatternPtr m_underlyingPattern; + }; + + struct Filter { + std::vector m_patterns; + + bool matches( TestCaseInfo const& testCase ) const; + }; + + public: + bool hasFilters() const; + bool matches( TestCaseInfo const& testCase ) const; + + private: + std::vector m_filters; + + friend class TestSpecParser; + }; +} + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + +// end catch_test_spec.h +// start catch_interfaces_tag_alias_registry.h + +#include + +namespace Catch { + + struct TagAlias; + + struct ITagAliasRegistry { + virtual ~ITagAliasRegistry(); + // Nullptr if not present + virtual TagAlias const* find( std::string const& alias ) const = 0; + virtual std::string expandAliases( std::string const& unexpandedTestSpec ) const = 0; + + static ITagAliasRegistry const& get(); + }; + +} // end namespace Catch + +// end catch_interfaces_tag_alias_registry.h +namespace Catch { + + class TestSpecParser { + enum Mode{ None, Name, QuotedName, Tag, EscapedName }; + Mode m_mode = None; + bool m_exclusion = false; + std::size_t m_start = std::string::npos, m_pos = 0; + std::string m_arg; + std::vector m_escapeChars; + TestSpec::Filter m_currentFilter; + TestSpec m_testSpec; + ITagAliasRegistry const* m_tagAliases = nullptr; + + public: + TestSpecParser( ITagAliasRegistry const& tagAliases ); + + TestSpecParser& parse( std::string const& arg ); + TestSpec testSpec(); + + private: + void visitChar( char c ); + void startNewMode( Mode mode, std::size_t start ); + void escape(); + std::string subString() const; + + template + void addPattern() { + std::string token = subString(); + for( std::size_t i = 0; i < m_escapeChars.size(); ++i ) + token = token.substr( 0, m_escapeChars[i]-m_start-i ) + token.substr( m_escapeChars[i]-m_start-i+1 ); + m_escapeChars.clear(); + if( startsWith( token, "exclude:" ) ) { + m_exclusion = true; + token = token.substr( 8 ); + } + if( !token.empty() ) { + TestSpec::PatternPtr pattern = std::make_shared( token ); + if( m_exclusion ) + pattern = std::make_shared( pattern ); + m_currentFilter.m_patterns.push_back( pattern ); + } + m_exclusion = false; + m_mode = None; + } + + void addFilter(); + }; + TestSpec parseTestSpec( std::string const& arg ); + +} // namespace Catch + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + +// end catch_test_spec_parser.h +// start catch_interfaces_config.h + +#include +#include +#include +#include + +namespace Catch { + + enum class Verbosity { + Quiet = 0, + Normal, + High + }; + + struct WarnAbout { enum What { + Nothing = 0x00, + NoAssertions = 0x01, + NoTests = 0x02 + }; }; + + struct ShowDurations { enum OrNot { + DefaultForReporter, + Always, + Never + }; }; + struct RunTests { enum InWhatOrder { + InDeclarationOrder, + InLexicographicalOrder, + InRandomOrder + }; }; + struct UseColour { enum YesOrNo { + Auto, + Yes, + No + }; }; + struct WaitForKeypress { enum When { + Never, + BeforeStart = 1, + BeforeExit = 2, + BeforeStartAndExit = BeforeStart | BeforeExit + }; }; + + class TestSpec; + + struct IConfig : NonCopyable { + + virtual ~IConfig(); + + virtual bool allowThrows() const = 0; + virtual std::ostream& stream() const = 0; + virtual std::string name() const = 0; + virtual bool includeSuccessfulResults() const = 0; + virtual bool shouldDebugBreak() const = 0; + virtual bool warnAboutMissingAssertions() const = 0; + virtual bool warnAboutNoTests() const = 0; + virtual int abortAfter() const = 0; + virtual bool showInvisibles() const = 0; + virtual ShowDurations::OrNot showDurations() const = 0; + virtual TestSpec const& testSpec() const = 0; + virtual bool hasTestFilters() const = 0; + virtual RunTests::InWhatOrder runOrder() const = 0; + virtual unsigned int rngSeed() const = 0; + virtual int benchmarkResolutionMultiple() const = 0; + virtual UseColour::YesOrNo useColour() const = 0; + virtual std::vector const& getSectionsToRun() const = 0; + virtual Verbosity verbosity() const = 0; + }; + + using IConfigPtr = std::shared_ptr; +} + +// end catch_interfaces_config.h +// Libstdc++ doesn't like incomplete classes for unique_ptr + +#include +#include +#include + +#ifndef CATCH_CONFIG_CONSOLE_WIDTH +#define CATCH_CONFIG_CONSOLE_WIDTH 80 +#endif + +namespace Catch { + + struct IStream; + + struct ConfigData { + bool listTests = false; + bool listTags = false; + bool listReporters = false; + bool listTestNamesOnly = false; + + bool showSuccessfulTests = false; + bool shouldDebugBreak = false; + bool noThrow = false; + bool showHelp = false; + bool showInvisibles = false; + bool filenamesAsTags = false; + bool libIdentify = false; + + int abortAfter = -1; + unsigned int rngSeed = 0; + int benchmarkResolutionMultiple = 100; + + Verbosity verbosity = Verbosity::Normal; + WarnAbout::What warnings = WarnAbout::Nothing; + ShowDurations::OrNot showDurations = ShowDurations::DefaultForReporter; + RunTests::InWhatOrder runOrder = RunTests::InDeclarationOrder; + UseColour::YesOrNo useColour = UseColour::Auto; + WaitForKeypress::When waitForKeypress = WaitForKeypress::Never; + + std::string outputFilename; + std::string name; + std::string processName; +#ifndef CATCH_CONFIG_DEFAULT_REPORTER +#define CATCH_CONFIG_DEFAULT_REPORTER "console" +#endif + std::string reporterName = CATCH_CONFIG_DEFAULT_REPORTER; +#undef CATCH_CONFIG_DEFAULT_REPORTER + + std::vector testsOrTags; + std::vector sectionsToRun; + }; + + class Config : public IConfig { + public: + + Config() = default; + Config( ConfigData const& data ); + virtual ~Config() = default; + + std::string const& getFilename() const; + + bool listTests() const; + bool listTestNamesOnly() const; + bool listTags() const; + bool listReporters() const; + + std::string getProcessName() const; + std::string const& getReporterName() const; + + std::vector const& getTestsOrTags() const; + std::vector const& getSectionsToRun() const override; + + virtual TestSpec const& testSpec() const override; + bool hasTestFilters() const override; + + bool showHelp() const; + + // IConfig interface + bool allowThrows() const override; + std::ostream& stream() const override; + std::string name() const override; + bool includeSuccessfulResults() const override; + bool warnAboutMissingAssertions() const override; + bool warnAboutNoTests() const override; + ShowDurations::OrNot showDurations() const override; + RunTests::InWhatOrder runOrder() const override; + unsigned int rngSeed() const override; + int benchmarkResolutionMultiple() const override; + UseColour::YesOrNo useColour() const override; + bool shouldDebugBreak() const override; + int abortAfter() const override; + bool showInvisibles() const override; + Verbosity verbosity() const override; + + private: + + IStream const* openStream(); + ConfigData m_data; + + std::unique_ptr m_stream; + TestSpec m_testSpec; + bool m_hasTestFilters = false; + }; + +} // end namespace Catch + +// end catch_config.hpp +// start catch_assertionresult.h + +#include + +namespace Catch { + + struct AssertionResultData + { + AssertionResultData() = delete; + + AssertionResultData( ResultWas::OfType _resultType, LazyExpression const& _lazyExpression ); + + std::string message; + mutable std::string reconstructedExpression; + LazyExpression lazyExpression; + ResultWas::OfType resultType; + + std::string reconstructExpression() const; + }; + + class AssertionResult { + public: + AssertionResult() = delete; + AssertionResult( AssertionInfo const& info, AssertionResultData const& data ); + + bool isOk() const; + bool succeeded() const; + ResultWas::OfType getResultType() const; + bool hasExpression() const; + bool hasMessage() const; + std::string getExpression() const; + std::string getExpressionInMacro() const; + bool hasExpandedExpression() const; + std::string getExpandedExpression() const; + std::string getMessage() const; + SourceLineInfo getSourceInfo() const; + StringRef getTestMacroName() const; + + //protected: + AssertionInfo m_info; + AssertionResultData m_resultData; + }; + +} // end namespace Catch + +// end catch_assertionresult.h +// start catch_option.hpp + +namespace Catch { + + // An optional type + template + class Option { + public: + Option() : nullableValue( nullptr ) {} + Option( T const& _value ) + : nullableValue( new( storage ) T( _value ) ) + {} + Option( Option const& _other ) + : nullableValue( _other ? new( storage ) T( *_other ) : nullptr ) + {} + + ~Option() { + reset(); + } + + Option& operator= ( Option const& _other ) { + if( &_other != this ) { + reset(); + if( _other ) + nullableValue = new( storage ) T( *_other ); + } + return *this; + } + Option& operator = ( T const& _value ) { + reset(); + nullableValue = new( storage ) T( _value ); + return *this; + } + + void reset() { + if( nullableValue ) + nullableValue->~T(); + nullableValue = nullptr; + } + + T& operator*() { return *nullableValue; } + T const& operator*() const { return *nullableValue; } + T* operator->() { return nullableValue; } + const T* operator->() const { return nullableValue; } + + T valueOr( T const& defaultValue ) const { + return nullableValue ? *nullableValue : defaultValue; + } + + bool some() const { return nullableValue != nullptr; } + bool none() const { return nullableValue == nullptr; } + + bool operator !() const { return nullableValue == nullptr; } + explicit operator bool() const { + return some(); + } + + private: + T *nullableValue; + alignas(alignof(T)) char storage[sizeof(T)]; + }; + +} // end namespace Catch + +// end catch_option.hpp +#include +#include +#include +#include +#include + +namespace Catch { + + struct ReporterConfig { + explicit ReporterConfig( IConfigPtr const& _fullConfig ); + + ReporterConfig( IConfigPtr const& _fullConfig, std::ostream& _stream ); + + std::ostream& stream() const; + IConfigPtr fullConfig() const; + + private: + std::ostream* m_stream; + IConfigPtr m_fullConfig; + }; + + struct ReporterPreferences { + bool shouldRedirectStdOut = false; + bool shouldReportAllAssertions = false; + }; + + template + struct LazyStat : Option { + LazyStat& operator=( T const& _value ) { + Option::operator=( _value ); + used = false; + return *this; + } + void reset() { + Option::reset(); + used = false; + } + bool used = false; + }; + + struct TestRunInfo { + TestRunInfo( std::string const& _name ); + std::string name; + }; + struct GroupInfo { + GroupInfo( std::string const& _name, + std::size_t _groupIndex, + std::size_t _groupsCount ); + + std::string name; + std::size_t groupIndex; + std::size_t groupsCounts; + }; + + struct AssertionStats { + AssertionStats( AssertionResult const& _assertionResult, + std::vector const& _infoMessages, + Totals const& _totals ); + + AssertionStats( AssertionStats const& ) = default; + AssertionStats( AssertionStats && ) = default; + AssertionStats& operator = ( AssertionStats const& ) = default; + AssertionStats& operator = ( AssertionStats && ) = default; + virtual ~AssertionStats(); + + AssertionResult assertionResult; + std::vector infoMessages; + Totals totals; + }; + + struct SectionStats { + SectionStats( SectionInfo const& _sectionInfo, + Counts const& _assertions, + double _durationInSeconds, + bool _missingAssertions ); + SectionStats( SectionStats const& ) = default; + SectionStats( SectionStats && ) = default; + SectionStats& operator = ( SectionStats const& ) = default; + SectionStats& operator = ( SectionStats && ) = default; + virtual ~SectionStats(); + + SectionInfo sectionInfo; + Counts assertions; + double durationInSeconds; + bool missingAssertions; + }; + + struct TestCaseStats { + TestCaseStats( TestCaseInfo const& _testInfo, + Totals const& _totals, + std::string const& _stdOut, + std::string const& _stdErr, + bool _aborting ); + + TestCaseStats( TestCaseStats const& ) = default; + TestCaseStats( TestCaseStats && ) = default; + TestCaseStats& operator = ( TestCaseStats const& ) = default; + TestCaseStats& operator = ( TestCaseStats && ) = default; + virtual ~TestCaseStats(); + + TestCaseInfo testInfo; + Totals totals; + std::string stdOut; + std::string stdErr; + bool aborting; + }; + + struct TestGroupStats { + TestGroupStats( GroupInfo const& _groupInfo, + Totals const& _totals, + bool _aborting ); + TestGroupStats( GroupInfo const& _groupInfo ); + + TestGroupStats( TestGroupStats const& ) = default; + TestGroupStats( TestGroupStats && ) = default; + TestGroupStats& operator = ( TestGroupStats const& ) = default; + TestGroupStats& operator = ( TestGroupStats && ) = default; + virtual ~TestGroupStats(); + + GroupInfo groupInfo; + Totals totals; + bool aborting; + }; + + struct TestRunStats { + TestRunStats( TestRunInfo const& _runInfo, + Totals const& _totals, + bool _aborting ); + + TestRunStats( TestRunStats const& ) = default; + TestRunStats( TestRunStats && ) = default; + TestRunStats& operator = ( TestRunStats const& ) = default; + TestRunStats& operator = ( TestRunStats && ) = default; + virtual ~TestRunStats(); + + TestRunInfo runInfo; + Totals totals; + bool aborting; + }; + + struct BenchmarkInfo { + std::string name; + }; + struct BenchmarkStats { + BenchmarkInfo info; + std::size_t iterations; + uint64_t elapsedTimeInNanoseconds; + }; + + struct IStreamingReporter { + virtual ~IStreamingReporter() = default; + + // Implementing class must also provide the following static methods: + // static std::string getDescription(); + // static std::set getSupportedVerbosities() + + virtual ReporterPreferences getPreferences() const = 0; + + virtual void noMatchingTestCases( std::string const& spec ) = 0; + + virtual void testRunStarting( TestRunInfo const& testRunInfo ) = 0; + virtual void testGroupStarting( GroupInfo const& groupInfo ) = 0; + + virtual void testCaseStarting( TestCaseInfo const& testInfo ) = 0; + virtual void sectionStarting( SectionInfo const& sectionInfo ) = 0; + + // *** experimental *** + virtual void benchmarkStarting( BenchmarkInfo const& ) {} + + virtual void assertionStarting( AssertionInfo const& assertionInfo ) = 0; + + // The return value indicates if the messages buffer should be cleared: + virtual bool assertionEnded( AssertionStats const& assertionStats ) = 0; + + // *** experimental *** + virtual void benchmarkEnded( BenchmarkStats const& ) {} + + virtual void sectionEnded( SectionStats const& sectionStats ) = 0; + virtual void testCaseEnded( TestCaseStats const& testCaseStats ) = 0; + virtual void testGroupEnded( TestGroupStats const& testGroupStats ) = 0; + virtual void testRunEnded( TestRunStats const& testRunStats ) = 0; + + virtual void skipTest( TestCaseInfo const& testInfo ) = 0; + + // Default empty implementation provided + virtual void fatalErrorEncountered( StringRef name ); + + virtual bool isMulti() const; + }; + using IStreamingReporterPtr = std::unique_ptr; + + struct IReporterFactory { + virtual ~IReporterFactory(); + virtual IStreamingReporterPtr create( ReporterConfig const& config ) const = 0; + virtual std::string getDescription() const = 0; + }; + using IReporterFactoryPtr = std::shared_ptr; + + struct IReporterRegistry { + using FactoryMap = std::map; + using Listeners = std::vector; + + virtual ~IReporterRegistry(); + virtual IStreamingReporterPtr create( std::string const& name, IConfigPtr const& config ) const = 0; + virtual FactoryMap const& getFactories() const = 0; + virtual Listeners const& getListeners() const = 0; + }; + +} // end namespace Catch + +// end catch_interfaces_reporter.h +#include +#include +#include +#include +#include +#include +#include + +namespace Catch { + void prepareExpandedExpression(AssertionResult& result); + + // Returns double formatted as %.3f (format expected on output) + std::string getFormattedDuration( double duration ); + + template + struct StreamingReporterBase : IStreamingReporter { + + StreamingReporterBase( ReporterConfig const& _config ) + : m_config( _config.fullConfig() ), + stream( _config.stream() ) + { + m_reporterPrefs.shouldRedirectStdOut = false; + if( !DerivedT::getSupportedVerbosities().count( m_config->verbosity() ) ) + throw std::domain_error( "Verbosity level not supported by this reporter" ); + } + + ReporterPreferences getPreferences() const override { + return m_reporterPrefs; + } + + static std::set getSupportedVerbosities() { + return { Verbosity::Normal }; + } + + ~StreamingReporterBase() override = default; + + void noMatchingTestCases(std::string const&) override {} + + void testRunStarting(TestRunInfo const& _testRunInfo) override { + currentTestRunInfo = _testRunInfo; + } + void testGroupStarting(GroupInfo const& _groupInfo) override { + currentGroupInfo = _groupInfo; + } + + void testCaseStarting(TestCaseInfo const& _testInfo) override { + currentTestCaseInfo = _testInfo; + } + void sectionStarting(SectionInfo const& _sectionInfo) override { + m_sectionStack.push_back(_sectionInfo); + } + + void sectionEnded(SectionStats const& /* _sectionStats */) override { + m_sectionStack.pop_back(); + } + void testCaseEnded(TestCaseStats const& /* _testCaseStats */) override { + currentTestCaseInfo.reset(); + } + void testGroupEnded(TestGroupStats const& /* _testGroupStats */) override { + currentGroupInfo.reset(); + } + void testRunEnded(TestRunStats const& /* _testRunStats */) override { + currentTestCaseInfo.reset(); + currentGroupInfo.reset(); + currentTestRunInfo.reset(); + } + + void skipTest(TestCaseInfo const&) override { + // Don't do anything with this by default. + // It can optionally be overridden in the derived class. + } + + IConfigPtr m_config; + std::ostream& stream; + + LazyStat currentTestRunInfo; + LazyStat currentGroupInfo; + LazyStat currentTestCaseInfo; + + std::vector m_sectionStack; + ReporterPreferences m_reporterPrefs; + }; + + template + struct CumulativeReporterBase : IStreamingReporter { + template + struct Node { + explicit Node( T const& _value ) : value( _value ) {} + virtual ~Node() {} + + using ChildNodes = std::vector>; + T value; + ChildNodes children; + }; + struct SectionNode { + explicit SectionNode(SectionStats const& _stats) : stats(_stats) {} + virtual ~SectionNode() = default; + + bool operator == (SectionNode const& other) const { + return stats.sectionInfo.lineInfo == other.stats.sectionInfo.lineInfo; + } + bool operator == (std::shared_ptr const& other) const { + return operator==(*other); + } + + SectionStats stats; + using ChildSections = std::vector>; + using Assertions = std::vector; + ChildSections childSections; + Assertions assertions; + std::string stdOut; + std::string stdErr; + }; + + struct BySectionInfo { + BySectionInfo( SectionInfo const& other ) : m_other( other ) {} + BySectionInfo( BySectionInfo const& other ) : m_other( other.m_other ) {} + bool operator() (std::shared_ptr const& node) const { + return ((node->stats.sectionInfo.name == m_other.name) && + (node->stats.sectionInfo.lineInfo == m_other.lineInfo)); + } + void operator=(BySectionInfo const&) = delete; + + private: + SectionInfo const& m_other; + }; + + using TestCaseNode = Node; + using TestGroupNode = Node; + using TestRunNode = Node; + + CumulativeReporterBase( ReporterConfig const& _config ) + : m_config( _config.fullConfig() ), + stream( _config.stream() ) + { + m_reporterPrefs.shouldRedirectStdOut = false; + if( !DerivedT::getSupportedVerbosities().count( m_config->verbosity() ) ) + throw std::domain_error( "Verbosity level not supported by this reporter" ); + } + ~CumulativeReporterBase() override = default; + + ReporterPreferences getPreferences() const override { + return m_reporterPrefs; + } + + static std::set getSupportedVerbosities() { + return { Verbosity::Normal }; + } + + void testRunStarting( TestRunInfo const& ) override {} + void testGroupStarting( GroupInfo const& ) override {} + + void testCaseStarting( TestCaseInfo const& ) override {} + + void sectionStarting( SectionInfo const& sectionInfo ) override { + SectionStats incompleteStats( sectionInfo, Counts(), 0, false ); + std::shared_ptr node; + if( m_sectionStack.empty() ) { + if( !m_rootSection ) + m_rootSection = std::make_shared( incompleteStats ); + node = m_rootSection; + } + else { + SectionNode& parentNode = *m_sectionStack.back(); + auto it = + std::find_if( parentNode.childSections.begin(), + parentNode.childSections.end(), + BySectionInfo( sectionInfo ) ); + if( it == parentNode.childSections.end() ) { + node = std::make_shared( incompleteStats ); + parentNode.childSections.push_back( node ); + } + else + node = *it; + } + m_sectionStack.push_back( node ); + m_deepestSection = std::move(node); + } + + void assertionStarting(AssertionInfo const&) override {} + + bool assertionEnded(AssertionStats const& assertionStats) override { + assert(!m_sectionStack.empty()); + // AssertionResult holds a pointer to a temporary DecomposedExpression, + // which getExpandedExpression() calls to build the expression string. + // Our section stack copy of the assertionResult will likely outlive the + // temporary, so it must be expanded or discarded now to avoid calling + // a destroyed object later. + prepareExpandedExpression(const_cast( assertionStats.assertionResult ) ); + SectionNode& sectionNode = *m_sectionStack.back(); + sectionNode.assertions.push_back(assertionStats); + return true; + } + void sectionEnded(SectionStats const& sectionStats) override { + assert(!m_sectionStack.empty()); + SectionNode& node = *m_sectionStack.back(); + node.stats = sectionStats; + m_sectionStack.pop_back(); + } + void testCaseEnded(TestCaseStats const& testCaseStats) override { + auto node = std::make_shared(testCaseStats); + assert(m_sectionStack.size() == 0); + node->children.push_back(m_rootSection); + m_testCases.push_back(node); + m_rootSection.reset(); + + assert(m_deepestSection); + m_deepestSection->stdOut = testCaseStats.stdOut; + m_deepestSection->stdErr = testCaseStats.stdErr; + } + void testGroupEnded(TestGroupStats const& testGroupStats) override { + auto node = std::make_shared(testGroupStats); + node->children.swap(m_testCases); + m_testGroups.push_back(node); + } + void testRunEnded(TestRunStats const& testRunStats) override { + auto node = std::make_shared(testRunStats); + node->children.swap(m_testGroups); + m_testRuns.push_back(node); + testRunEndedCumulative(); + } + virtual void testRunEndedCumulative() = 0; + + void skipTest(TestCaseInfo const&) override {} + + IConfigPtr m_config; + std::ostream& stream; + std::vector m_assertions; + std::vector>> m_sections; + std::vector> m_testCases; + std::vector> m_testGroups; + + std::vector> m_testRuns; + + std::shared_ptr m_rootSection; + std::shared_ptr m_deepestSection; + std::vector> m_sectionStack; + ReporterPreferences m_reporterPrefs; + }; + + template + char const* getLineOfChars() { + static char line[CATCH_CONFIG_CONSOLE_WIDTH] = {0}; + if( !*line ) { + std::memset( line, C, CATCH_CONFIG_CONSOLE_WIDTH-1 ); + line[CATCH_CONFIG_CONSOLE_WIDTH-1] = 0; + } + return line; + } + + struct TestEventListenerBase : StreamingReporterBase { + TestEventListenerBase( ReporterConfig const& _config ); + + void assertionStarting(AssertionInfo const&) override; + bool assertionEnded(AssertionStats const&) override; + }; + +} // end namespace Catch + +// end catch_reporter_bases.hpp +// start catch_console_colour.h + +namespace Catch { + + struct Colour { + enum Code { + None = 0, + + White, + Red, + Green, + Blue, + Cyan, + Yellow, + Grey, + + Bright = 0x10, + + BrightRed = Bright | Red, + BrightGreen = Bright | Green, + LightGrey = Bright | Grey, + BrightWhite = Bright | White, + BrightYellow = Bright | Yellow, + + // By intention + FileName = LightGrey, + Warning = BrightYellow, + ResultError = BrightRed, + ResultSuccess = BrightGreen, + ResultExpectedFailure = Warning, + + Error = BrightRed, + Success = Green, + + OriginalExpression = Cyan, + ReconstructedExpression = BrightYellow, + + SecondaryText = LightGrey, + Headers = White + }; + + // Use constructed object for RAII guard + Colour( Code _colourCode ); + Colour( Colour&& other ) noexcept; + Colour& operator=( Colour&& other ) noexcept; + ~Colour(); + + // Use static method for one-shot changes + static void use( Code _colourCode ); + + private: + bool m_moved = false; + }; + + std::ostream& operator << ( std::ostream& os, Colour const& ); + +} // end namespace Catch + +// end catch_console_colour.h +// start catch_reporter_registrars.hpp + + +namespace Catch { + + template + class ReporterRegistrar { + + class ReporterFactory : public IReporterFactory { + + virtual IStreamingReporterPtr create( ReporterConfig const& config ) const override { + return std::unique_ptr( new T( config ) ); + } + + virtual std::string getDescription() const override { + return T::getDescription(); + } + }; + + public: + + explicit ReporterRegistrar( std::string const& name ) { + getMutableRegistryHub().registerReporter( name, std::make_shared() ); + } + }; + + template + class ListenerRegistrar { + + class ListenerFactory : public IReporterFactory { + + virtual IStreamingReporterPtr create( ReporterConfig const& config ) const override { + return std::unique_ptr( new T( config ) ); + } + virtual std::string getDescription() const override { + return std::string(); + } + }; + + public: + + ListenerRegistrar() { + getMutableRegistryHub().registerListener( std::make_shared() ); + } + }; +} + +#if !defined(CATCH_CONFIG_DISABLE) + +#define CATCH_REGISTER_REPORTER( name, reporterType ) \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + namespace{ Catch::ReporterRegistrar catch_internal_RegistrarFor##reporterType( name ); } \ + CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS + +#define CATCH_REGISTER_LISTENER( listenerType ) \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + namespace{ Catch::ListenerRegistrar catch_internal_RegistrarFor##listenerType; } \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS +#else // CATCH_CONFIG_DISABLE + +#define CATCH_REGISTER_REPORTER(name, reporterType) +#define CATCH_REGISTER_LISTENER(listenerType) + +#endif // CATCH_CONFIG_DISABLE + +// end catch_reporter_registrars.hpp +// Allow users to base their work off existing reporters +// start catch_reporter_compact.h + +namespace Catch { + + struct CompactReporter : StreamingReporterBase { + + using StreamingReporterBase::StreamingReporterBase; + + ~CompactReporter() override; + + static std::string getDescription(); + + ReporterPreferences getPreferences() const override; + + void noMatchingTestCases(std::string const& spec) override; + + void assertionStarting(AssertionInfo const&) override; + + bool assertionEnded(AssertionStats const& _assertionStats) override; + + void sectionEnded(SectionStats const& _sectionStats) override; + + void testRunEnded(TestRunStats const& _testRunStats) override; + + }; + +} // end namespace Catch + +// end catch_reporter_compact.h +// start catch_reporter_console.h + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable:4061) // Not all labels are EXPLICITLY handled in switch + // Note that 4062 (not all labels are handled + // and default is missing) is enabled +#endif + +namespace Catch { + // Fwd decls + struct SummaryColumn; + class TablePrinter; + + struct ConsoleReporter : StreamingReporterBase { + std::unique_ptr m_tablePrinter; + + ConsoleReporter(ReporterConfig const& config); + ~ConsoleReporter() override; + static std::string getDescription(); + + void noMatchingTestCases(std::string const& spec) override; + + void assertionStarting(AssertionInfo const&) override; + + bool assertionEnded(AssertionStats const& _assertionStats) override; + + void sectionStarting(SectionInfo const& _sectionInfo) override; + void sectionEnded(SectionStats const& _sectionStats) override; + + void benchmarkStarting(BenchmarkInfo const& info) override; + void benchmarkEnded(BenchmarkStats const& stats) override; + + void testCaseEnded(TestCaseStats const& _testCaseStats) override; + void testGroupEnded(TestGroupStats const& _testGroupStats) override; + void testRunEnded(TestRunStats const& _testRunStats) override; + + private: + + void lazyPrint(); + + void lazyPrintWithoutClosingBenchmarkTable(); + void lazyPrintRunInfo(); + void lazyPrintGroupInfo(); + void printTestCaseAndSectionHeader(); + + void printClosedHeader(std::string const& _name); + void printOpenHeader(std::string const& _name); + + // if string has a : in first line will set indent to follow it on + // subsequent lines + void printHeaderString(std::string const& _string, std::size_t indent = 0); + + void printTotals(Totals const& totals); + void printSummaryRow(std::string const& label, std::vector const& cols, std::size_t row); + + void printTotalsDivider(Totals const& totals); + void printSummaryDivider(); + + private: + bool m_headerPrinted = false; + }; + +} // end namespace Catch + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +// end catch_reporter_console.h +// start catch_reporter_junit.h + +// start catch_xmlwriter.h + +#include + +namespace Catch { + + class XmlEncode { + public: + enum ForWhat { ForTextNodes, ForAttributes }; + + XmlEncode( std::string const& str, ForWhat forWhat = ForTextNodes ); + + void encodeTo( std::ostream& os ) const; + + friend std::ostream& operator << ( std::ostream& os, XmlEncode const& xmlEncode ); + + private: + std::string m_str; + ForWhat m_forWhat; + }; + + class XmlWriter { + public: + + class ScopedElement { + public: + ScopedElement( XmlWriter* writer ); + + ScopedElement( ScopedElement&& other ) noexcept; + ScopedElement& operator=( ScopedElement&& other ) noexcept; + + ~ScopedElement(); + + ScopedElement& writeText( std::string const& text, bool indent = true ); + + template + ScopedElement& writeAttribute( std::string const& name, T const& attribute ) { + m_writer->writeAttribute( name, attribute ); + return *this; + } + + private: + mutable XmlWriter* m_writer = nullptr; + }; + + XmlWriter( std::ostream& os = Catch::cout() ); + ~XmlWriter(); + + XmlWriter( XmlWriter const& ) = delete; + XmlWriter& operator=( XmlWriter const& ) = delete; + + XmlWriter& startElement( std::string const& name ); + + ScopedElement scopedElement( std::string const& name ); + + XmlWriter& endElement(); + + XmlWriter& writeAttribute( std::string const& name, std::string const& attribute ); + + XmlWriter& writeAttribute( std::string const& name, bool attribute ); + + template + XmlWriter& writeAttribute( std::string const& name, T const& attribute ) { + ReusableStringStream rss; + rss << attribute; + return writeAttribute( name, rss.str() ); + } + + XmlWriter& writeText( std::string const& text, bool indent = true ); + + XmlWriter& writeComment( std::string const& text ); + + void writeStylesheetRef( std::string const& url ); + + XmlWriter& writeBlankLine(); + + void ensureTagClosed(); + + private: + + void writeDeclaration(); + + void newlineIfNecessary(); + + bool m_tagIsOpen = false; + bool m_needsNewline = false; + std::vector m_tags; + std::string m_indent; + std::ostream& m_os; + }; + +} + +// end catch_xmlwriter.h +namespace Catch { + + class JunitReporter : public CumulativeReporterBase { + public: + JunitReporter(ReporterConfig const& _config); + + ~JunitReporter() override; + + static std::string getDescription(); + + void noMatchingTestCases(std::string const& /*spec*/) override; + + void testRunStarting(TestRunInfo const& runInfo) override; + + void testGroupStarting(GroupInfo const& groupInfo) override; + + void testCaseStarting(TestCaseInfo const& testCaseInfo) override; + bool assertionEnded(AssertionStats const& assertionStats) override; + + void testCaseEnded(TestCaseStats const& testCaseStats) override; + + void testGroupEnded(TestGroupStats const& testGroupStats) override; + + void testRunEndedCumulative() override; + + void writeGroup(TestGroupNode const& groupNode, double suiteTime); + + void writeTestCase(TestCaseNode const& testCaseNode); + + void writeSection(std::string const& className, + std::string const& rootName, + SectionNode const& sectionNode); + + void writeAssertions(SectionNode const& sectionNode); + void writeAssertion(AssertionStats const& stats); + + XmlWriter xml; + Timer suiteTimer; + std::string stdOutForSuite; + std::string stdErrForSuite; + unsigned int unexpectedExceptions = 0; + bool m_okToFail = false; + }; + +} // end namespace Catch + +// end catch_reporter_junit.h +// start catch_reporter_xml.h + +namespace Catch { + class XmlReporter : public StreamingReporterBase { + public: + XmlReporter(ReporterConfig const& _config); + + ~XmlReporter() override; + + static std::string getDescription(); + + virtual std::string getStylesheetRef() const; + + void writeSourceInfo(SourceLineInfo const& sourceInfo); + + public: // StreamingReporterBase + + void noMatchingTestCases(std::string const& s) override; + + void testRunStarting(TestRunInfo const& testInfo) override; + + void testGroupStarting(GroupInfo const& groupInfo) override; + + void testCaseStarting(TestCaseInfo const& testInfo) override; + + void sectionStarting(SectionInfo const& sectionInfo) override; + + void assertionStarting(AssertionInfo const&) override; + + bool assertionEnded(AssertionStats const& assertionStats) override; + + void sectionEnded(SectionStats const& sectionStats) override; + + void testCaseEnded(TestCaseStats const& testCaseStats) override; + + void testGroupEnded(TestGroupStats const& testGroupStats) override; + + void testRunEnded(TestRunStats const& testRunStats) override; + + private: + Timer m_testCaseTimer; + XmlWriter m_xml; + int m_sectionDepth = 0; + }; + +} // end namespace Catch + +// end catch_reporter_xml.h + +// end catch_external_interfaces.h +#endif + +#endif // ! CATCH_CONFIG_IMPL_ONLY + +#ifdef CATCH_IMPL +// start catch_impl.hpp + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wweak-vtables" +#endif + +// Keep these here for external reporters +// start catch_test_case_tracker.h + +#include +#include +#include + +namespace Catch { +namespace TestCaseTracking { + + struct NameAndLocation { + std::string name; + SourceLineInfo location; + + NameAndLocation( std::string const& _name, SourceLineInfo const& _location ); + }; + + struct ITracker; + + using ITrackerPtr = std::shared_ptr; + + struct ITracker { + virtual ~ITracker(); + + // static queries + virtual NameAndLocation const& nameAndLocation() const = 0; + + // dynamic queries + virtual bool isComplete() const = 0; // Successfully completed or failed + virtual bool isSuccessfullyCompleted() const = 0; + virtual bool isOpen() const = 0; // Started but not complete + virtual bool hasChildren() const = 0; + + virtual ITracker& parent() = 0; + + // actions + virtual void close() = 0; // Successfully complete + virtual void fail() = 0; + virtual void markAsNeedingAnotherRun() = 0; + + virtual void addChild( ITrackerPtr const& child ) = 0; + virtual ITrackerPtr findChild( NameAndLocation const& nameAndLocation ) = 0; + virtual void openChild() = 0; + + // Debug/ checking + virtual bool isSectionTracker() const = 0; + virtual bool isIndexTracker() const = 0; + }; + + class TrackerContext { + + enum RunState { + NotStarted, + Executing, + CompletedCycle + }; + + ITrackerPtr m_rootTracker; + ITracker* m_currentTracker = nullptr; + RunState m_runState = NotStarted; + + public: + + static TrackerContext& instance(); + + ITracker& startRun(); + void endRun(); + + void startCycle(); + void completeCycle(); + + bool completedCycle() const; + ITracker& currentTracker(); + void setCurrentTracker( ITracker* tracker ); + }; + + class TrackerBase : public ITracker { + protected: + enum CycleState { + NotStarted, + Executing, + ExecutingChildren, + NeedsAnotherRun, + CompletedSuccessfully, + Failed + }; + + class TrackerHasName { + NameAndLocation m_nameAndLocation; + public: + TrackerHasName( NameAndLocation const& nameAndLocation ); + bool operator ()( ITrackerPtr const& tracker ) const; + }; + + using Children = std::vector; + NameAndLocation m_nameAndLocation; + TrackerContext& m_ctx; + ITracker* m_parent; + Children m_children; + CycleState m_runState = NotStarted; + + public: + TrackerBase( NameAndLocation const& nameAndLocation, TrackerContext& ctx, ITracker* parent ); + + NameAndLocation const& nameAndLocation() const override; + bool isComplete() const override; + bool isSuccessfullyCompleted() const override; + bool isOpen() const override; + bool hasChildren() const override; + + void addChild( ITrackerPtr const& child ) override; + + ITrackerPtr findChild( NameAndLocation const& nameAndLocation ) override; + ITracker& parent() override; + + void openChild() override; + + bool isSectionTracker() const override; + bool isIndexTracker() const override; + + void open(); + + void close() override; + void fail() override; + void markAsNeedingAnotherRun() override; + + private: + void moveToParent(); + void moveToThis(); + }; + + class SectionTracker : public TrackerBase { + std::vector m_filters; + public: + SectionTracker( NameAndLocation const& nameAndLocation, TrackerContext& ctx, ITracker* parent ); + + bool isSectionTracker() const override; + + static SectionTracker& acquire( TrackerContext& ctx, NameAndLocation const& nameAndLocation ); + + void tryOpen(); + + void addInitialFilters( std::vector const& filters ); + void addNextFilters( std::vector const& filters ); + }; + + class IndexTracker : public TrackerBase { + int m_size; + int m_index = -1; + public: + IndexTracker( NameAndLocation const& nameAndLocation, TrackerContext& ctx, ITracker* parent, int size ); + + bool isIndexTracker() const override; + void close() override; + + static IndexTracker& acquire( TrackerContext& ctx, NameAndLocation const& nameAndLocation, int size ); + + int index() const; + + void moveNext(); + }; + +} // namespace TestCaseTracking + +using TestCaseTracking::ITracker; +using TestCaseTracking::TrackerContext; +using TestCaseTracking::SectionTracker; +using TestCaseTracking::IndexTracker; + +} // namespace Catch + +// end catch_test_case_tracker.h + +// start catch_leak_detector.h + +namespace Catch { + + struct LeakDetector { + LeakDetector(); + }; + +} +// end catch_leak_detector.h +// Cpp files will be included in the single-header file here +// start catch_approx.cpp + +#include +#include + +namespace { + +// Performs equivalent check of std::fabs(lhs - rhs) <= margin +// But without the subtraction to allow for INFINITY in comparison +bool marginComparison(double lhs, double rhs, double margin) { + return (lhs + margin >= rhs) && (rhs + margin >= lhs); +} + +} + +namespace Catch { +namespace Detail { + + Approx::Approx ( double value ) + : m_epsilon( std::numeric_limits::epsilon()*100 ), + m_margin( 0.0 ), + m_scale( 0.0 ), + m_value( value ) + {} + + Approx Approx::custom() { + return Approx( 0 ); + } + + Approx Approx::operator-() const { + auto temp(*this); + temp.m_value = -temp.m_value; + return temp; + } + + std::string Approx::toString() const { + ReusableStringStream rss; + rss << "Approx( " << ::Catch::Detail::stringify( m_value ) << " )"; + return rss.str(); + } + + bool Approx::equalityComparisonImpl(const double other) const { + // First try with fixed margin, then compute margin based on epsilon, scale and Approx's value + // Thanks to Richard Harris for his help refining the scaled margin value + return marginComparison(m_value, other, m_margin) || marginComparison(m_value, other, m_epsilon * (m_scale + std::fabs(m_value))); + } + +} // end namespace Detail + +namespace literals { + Detail::Approx operator "" _a(long double val) { + return Detail::Approx(val); + } + Detail::Approx operator "" _a(unsigned long long val) { + return Detail::Approx(val); + } +} // end namespace literals + +std::string StringMaker::convert(Catch::Detail::Approx const& value) { + return value.toString(); +} + +} // end namespace Catch +// end catch_approx.cpp +// start catch_assertionhandler.cpp + +// start catch_context.h + +#include + +namespace Catch { + + struct IResultCapture; + struct IRunner; + struct IConfig; + struct IMutableContext; + + using IConfigPtr = std::shared_ptr; + + struct IContext + { + virtual ~IContext(); + + virtual IResultCapture* getResultCapture() = 0; + virtual IRunner* getRunner() = 0; + virtual IConfigPtr const& getConfig() const = 0; + }; + + struct IMutableContext : IContext + { + virtual ~IMutableContext(); + virtual void setResultCapture( IResultCapture* resultCapture ) = 0; + virtual void setRunner( IRunner* runner ) = 0; + virtual void setConfig( IConfigPtr const& config ) = 0; + + private: + static IMutableContext *currentContext; + friend IMutableContext& getCurrentMutableContext(); + friend void cleanUpContext(); + static void createContext(); + }; + + inline IMutableContext& getCurrentMutableContext() + { + if( !IMutableContext::currentContext ) + IMutableContext::createContext(); + return *IMutableContext::currentContext; + } + + inline IContext& getCurrentContext() + { + return getCurrentMutableContext(); + } + + void cleanUpContext(); +} + +// end catch_context.h +// start catch_debugger.h + +namespace Catch { + bool isDebuggerActive(); +} + +#ifdef CATCH_PLATFORM_MAC + + #define CATCH_TRAP() __asm__("int $3\n" : : ) /* NOLINT */ + +#elif defined(CATCH_PLATFORM_LINUX) + // If we can use inline assembler, do it because this allows us to break + // directly at the location of the failing check instead of breaking inside + // raise() called from it, i.e. one stack frame below. + #if defined(__GNUC__) && (defined(__i386) || defined(__x86_64)) + #define CATCH_TRAP() asm volatile ("int $3") /* NOLINT */ + #else // Fall back to the generic way. + #include + + #define CATCH_TRAP() raise(SIGTRAP) + #endif +#elif defined(_MSC_VER) + #define CATCH_TRAP() __debugbreak() +#elif defined(__MINGW32__) + extern "C" __declspec(dllimport) void __stdcall DebugBreak(); + #define CATCH_TRAP() DebugBreak() +#endif + +#ifdef CATCH_TRAP + #define CATCH_BREAK_INTO_DEBUGGER() if( Catch::isDebuggerActive() ) { CATCH_TRAP(); } +#else + namespace Catch { + inline void doNothing() {} + } + #define CATCH_BREAK_INTO_DEBUGGER() Catch::doNothing() +#endif + +// end catch_debugger.h +// start catch_run_context.h + +// start catch_fatal_condition.h + +// start catch_windows_h_proxy.h + + +#if defined(CATCH_PLATFORM_WINDOWS) + +#if !defined(NOMINMAX) && !defined(CATCH_CONFIG_NO_NOMINMAX) +# define CATCH_DEFINED_NOMINMAX +# define NOMINMAX +#endif +#if !defined(WIN32_LEAN_AND_MEAN) && !defined(CATCH_CONFIG_NO_WIN32_LEAN_AND_MEAN) +# define CATCH_DEFINED_WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +#ifdef __AFXDLL +#include +#else +#include +#endif + +#ifdef CATCH_DEFINED_NOMINMAX +# undef NOMINMAX +#endif +#ifdef CATCH_DEFINED_WIN32_LEAN_AND_MEAN +# undef WIN32_LEAN_AND_MEAN +#endif + +#endif // defined(CATCH_PLATFORM_WINDOWS) + +// end catch_windows_h_proxy.h +#if defined( CATCH_CONFIG_WINDOWS_SEH ) + +namespace Catch { + + struct FatalConditionHandler { + + static LONG CALLBACK handleVectoredException(PEXCEPTION_POINTERS ExceptionInfo); + FatalConditionHandler(); + static void reset(); + ~FatalConditionHandler(); + + private: + static bool isSet; + static ULONG guaranteeSize; + static PVOID exceptionHandlerHandle; + }; + +} // namespace Catch + +#elif defined ( CATCH_CONFIG_POSIX_SIGNALS ) + +#include + +namespace Catch { + + struct FatalConditionHandler { + + static bool isSet; + static struct sigaction oldSigActions[]; + static stack_t oldSigStack; + static char altStackMem[]; + + static void handleSignal( int sig ); + + FatalConditionHandler(); + ~FatalConditionHandler(); + static void reset(); + }; + +} // namespace Catch + +#else + +namespace Catch { + struct FatalConditionHandler { + void reset(); + }; +} + +#endif + +// end catch_fatal_condition.h +#include + +namespace Catch { + + struct IMutableContext; + + /////////////////////////////////////////////////////////////////////////// + + class RunContext : public IResultCapture, public IRunner { + + public: + RunContext( RunContext const& ) = delete; + RunContext& operator =( RunContext const& ) = delete; + + explicit RunContext( IConfigPtr const& _config, IStreamingReporterPtr&& reporter ); + + ~RunContext() override; + + void testGroupStarting( std::string const& testSpec, std::size_t groupIndex, std::size_t groupsCount ); + void testGroupEnded( std::string const& testSpec, Totals const& totals, std::size_t groupIndex, std::size_t groupsCount ); + + Totals runTest(TestCase const& testCase); + + IConfigPtr config() const; + IStreamingReporter& reporter() const; + + public: // IResultCapture + + // Assertion handlers + void handleExpr + ( AssertionInfo const& info, + ITransientExpression const& expr, + AssertionReaction& reaction ) override; + void handleMessage + ( AssertionInfo const& info, + ResultWas::OfType resultType, + StringRef const& message, + AssertionReaction& reaction ) override; + void handleUnexpectedExceptionNotThrown + ( AssertionInfo const& info, + AssertionReaction& reaction ) override; + void handleUnexpectedInflightException + ( AssertionInfo const& info, + std::string const& message, + AssertionReaction& reaction ) override; + void handleIncomplete + ( AssertionInfo const& info ) override; + void handleNonExpr + ( AssertionInfo const &info, + ResultWas::OfType resultType, + AssertionReaction &reaction ) override; + + bool sectionStarted( SectionInfo const& sectionInfo, Counts& assertions ) override; + + void sectionEnded( SectionEndInfo const& endInfo ) override; + void sectionEndedEarly( SectionEndInfo const& endInfo ) override; + + void benchmarkStarting( BenchmarkInfo const& info ) override; + void benchmarkEnded( BenchmarkStats const& stats ) override; + + void pushScopedMessage( MessageInfo const& message ) override; + void popScopedMessage( MessageInfo const& message ) override; + + std::string getCurrentTestName() const override; + + const AssertionResult* getLastResult() const override; + + void exceptionEarlyReported() override; + + void handleFatalErrorCondition( StringRef message ) override; + + bool lastAssertionPassed() override; + + void assertionPassed() override; + + public: + // !TBD We need to do this another way! + bool aborting() const final; + + private: + + void runCurrentTest( std::string& redirectedCout, std::string& redirectedCerr ); + void invokeActiveTestCase(); + + void resetAssertionInfo(); + bool testForMissingAssertions( Counts& assertions ); + + void assertionEnded( AssertionResult const& result ); + void reportExpr + ( AssertionInfo const &info, + ResultWas::OfType resultType, + ITransientExpression const *expr, + bool negated ); + + void populateReaction( AssertionReaction& reaction ); + + private: + + void handleUnfinishedSections(); + + TestRunInfo m_runInfo; + IMutableContext& m_context; + TestCase const* m_activeTestCase = nullptr; + ITracker* m_testCaseTracker; + Option m_lastResult; + + IConfigPtr m_config; + Totals m_totals; + IStreamingReporterPtr m_reporter; + std::vector m_messages; + AssertionInfo m_lastAssertionInfo; + std::vector m_unfinishedSections; + std::vector m_activeSections; + TrackerContext m_trackerContext; + bool m_lastAssertionPassed = false; + bool m_shouldReportUnexpected = true; + bool m_includeSuccessfulResults; + }; + +} // end namespace Catch + +// end catch_run_context.h +namespace Catch { + + namespace { + auto operator <<( std::ostream& os, ITransientExpression const& expr ) -> std::ostream& { + expr.streamReconstructedExpression( os ); + return os; + } + } + + LazyExpression::LazyExpression( bool isNegated ) + : m_isNegated( isNegated ) + {} + + LazyExpression::LazyExpression( LazyExpression const& other ) : m_isNegated( other.m_isNegated ) {} + + LazyExpression::operator bool() const { + return m_transientExpression != nullptr; + } + + auto operator << ( std::ostream& os, LazyExpression const& lazyExpr ) -> std::ostream& { + if( lazyExpr.m_isNegated ) + os << "!"; + + if( lazyExpr ) { + if( lazyExpr.m_isNegated && lazyExpr.m_transientExpression->isBinaryExpression() ) + os << "(" << *lazyExpr.m_transientExpression << ")"; + else + os << *lazyExpr.m_transientExpression; + } + else { + os << "{** error - unchecked empty expression requested **}"; + } + return os; + } + + AssertionHandler::AssertionHandler + ( StringRef macroName, + SourceLineInfo const& lineInfo, + StringRef capturedExpression, + ResultDisposition::Flags resultDisposition ) + : m_assertionInfo{ macroName, lineInfo, capturedExpression, resultDisposition }, + m_resultCapture( getResultCapture() ) + {} + + void AssertionHandler::handleExpr( ITransientExpression const& expr ) { + m_resultCapture.handleExpr( m_assertionInfo, expr, m_reaction ); + } + void AssertionHandler::handleMessage(ResultWas::OfType resultType, StringRef const& message) { + m_resultCapture.handleMessage( m_assertionInfo, resultType, message, m_reaction ); + } + + auto AssertionHandler::allowThrows() const -> bool { + return getCurrentContext().getConfig()->allowThrows(); + } + + void AssertionHandler::complete() { + setCompleted(); + if( m_reaction.shouldDebugBreak ) { + + // If you find your debugger stopping you here then go one level up on the + // call-stack for the code that caused it (typically a failed assertion) + + // (To go back to the test and change execution, jump over the throw, next) + CATCH_BREAK_INTO_DEBUGGER(); + } + if( m_reaction.shouldThrow ) + throw Catch::TestFailureException(); + } + void AssertionHandler::setCompleted() { + m_completed = true; + } + + void AssertionHandler::handleUnexpectedInflightException() { + m_resultCapture.handleUnexpectedInflightException( m_assertionInfo, Catch::translateActiveException(), m_reaction ); + } + + void AssertionHandler::handleExceptionThrownAsExpected() { + m_resultCapture.handleNonExpr(m_assertionInfo, ResultWas::Ok, m_reaction); + } + void AssertionHandler::handleExceptionNotThrownAsExpected() { + m_resultCapture.handleNonExpr(m_assertionInfo, ResultWas::Ok, m_reaction); + } + + void AssertionHandler::handleUnexpectedExceptionNotThrown() { + m_resultCapture.handleUnexpectedExceptionNotThrown( m_assertionInfo, m_reaction ); + } + + void AssertionHandler::handleThrowingCallSkipped() { + m_resultCapture.handleNonExpr(m_assertionInfo, ResultWas::Ok, m_reaction); + } + + // This is the overload that takes a string and infers the Equals matcher from it + // The more general overload, that takes any string matcher, is in catch_capture_matchers.cpp + void handleExceptionMatchExpr( AssertionHandler& handler, std::string const& str, StringRef matcherString ) { + handleExceptionMatchExpr( handler, Matchers::Equals( str ), matcherString ); + } + +} // namespace Catch +// end catch_assertionhandler.cpp +// start catch_assertionresult.cpp + +namespace Catch { + AssertionResultData::AssertionResultData(ResultWas::OfType _resultType, LazyExpression const & _lazyExpression): + lazyExpression(_lazyExpression), + resultType(_resultType) {} + + std::string AssertionResultData::reconstructExpression() const { + + if( reconstructedExpression.empty() ) { + if( lazyExpression ) { + ReusableStringStream rss; + rss << lazyExpression; + reconstructedExpression = rss.str(); + } + } + return reconstructedExpression; + } + + AssertionResult::AssertionResult( AssertionInfo const& info, AssertionResultData const& data ) + : m_info( info ), + m_resultData( data ) + {} + + // Result was a success + bool AssertionResult::succeeded() const { + return Catch::isOk( m_resultData.resultType ); + } + + // Result was a success, or failure is suppressed + bool AssertionResult::isOk() const { + return Catch::isOk( m_resultData.resultType ) || shouldSuppressFailure( m_info.resultDisposition ); + } + + ResultWas::OfType AssertionResult::getResultType() const { + return m_resultData.resultType; + } + + bool AssertionResult::hasExpression() const { + return m_info.capturedExpression[0] != 0; + } + + bool AssertionResult::hasMessage() const { + return !m_resultData.message.empty(); + } + + std::string AssertionResult::getExpression() const { + if( isFalseTest( m_info.resultDisposition ) ) + return "!(" + m_info.capturedExpression + ")"; + else + return m_info.capturedExpression; + } + + std::string AssertionResult::getExpressionInMacro() const { + std::string expr; + if( m_info.macroName[0] == 0 ) + expr = m_info.capturedExpression; + else { + expr.reserve( m_info.macroName.size() + m_info.capturedExpression.size() + 4 ); + expr += m_info.macroName; + expr += "( "; + expr += m_info.capturedExpression; + expr += " )"; + } + return expr; + } + + bool AssertionResult::hasExpandedExpression() const { + return hasExpression() && getExpandedExpression() != getExpression(); + } + + std::string AssertionResult::getExpandedExpression() const { + std::string expr = m_resultData.reconstructExpression(); + return expr.empty() + ? getExpression() + : expr; + } + + std::string AssertionResult::getMessage() const { + return m_resultData.message; + } + SourceLineInfo AssertionResult::getSourceInfo() const { + return m_info.lineInfo; + } + + StringRef AssertionResult::getTestMacroName() const { + return m_info.macroName; + } + +} // end namespace Catch +// end catch_assertionresult.cpp +// start catch_benchmark.cpp + +namespace Catch { + + auto BenchmarkLooper::getResolution() -> uint64_t { + return getEstimatedClockResolution() * getCurrentContext().getConfig()->benchmarkResolutionMultiple(); + } + + void BenchmarkLooper::reportStart() { + getResultCapture().benchmarkStarting( { m_name } ); + } + auto BenchmarkLooper::needsMoreIterations() -> bool { + auto elapsed = m_timer.getElapsedNanoseconds(); + + // Exponentially increasing iterations until we're confident in our timer resolution + if( elapsed < m_resolution ) { + m_iterationsToRun *= 10; + return true; + } + + getResultCapture().benchmarkEnded( { { m_name }, m_count, elapsed } ); + return false; + } + +} // end namespace Catch +// end catch_benchmark.cpp +// start catch_capture_matchers.cpp + +namespace Catch { + + using StringMatcher = Matchers::Impl::MatcherBase; + + // This is the general overload that takes a any string matcher + // There is another overload, in catch_assertionhandler.h/.cpp, that only takes a string and infers + // the Equals matcher (so the header does not mention matchers) + void handleExceptionMatchExpr( AssertionHandler& handler, StringMatcher const& matcher, StringRef matcherString ) { + std::string exceptionMessage = Catch::translateActiveException(); + MatchExpr expr( exceptionMessage, matcher, matcherString ); + handler.handleExpr( expr ); + } + +} // namespace Catch +// end catch_capture_matchers.cpp +// start catch_commandline.cpp + +// start catch_commandline.h + +// start catch_clara.h + +// Use Catch's value for console width (store Clara's off to the side, if present) +#ifdef CLARA_CONFIG_CONSOLE_WIDTH +#define CATCH_TEMP_CLARA_CONFIG_CONSOLE_WIDTH CATCH_CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH +#undef CATCH_CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH +#endif +#define CATCH_CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH CATCH_CONFIG_CONSOLE_WIDTH-1 + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wweak-vtables" +#pragma clang diagnostic ignored "-Wexit-time-destructors" +#pragma clang diagnostic ignored "-Wshadow" +#endif + +// start clara.hpp +// Copyright 2017 Two Blue Cubes Ltd. All rights reserved. +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// See https://github.com/philsquared/Clara for more details + +// Clara v1.1.4 + + +#ifndef CATCH_CLARA_CONFIG_CONSOLE_WIDTH +#define CATCH_CLARA_CONFIG_CONSOLE_WIDTH 80 +#endif + +#ifndef CATCH_CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH +#define CATCH_CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH CATCH_CLARA_CONFIG_CONSOLE_WIDTH +#endif + +#ifndef CLARA_CONFIG_OPTIONAL_TYPE +#ifdef __has_include +#if __has_include() && __cplusplus >= 201703L +#include +#define CLARA_CONFIG_OPTIONAL_TYPE std::optional +#endif +#endif +#endif + +// ----------- #included from clara_textflow.hpp ----------- + +// TextFlowCpp +// +// A single-header library for wrapping and laying out basic text, by Phil Nash +// +// This work is licensed under the BSD 2-Clause license. +// See the accompanying LICENSE file, or the one at https://opensource.org/licenses/BSD-2-Clause +// +// This project is hosted at https://github.com/philsquared/textflowcpp + + +#include +#include +#include +#include + +#ifndef CATCH_CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH +#define CATCH_CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH 80 +#endif + +namespace Catch { namespace clara { namespace TextFlow { + + inline auto isWhitespace( char c ) -> bool { + static std::string chars = " \t\n\r"; + return chars.find( c ) != std::string::npos; + } + inline auto isBreakableBefore( char c ) -> bool { + static std::string chars = "[({<|"; + return chars.find( c ) != std::string::npos; + } + inline auto isBreakableAfter( char c ) -> bool { + static std::string chars = "])}>.,:;*+-=&/\\"; + return chars.find( c ) != std::string::npos; + } + + class Columns; + + class Column { + std::vector m_strings; + size_t m_width = CATCH_CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH; + size_t m_indent = 0; + size_t m_initialIndent = std::string::npos; + + public: + class iterator { + friend Column; + + Column const& m_column; + size_t m_stringIndex = 0; + size_t m_pos = 0; + + size_t m_len = 0; + size_t m_end = 0; + bool m_suffix = false; + + iterator( Column const& column, size_t stringIndex ) + : m_column( column ), + m_stringIndex( stringIndex ) + {} + + auto line() const -> std::string const& { return m_column.m_strings[m_stringIndex]; } + + auto isBoundary( size_t at ) const -> bool { + assert( at > 0 ); + assert( at <= line().size() ); + + return at == line().size() || + ( isWhitespace( line()[at] ) && !isWhitespace( line()[at-1] ) ) || + isBreakableBefore( line()[at] ) || + isBreakableAfter( line()[at-1] ); + } + + void calcLength() { + assert( m_stringIndex < m_column.m_strings.size() ); + + m_suffix = false; + auto width = m_column.m_width-indent(); + m_end = m_pos; + while( m_end < line().size() && line()[m_end] != '\n' ) + ++m_end; + + if( m_end < m_pos + width ) { + m_len = m_end - m_pos; + } + else { + size_t len = width; + while (len > 0 && !isBoundary(m_pos + len)) + --len; + while (len > 0 && isWhitespace( line()[m_pos + len - 1] )) + --len; + + if (len > 0) { + m_len = len; + } else { + m_suffix = true; + m_len = width - 1; + } + } + } + + auto indent() const -> size_t { + auto initial = m_pos == 0 && m_stringIndex == 0 ? m_column.m_initialIndent : std::string::npos; + return initial == std::string::npos ? m_column.m_indent : initial; + } + + auto addIndentAndSuffix(std::string const &plain) const -> std::string { + return std::string( indent(), ' ' ) + (m_suffix ? plain + "-" : plain); + } + + public: + explicit iterator( Column const& column ) : m_column( column ) { + assert( m_column.m_width > m_column.m_indent ); + assert( m_column.m_initialIndent == std::string::npos || m_column.m_width > m_column.m_initialIndent ); + calcLength(); + if( m_len == 0 ) + m_stringIndex++; // Empty string + } + + auto operator *() const -> std::string { + assert( m_stringIndex < m_column.m_strings.size() ); + assert( m_pos <= m_end ); + if( m_pos + m_column.m_width < m_end ) + return addIndentAndSuffix(line().substr(m_pos, m_len)); + else + return addIndentAndSuffix(line().substr(m_pos, m_end - m_pos)); + } + + auto operator ++() -> iterator& { + m_pos += m_len; + if( m_pos < line().size() && line()[m_pos] == '\n' ) + m_pos += 1; + else + while( m_pos < line().size() && isWhitespace( line()[m_pos] ) ) + ++m_pos; + + if( m_pos == line().size() ) { + m_pos = 0; + ++m_stringIndex; + } + if( m_stringIndex < m_column.m_strings.size() ) + calcLength(); + return *this; + } + auto operator ++(int) -> iterator { + iterator prev( *this ); + operator++(); + return prev; + } + + auto operator ==( iterator const& other ) const -> bool { + return + m_pos == other.m_pos && + m_stringIndex == other.m_stringIndex && + &m_column == &other.m_column; + } + auto operator !=( iterator const& other ) const -> bool { + return !operator==( other ); + } + }; + using const_iterator = iterator; + + explicit Column( std::string const& text ) { m_strings.push_back( text ); } + + auto width( size_t newWidth ) -> Column& { + assert( newWidth > 0 ); + m_width = newWidth; + return *this; + } + auto indent( size_t newIndent ) -> Column& { + m_indent = newIndent; + return *this; + } + auto initialIndent( size_t newIndent ) -> Column& { + m_initialIndent = newIndent; + return *this; + } + + auto width() const -> size_t { return m_width; } + auto begin() const -> iterator { return iterator( *this ); } + auto end() const -> iterator { return { *this, m_strings.size() }; } + + inline friend std::ostream& operator << ( std::ostream& os, Column const& col ) { + bool first = true; + for( auto line : col ) { + if( first ) + first = false; + else + os << "\n"; + os << line; + } + return os; + } + + auto operator + ( Column const& other ) -> Columns; + + auto toString() const -> std::string { + std::ostringstream oss; + oss << *this; + return oss.str(); + } + }; + + class Spacer : public Column { + + public: + explicit Spacer( size_t spaceWidth ) : Column( "" ) { + width( spaceWidth ); + } + }; + + class Columns { + std::vector m_columns; + + public: + + class iterator { + friend Columns; + struct EndTag {}; + + std::vector const& m_columns; + std::vector m_iterators; + size_t m_activeIterators; + + iterator( Columns const& columns, EndTag ) + : m_columns( columns.m_columns ), + m_activeIterators( 0 ) + { + m_iterators.reserve( m_columns.size() ); + + for( auto const& col : m_columns ) + m_iterators.push_back( col.end() ); + } + + public: + explicit iterator( Columns const& columns ) + : m_columns( columns.m_columns ), + m_activeIterators( m_columns.size() ) + { + m_iterators.reserve( m_columns.size() ); + + for( auto const& col : m_columns ) + m_iterators.push_back( col.begin() ); + } + + auto operator ==( iterator const& other ) const -> bool { + return m_iterators == other.m_iterators; + } + auto operator !=( iterator const& other ) const -> bool { + return m_iterators != other.m_iterators; + } + auto operator *() const -> std::string { + std::string row, padding; + + for( size_t i = 0; i < m_columns.size(); ++i ) { + auto width = m_columns[i].width(); + if( m_iterators[i] != m_columns[i].end() ) { + std::string col = *m_iterators[i]; + row += padding + col; + if( col.size() < width ) + padding = std::string( width - col.size(), ' ' ); + else + padding = ""; + } + else { + padding += std::string( width, ' ' ); + } + } + return row; + } + auto operator ++() -> iterator& { + for( size_t i = 0; i < m_columns.size(); ++i ) { + if (m_iterators[i] != m_columns[i].end()) + ++m_iterators[i]; + } + return *this; + } + auto operator ++(int) -> iterator { + iterator prev( *this ); + operator++(); + return prev; + } + }; + using const_iterator = iterator; + + auto begin() const -> iterator { return iterator( *this ); } + auto end() const -> iterator { return { *this, iterator::EndTag() }; } + + auto operator += ( Column const& col ) -> Columns& { + m_columns.push_back( col ); + return *this; + } + auto operator + ( Column const& col ) -> Columns { + Columns combined = *this; + combined += col; + return combined; + } + + inline friend std::ostream& operator << ( std::ostream& os, Columns const& cols ) { + + bool first = true; + for( auto line : cols ) { + if( first ) + first = false; + else + os << "\n"; + os << line; + } + return os; + } + + auto toString() const -> std::string { + std::ostringstream oss; + oss << *this; + return oss.str(); + } + }; + + inline auto Column::operator + ( Column const& other ) -> Columns { + Columns cols; + cols += *this; + cols += other; + return cols; + } +}}} // namespace Catch::clara::TextFlow + +// ----------- end of #include from clara_textflow.hpp ----------- +// ........... back in clara.hpp + +#include +#include +#include + +#if !defined(CATCH_PLATFORM_WINDOWS) && ( defined(WIN32) || defined(__WIN32__) || defined(_WIN32) || defined(_MSC_VER) ) +#define CATCH_PLATFORM_WINDOWS +#endif + +namespace Catch { namespace clara { +namespace detail { + + // Traits for extracting arg and return type of lambdas (for single argument lambdas) + template + struct UnaryLambdaTraits : UnaryLambdaTraits {}; + + template + struct UnaryLambdaTraits { + static const bool isValid = false; + }; + + template + struct UnaryLambdaTraits { + static const bool isValid = true; + using ArgType = typename std::remove_const::type>::type; + using ReturnType = ReturnT; + }; + + class TokenStream; + + // Transport for raw args (copied from main args, or supplied via init list for testing) + class Args { + friend TokenStream; + std::string m_exeName; + std::vector m_args; + + public: + Args( int argc, char const* const* argv ) + : m_exeName(argv[0]), + m_args(argv + 1, argv + argc) {} + + Args( std::initializer_list args ) + : m_exeName( *args.begin() ), + m_args( args.begin()+1, args.end() ) + {} + + auto exeName() const -> std::string { + return m_exeName; + } + }; + + // Wraps a token coming from a token stream. These may not directly correspond to strings as a single string + // may encode an option + its argument if the : or = form is used + enum class TokenType { + Option, Argument + }; + struct Token { + TokenType type; + std::string token; + }; + + inline auto isOptPrefix( char c ) -> bool { + return c == '-' +#ifdef CATCH_PLATFORM_WINDOWS + || c == '/' +#endif + ; + } + + // Abstracts iterators into args as a stream of tokens, with option arguments uniformly handled + class TokenStream { + using Iterator = std::vector::const_iterator; + Iterator it; + Iterator itEnd; + std::vector m_tokenBuffer; + + void loadBuffer() { + m_tokenBuffer.resize( 0 ); + + // Skip any empty strings + while( it != itEnd && it->empty() ) + ++it; + + if( it != itEnd ) { + auto const &next = *it; + if( isOptPrefix( next[0] ) ) { + auto delimiterPos = next.find_first_of( " :=" ); + if( delimiterPos != std::string::npos ) { + m_tokenBuffer.push_back( { TokenType::Option, next.substr( 0, delimiterPos ) } ); + m_tokenBuffer.push_back( { TokenType::Argument, next.substr( delimiterPos + 1 ) } ); + } else { + if( next[1] != '-' && next.size() > 2 ) { + std::string opt = "- "; + for( size_t i = 1; i < next.size(); ++i ) { + opt[1] = next[i]; + m_tokenBuffer.push_back( { TokenType::Option, opt } ); + } + } else { + m_tokenBuffer.push_back( { TokenType::Option, next } ); + } + } + } else { + m_tokenBuffer.push_back( { TokenType::Argument, next } ); + } + } + } + + public: + explicit TokenStream( Args const &args ) : TokenStream( args.m_args.begin(), args.m_args.end() ) {} + + TokenStream( Iterator it, Iterator itEnd ) : it( it ), itEnd( itEnd ) { + loadBuffer(); + } + + explicit operator bool() const { + return !m_tokenBuffer.empty() || it != itEnd; + } + + auto count() const -> size_t { return m_tokenBuffer.size() + (itEnd - it); } + + auto operator*() const -> Token { + assert( !m_tokenBuffer.empty() ); + return m_tokenBuffer.front(); + } + + auto operator->() const -> Token const * { + assert( !m_tokenBuffer.empty() ); + return &m_tokenBuffer.front(); + } + + auto operator++() -> TokenStream & { + if( m_tokenBuffer.size() >= 2 ) { + m_tokenBuffer.erase( m_tokenBuffer.begin() ); + } else { + if( it != itEnd ) + ++it; + loadBuffer(); + } + return *this; + } + }; + + class ResultBase { + public: + enum Type { + Ok, LogicError, RuntimeError + }; + + protected: + ResultBase( Type type ) : m_type( type ) {} + virtual ~ResultBase() = default; + + virtual void enforceOk() const = 0; + + Type m_type; + }; + + template + class ResultValueBase : public ResultBase { + public: + auto value() const -> T const & { + enforceOk(); + return m_value; + } + + protected: + ResultValueBase( Type type ) : ResultBase( type ) {} + + ResultValueBase( ResultValueBase const &other ) : ResultBase( other ) { + if( m_type == ResultBase::Ok ) + new( &m_value ) T( other.m_value ); + } + + ResultValueBase( Type, T const &value ) : ResultBase( Ok ) { + new( &m_value ) T( value ); + } + + auto operator=( ResultValueBase const &other ) -> ResultValueBase & { + if( m_type == ResultBase::Ok ) + m_value.~T(); + ResultBase::operator=(other); + if( m_type == ResultBase::Ok ) + new( &m_value ) T( other.m_value ); + return *this; + } + + ~ResultValueBase() override { + if( m_type == Ok ) + m_value.~T(); + } + + union { + T m_value; + }; + }; + + template<> + class ResultValueBase : public ResultBase { + protected: + using ResultBase::ResultBase; + }; + + template + class BasicResult : public ResultValueBase { + public: + template + explicit BasicResult( BasicResult const &other ) + : ResultValueBase( other.type() ), + m_errorMessage( other.errorMessage() ) + { + assert( type() != ResultBase::Ok ); + } + + template + static auto ok( U const &value ) -> BasicResult { return { ResultBase::Ok, value }; } + static auto ok() -> BasicResult { return { ResultBase::Ok }; } + static auto logicError( std::string const &message ) -> BasicResult { return { ResultBase::LogicError, message }; } + static auto runtimeError( std::string const &message ) -> BasicResult { return { ResultBase::RuntimeError, message }; } + + explicit operator bool() const { return m_type == ResultBase::Ok; } + auto type() const -> ResultBase::Type { return m_type; } + auto errorMessage() const -> std::string { return m_errorMessage; } + + protected: + void enforceOk() const override { + + // Errors shouldn't reach this point, but if they do + // the actual error message will be in m_errorMessage + assert( m_type != ResultBase::LogicError ); + assert( m_type != ResultBase::RuntimeError ); + if( m_type != ResultBase::Ok ) + std::abort(); + } + + std::string m_errorMessage; // Only populated if resultType is an error + + BasicResult( ResultBase::Type type, std::string const &message ) + : ResultValueBase(type), + m_errorMessage(message) + { + assert( m_type != ResultBase::Ok ); + } + + using ResultValueBase::ResultValueBase; + using ResultBase::m_type; + }; + + enum class ParseResultType { + Matched, NoMatch, ShortCircuitAll, ShortCircuitSame + }; + + class ParseState { + public: + + ParseState( ParseResultType type, TokenStream const &remainingTokens ) + : m_type(type), + m_remainingTokens( remainingTokens ) + {} + + auto type() const -> ParseResultType { return m_type; } + auto remainingTokens() const -> TokenStream { return m_remainingTokens; } + + private: + ParseResultType m_type; + TokenStream m_remainingTokens; + }; + + using Result = BasicResult; + using ParserResult = BasicResult; + using InternalParseResult = BasicResult; + + struct HelpColumns { + std::string left; + std::string right; + }; + + template + inline auto convertInto( std::string const &source, T& target ) -> ParserResult { + std::stringstream ss; + ss << source; + ss >> target; + if( ss.fail() ) + return ParserResult::runtimeError( "Unable to convert '" + source + "' to destination type" ); + else + return ParserResult::ok( ParseResultType::Matched ); + } + inline auto convertInto( std::string const &source, std::string& target ) -> ParserResult { + target = source; + return ParserResult::ok( ParseResultType::Matched ); + } + inline auto convertInto( std::string const &source, bool &target ) -> ParserResult { + std::string srcLC = source; + std::transform( srcLC.begin(), srcLC.end(), srcLC.begin(), []( char c ) { return static_cast( ::tolower(c) ); } ); + if (srcLC == "y" || srcLC == "1" || srcLC == "true" || srcLC == "yes" || srcLC == "on") + target = true; + else if (srcLC == "n" || srcLC == "0" || srcLC == "false" || srcLC == "no" || srcLC == "off") + target = false; + else + return ParserResult::runtimeError( "Expected a boolean value but did not recognise: '" + source + "'" ); + return ParserResult::ok( ParseResultType::Matched ); + } +#ifdef CLARA_CONFIG_OPTIONAL_TYPE + template + inline auto convertInto( std::string const &source, CLARA_CONFIG_OPTIONAL_TYPE& target ) -> ParserResult { + T temp; + auto result = convertInto( source, temp ); + if( result ) + target = std::move(temp); + return result; + } +#endif // CLARA_CONFIG_OPTIONAL_TYPE + + struct NonCopyable { + NonCopyable() = default; + NonCopyable( NonCopyable const & ) = delete; + NonCopyable( NonCopyable && ) = delete; + NonCopyable &operator=( NonCopyable const & ) = delete; + NonCopyable &operator=( NonCopyable && ) = delete; + }; + + struct BoundRef : NonCopyable { + virtual ~BoundRef() = default; + virtual auto isContainer() const -> bool { return false; } + virtual auto isFlag() const -> bool { return false; } + }; + struct BoundValueRefBase : BoundRef { + virtual auto setValue( std::string const &arg ) -> ParserResult = 0; + }; + struct BoundFlagRefBase : BoundRef { + virtual auto setFlag( bool flag ) -> ParserResult = 0; + virtual auto isFlag() const -> bool { return true; } + }; + + template + struct BoundValueRef : BoundValueRefBase { + T &m_ref; + + explicit BoundValueRef( T &ref ) : m_ref( ref ) {} + + auto setValue( std::string const &arg ) -> ParserResult override { + return convertInto( arg, m_ref ); + } + }; + + template + struct BoundValueRef> : BoundValueRefBase { + std::vector &m_ref; + + explicit BoundValueRef( std::vector &ref ) : m_ref( ref ) {} + + auto isContainer() const -> bool override { return true; } + + auto setValue( std::string const &arg ) -> ParserResult override { + T temp; + auto result = convertInto( arg, temp ); + if( result ) + m_ref.push_back( temp ); + return result; + } + }; + + struct BoundFlagRef : BoundFlagRefBase { + bool &m_ref; + + explicit BoundFlagRef( bool &ref ) : m_ref( ref ) {} + + auto setFlag( bool flag ) -> ParserResult override { + m_ref = flag; + return ParserResult::ok( ParseResultType::Matched ); + } + }; + + template + struct LambdaInvoker { + static_assert( std::is_same::value, "Lambda must return void or clara::ParserResult" ); + + template + static auto invoke( L const &lambda, ArgType const &arg ) -> ParserResult { + return lambda( arg ); + } + }; + + template<> + struct LambdaInvoker { + template + static auto invoke( L const &lambda, ArgType const &arg ) -> ParserResult { + lambda( arg ); + return ParserResult::ok( ParseResultType::Matched ); + } + }; + + template + inline auto invokeLambda( L const &lambda, std::string const &arg ) -> ParserResult { + ArgType temp{}; + auto result = convertInto( arg, temp ); + return !result + ? result + : LambdaInvoker::ReturnType>::invoke( lambda, temp ); + } + + template + struct BoundLambda : BoundValueRefBase { + L m_lambda; + + static_assert( UnaryLambdaTraits::isValid, "Supplied lambda must take exactly one argument" ); + explicit BoundLambda( L const &lambda ) : m_lambda( lambda ) {} + + auto setValue( std::string const &arg ) -> ParserResult override { + return invokeLambda::ArgType>( m_lambda, arg ); + } + }; + + template + struct BoundFlagLambda : BoundFlagRefBase { + L m_lambda; + + static_assert( UnaryLambdaTraits::isValid, "Supplied lambda must take exactly one argument" ); + static_assert( std::is_same::ArgType, bool>::value, "flags must be boolean" ); + + explicit BoundFlagLambda( L const &lambda ) : m_lambda( lambda ) {} + + auto setFlag( bool flag ) -> ParserResult override { + return LambdaInvoker::ReturnType>::invoke( m_lambda, flag ); + } + }; + + enum class Optionality { Optional, Required }; + + struct Parser; + + class ParserBase { + public: + virtual ~ParserBase() = default; + virtual auto validate() const -> Result { return Result::ok(); } + virtual auto parse( std::string const& exeName, TokenStream const &tokens) const -> InternalParseResult = 0; + virtual auto cardinality() const -> size_t { return 1; } + + auto parse( Args const &args ) const -> InternalParseResult { + return parse( args.exeName(), TokenStream( args ) ); + } + }; + + template + class ComposableParserImpl : public ParserBase { + public: + template + auto operator|( T const &other ) const -> Parser; + + template + auto operator+( T const &other ) const -> Parser; + }; + + // Common code and state for Args and Opts + template + class ParserRefImpl : public ComposableParserImpl { + protected: + Optionality m_optionality = Optionality::Optional; + std::shared_ptr m_ref; + std::string m_hint; + std::string m_description; + + explicit ParserRefImpl( std::shared_ptr const &ref ) : m_ref( ref ) {} + + public: + template + ParserRefImpl( T &ref, std::string const &hint ) + : m_ref( std::make_shared>( ref ) ), + m_hint( hint ) + {} + + template + ParserRefImpl( LambdaT const &ref, std::string const &hint ) + : m_ref( std::make_shared>( ref ) ), + m_hint(hint) + {} + + auto operator()( std::string const &description ) -> DerivedT & { + m_description = description; + return static_cast( *this ); + } + + auto optional() -> DerivedT & { + m_optionality = Optionality::Optional; + return static_cast( *this ); + }; + + auto required() -> DerivedT & { + m_optionality = Optionality::Required; + return static_cast( *this ); + }; + + auto isOptional() const -> bool { + return m_optionality == Optionality::Optional; + } + + auto cardinality() const -> size_t override { + if( m_ref->isContainer() ) + return 0; + else + return 1; + } + + auto hint() const -> std::string { return m_hint; } + }; + + class ExeName : public ComposableParserImpl { + std::shared_ptr m_name; + std::shared_ptr m_ref; + + template + static auto makeRef(LambdaT const &lambda) -> std::shared_ptr { + return std::make_shared>( lambda) ; + } + + public: + ExeName() : m_name( std::make_shared( "" ) ) {} + + explicit ExeName( std::string &ref ) : ExeName() { + m_ref = std::make_shared>( ref ); + } + + template + explicit ExeName( LambdaT const& lambda ) : ExeName() { + m_ref = std::make_shared>( lambda ); + } + + // The exe name is not parsed out of the normal tokens, but is handled specially + auto parse( std::string const&, TokenStream const &tokens ) const -> InternalParseResult override { + return InternalParseResult::ok( ParseState( ParseResultType::NoMatch, tokens ) ); + } + + auto name() const -> std::string { return *m_name; } + auto set( std::string const& newName ) -> ParserResult { + + auto lastSlash = newName.find_last_of( "\\/" ); + auto filename = ( lastSlash == std::string::npos ) + ? newName + : newName.substr( lastSlash+1 ); + + *m_name = filename; + if( m_ref ) + return m_ref->setValue( filename ); + else + return ParserResult::ok( ParseResultType::Matched ); + } + }; + + class Arg : public ParserRefImpl { + public: + using ParserRefImpl::ParserRefImpl; + + auto parse( std::string const &, TokenStream const &tokens ) const -> InternalParseResult override { + auto validationResult = validate(); + if( !validationResult ) + return InternalParseResult( validationResult ); + + auto remainingTokens = tokens; + auto const &token = *remainingTokens; + if( token.type != TokenType::Argument ) + return InternalParseResult::ok( ParseState( ParseResultType::NoMatch, remainingTokens ) ); + + assert( !m_ref->isFlag() ); + auto valueRef = static_cast( m_ref.get() ); + + auto result = valueRef->setValue( remainingTokens->token ); + if( !result ) + return InternalParseResult( result ); + else + return InternalParseResult::ok( ParseState( ParseResultType::Matched, ++remainingTokens ) ); + } + }; + + inline auto normaliseOpt( std::string const &optName ) -> std::string { +#ifdef CATCH_PLATFORM_WINDOWS + if( optName[0] == '/' ) + return "-" + optName.substr( 1 ); + else +#endif + return optName; + } + + class Opt : public ParserRefImpl { + protected: + std::vector m_optNames; + + public: + template + explicit Opt( LambdaT const &ref ) : ParserRefImpl( std::make_shared>( ref ) ) {} + + explicit Opt( bool &ref ) : ParserRefImpl( std::make_shared( ref ) ) {} + + template + Opt( LambdaT const &ref, std::string const &hint ) : ParserRefImpl( ref, hint ) {} + + template + Opt( T &ref, std::string const &hint ) : ParserRefImpl( ref, hint ) {} + + auto operator[]( std::string const &optName ) -> Opt & { + m_optNames.push_back( optName ); + return *this; + } + + auto getHelpColumns() const -> std::vector { + std::ostringstream oss; + bool first = true; + for( auto const &opt : m_optNames ) { + if (first) + first = false; + else + oss << ", "; + oss << opt; + } + if( !m_hint.empty() ) + oss << " <" << m_hint << ">"; + return { { oss.str(), m_description } }; + } + + auto isMatch( std::string const &optToken ) const -> bool { + auto normalisedToken = normaliseOpt( optToken ); + for( auto const &name : m_optNames ) { + if( normaliseOpt( name ) == normalisedToken ) + return true; + } + return false; + } + + using ParserBase::parse; + + auto parse( std::string const&, TokenStream const &tokens ) const -> InternalParseResult override { + auto validationResult = validate(); + if( !validationResult ) + return InternalParseResult( validationResult ); + + auto remainingTokens = tokens; + if( remainingTokens && remainingTokens->type == TokenType::Option ) { + auto const &token = *remainingTokens; + if( isMatch(token.token ) ) { + if( m_ref->isFlag() ) { + auto flagRef = static_cast( m_ref.get() ); + auto result = flagRef->setFlag( true ); + if( !result ) + return InternalParseResult( result ); + if( result.value() == ParseResultType::ShortCircuitAll ) + return InternalParseResult::ok( ParseState( result.value(), remainingTokens ) ); + } else { + auto valueRef = static_cast( m_ref.get() ); + ++remainingTokens; + if( !remainingTokens ) + return InternalParseResult::runtimeError( "Expected argument following " + token.token ); + auto const &argToken = *remainingTokens; + if( argToken.type != TokenType::Argument ) + return InternalParseResult::runtimeError( "Expected argument following " + token.token ); + auto result = valueRef->setValue( argToken.token ); + if( !result ) + return InternalParseResult( result ); + if( result.value() == ParseResultType::ShortCircuitAll ) + return InternalParseResult::ok( ParseState( result.value(), remainingTokens ) ); + } + return InternalParseResult::ok( ParseState( ParseResultType::Matched, ++remainingTokens ) ); + } + } + return InternalParseResult::ok( ParseState( ParseResultType::NoMatch, remainingTokens ) ); + } + + auto validate() const -> Result override { + if( m_optNames.empty() ) + return Result::logicError( "No options supplied to Opt" ); + for( auto const &name : m_optNames ) { + if( name.empty() ) + return Result::logicError( "Option name cannot be empty" ); +#ifdef CATCH_PLATFORM_WINDOWS + if( name[0] != '-' && name[0] != '/' ) + return Result::logicError( "Option name must begin with '-' or '/'" ); +#else + if( name[0] != '-' ) + return Result::logicError( "Option name must begin with '-'" ); +#endif + } + return ParserRefImpl::validate(); + } + }; + + struct Help : Opt { + Help( bool &showHelpFlag ) + : Opt([&]( bool flag ) { + showHelpFlag = flag; + return ParserResult::ok( ParseResultType::ShortCircuitAll ); + }) + { + static_cast( *this ) + ("display usage information") + ["-?"]["-h"]["--help"] + .optional(); + } + }; + + struct Parser : ParserBase { + + mutable ExeName m_exeName; + std::vector m_options; + std::vector m_args; + + auto operator|=( ExeName const &exeName ) -> Parser & { + m_exeName = exeName; + return *this; + } + + auto operator|=( Arg const &arg ) -> Parser & { + m_args.push_back(arg); + return *this; + } + + auto operator|=( Opt const &opt ) -> Parser & { + m_options.push_back(opt); + return *this; + } + + auto operator|=( Parser const &other ) -> Parser & { + m_options.insert(m_options.end(), other.m_options.begin(), other.m_options.end()); + m_args.insert(m_args.end(), other.m_args.begin(), other.m_args.end()); + return *this; + } + + template + auto operator|( T const &other ) const -> Parser { + return Parser( *this ) |= other; + } + + // Forward deprecated interface with '+' instead of '|' + template + auto operator+=( T const &other ) -> Parser & { return operator|=( other ); } + template + auto operator+( T const &other ) const -> Parser { return operator|( other ); } + + auto getHelpColumns() const -> std::vector { + std::vector cols; + for (auto const &o : m_options) { + auto childCols = o.getHelpColumns(); + cols.insert( cols.end(), childCols.begin(), childCols.end() ); + } + return cols; + } + + void writeToStream( std::ostream &os ) const { + if (!m_exeName.name().empty()) { + os << "usage:\n" << " " << m_exeName.name() << " "; + bool required = true, first = true; + for( auto const &arg : m_args ) { + if (first) + first = false; + else + os << " "; + if( arg.isOptional() && required ) { + os << "["; + required = false; + } + os << "<" << arg.hint() << ">"; + if( arg.cardinality() == 0 ) + os << " ... "; + } + if( !required ) + os << "]"; + if( !m_options.empty() ) + os << " options"; + os << "\n\nwhere options are:" << std::endl; + } + + auto rows = getHelpColumns(); + size_t consoleWidth = CATCH_CLARA_CONFIG_CONSOLE_WIDTH; + size_t optWidth = 0; + for( auto const &cols : rows ) + optWidth = (std::max)(optWidth, cols.left.size() + 2); + + optWidth = (std::min)(optWidth, consoleWidth/2); + + for( auto const &cols : rows ) { + auto row = + TextFlow::Column( cols.left ).width( optWidth ).indent( 2 ) + + TextFlow::Spacer(4) + + TextFlow::Column( cols.right ).width( consoleWidth - 7 - optWidth ); + os << row << std::endl; + } + } + + friend auto operator<<( std::ostream &os, Parser const &parser ) -> std::ostream& { + parser.writeToStream( os ); + return os; + } + + auto validate() const -> Result override { + for( auto const &opt : m_options ) { + auto result = opt.validate(); + if( !result ) + return result; + } + for( auto const &arg : m_args ) { + auto result = arg.validate(); + if( !result ) + return result; + } + return Result::ok(); + } + + using ParserBase::parse; + + auto parse( std::string const& exeName, TokenStream const &tokens ) const -> InternalParseResult override { + + struct ParserInfo { + ParserBase const* parser = nullptr; + size_t count = 0; + }; + const size_t totalParsers = m_options.size() + m_args.size(); + assert( totalParsers < 512 ); + // ParserInfo parseInfos[totalParsers]; // <-- this is what we really want to do + ParserInfo parseInfos[512]; + + { + size_t i = 0; + for (auto const &opt : m_options) parseInfos[i++].parser = &opt; + for (auto const &arg : m_args) parseInfos[i++].parser = &arg; + } + + m_exeName.set( exeName ); + + auto result = InternalParseResult::ok( ParseState( ParseResultType::NoMatch, tokens ) ); + while( result.value().remainingTokens() ) { + bool tokenParsed = false; + + for( size_t i = 0; i < totalParsers; ++i ) { + auto& parseInfo = parseInfos[i]; + if( parseInfo.parser->cardinality() == 0 || parseInfo.count < parseInfo.parser->cardinality() ) { + result = parseInfo.parser->parse(exeName, result.value().remainingTokens()); + if (!result) + return result; + if (result.value().type() != ParseResultType::NoMatch) { + tokenParsed = true; + ++parseInfo.count; + break; + } + } + } + + if( result.value().type() == ParseResultType::ShortCircuitAll ) + return result; + if( !tokenParsed ) + return InternalParseResult::runtimeError( "Unrecognised token: " + result.value().remainingTokens()->token ); + } + // !TBD Check missing required options + return result; + } + }; + + template + template + auto ComposableParserImpl::operator|( T const &other ) const -> Parser { + return Parser() | static_cast( *this ) | other; + } +} // namespace detail + +// A Combined parser +using detail::Parser; + +// A parser for options +using detail::Opt; + +// A parser for arguments +using detail::Arg; + +// Wrapper for argc, argv from main() +using detail::Args; + +// Specifies the name of the executable +using detail::ExeName; + +// Convenience wrapper for option parser that specifies the help option +using detail::Help; + +// enum of result types from a parse +using detail::ParseResultType; + +// Result type for parser operation +using detail::ParserResult; + +}} // namespace Catch::clara + +// end clara.hpp +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + +// Restore Clara's value for console width, if present +#ifdef CATCH_TEMP_CLARA_CONFIG_CONSOLE_WIDTH +#define CATCH_CLARA_TEXTFLOW_CONFIG_CONSOLE_WIDTH CATCH_TEMP_CLARA_CONFIG_CONSOLE_WIDTH +#undef CATCH_TEMP_CLARA_CONFIG_CONSOLE_WIDTH +#endif + +// end catch_clara.h +namespace Catch { + + clara::Parser makeCommandLineParser( ConfigData& config ); + +} // end namespace Catch + +// end catch_commandline.h +#include +#include + +namespace Catch { + + clara::Parser makeCommandLineParser( ConfigData& config ) { + + using namespace clara; + + auto const setWarning = [&]( std::string const& warning ) { + auto warningSet = [&]() { + if( warning == "NoAssertions" ) + return WarnAbout::NoAssertions; + + if ( warning == "NoTests" ) + return WarnAbout::NoTests; + + return WarnAbout::Nothing; + }(); + + if (warningSet == WarnAbout::Nothing) + return ParserResult::runtimeError( "Unrecognised warning: '" + warning + "'" ); + config.warnings = static_cast( config.warnings | warningSet ); + return ParserResult::ok( ParseResultType::Matched ); + }; + auto const loadTestNamesFromFile = [&]( std::string const& filename ) { + std::ifstream f( filename.c_str() ); + if( !f.is_open() ) + return ParserResult::runtimeError( "Unable to load input file: '" + filename + "'" ); + + std::string line; + while( std::getline( f, line ) ) { + line = trim(line); + if( !line.empty() && !startsWith( line, '#' ) ) { + if( !startsWith( line, '"' ) ) + line = '"' + line + '"'; + config.testsOrTags.push_back( line + ',' ); + } + } + return ParserResult::ok( ParseResultType::Matched ); + }; + auto const setTestOrder = [&]( std::string const& order ) { + if( startsWith( "declared", order ) ) + config.runOrder = RunTests::InDeclarationOrder; + else if( startsWith( "lexical", order ) ) + config.runOrder = RunTests::InLexicographicalOrder; + else if( startsWith( "random", order ) ) + config.runOrder = RunTests::InRandomOrder; + else + return clara::ParserResult::runtimeError( "Unrecognised ordering: '" + order + "'" ); + return ParserResult::ok( ParseResultType::Matched ); + }; + auto const setRngSeed = [&]( std::string const& seed ) { + if( seed != "time" ) + return clara::detail::convertInto( seed, config.rngSeed ); + config.rngSeed = static_cast( std::time(nullptr) ); + return ParserResult::ok( ParseResultType::Matched ); + }; + auto const setColourUsage = [&]( std::string const& useColour ) { + auto mode = toLower( useColour ); + + if( mode == "yes" ) + config.useColour = UseColour::Yes; + else if( mode == "no" ) + config.useColour = UseColour::No; + else if( mode == "auto" ) + config.useColour = UseColour::Auto; + else + return ParserResult::runtimeError( "colour mode must be one of: auto, yes or no. '" + useColour + "' not recognised" ); + return ParserResult::ok( ParseResultType::Matched ); + }; + auto const setWaitForKeypress = [&]( std::string const& keypress ) { + auto keypressLc = toLower( keypress ); + if( keypressLc == "start" ) + config.waitForKeypress = WaitForKeypress::BeforeStart; + else if( keypressLc == "exit" ) + config.waitForKeypress = WaitForKeypress::BeforeExit; + else if( keypressLc == "both" ) + config.waitForKeypress = WaitForKeypress::BeforeStartAndExit; + else + return ParserResult::runtimeError( "keypress argument must be one of: start, exit or both. '" + keypress + "' not recognised" ); + return ParserResult::ok( ParseResultType::Matched ); + }; + auto const setVerbosity = [&]( std::string const& verbosity ) { + auto lcVerbosity = toLower( verbosity ); + if( lcVerbosity == "quiet" ) + config.verbosity = Verbosity::Quiet; + else if( lcVerbosity == "normal" ) + config.verbosity = Verbosity::Normal; + else if( lcVerbosity == "high" ) + config.verbosity = Verbosity::High; + else + return ParserResult::runtimeError( "Unrecognised verbosity, '" + verbosity + "'" ); + return ParserResult::ok( ParseResultType::Matched ); + }; + + auto cli + = ExeName( config.processName ) + | Help( config.showHelp ) + | Opt( config.listTests ) + ["-l"]["--list-tests"] + ( "list all/matching test cases" ) + | Opt( config.listTags ) + ["-t"]["--list-tags"] + ( "list all/matching tags" ) + | Opt( config.showSuccessfulTests ) + ["-s"]["--success"] + ( "include successful tests in output" ) + | Opt( config.shouldDebugBreak ) + ["-b"]["--break"] + ( "break into debugger on failure" ) + | Opt( config.noThrow ) + ["-e"]["--nothrow"] + ( "skip exception tests" ) + | Opt( config.showInvisibles ) + ["-i"]["--invisibles"] + ( "show invisibles (tabs, newlines)" ) + | Opt( config.outputFilename, "filename" ) + ["-o"]["--out"] + ( "output filename" ) + | Opt( config.reporterName, "name" ) + ["-r"]["--reporter"] + ( "reporter to use (defaults to console)" ) + | Opt( config.name, "name" ) + ["-n"]["--name"] + ( "suite name" ) + | Opt( [&]( bool ){ config.abortAfter = 1; } ) + ["-a"]["--abort"] + ( "abort at first failure" ) + | Opt( [&]( int x ){ config.abortAfter = x; }, "no. failures" ) + ["-x"]["--abortx"] + ( "abort after x failures" ) + | Opt( setWarning, "warning name" ) + ["-w"]["--warn"] + ( "enable warnings" ) + | Opt( [&]( bool flag ) { config.showDurations = flag ? ShowDurations::Always : ShowDurations::Never; }, "yes|no" ) + ["-d"]["--durations"] + ( "show test durations" ) + | Opt( loadTestNamesFromFile, "filename" ) + ["-f"]["--input-file"] + ( "load test names to run from a file" ) + | Opt( config.filenamesAsTags ) + ["-#"]["--filenames-as-tags"] + ( "adds a tag for the filename" ) + | Opt( config.sectionsToRun, "section name" ) + ["-c"]["--section"] + ( "specify section to run" ) + | Opt( setVerbosity, "quiet|normal|high" ) + ["-v"]["--verbosity"] + ( "set output verbosity" ) + | Opt( config.listTestNamesOnly ) + ["--list-test-names-only"] + ( "list all/matching test cases names only" ) + | Opt( config.listReporters ) + ["--list-reporters"] + ( "list all reporters" ) + | Opt( setTestOrder, "decl|lex|rand" ) + ["--order"] + ( "test case order (defaults to decl)" ) + | Opt( setRngSeed, "'time'|number" ) + ["--rng-seed"] + ( "set a specific seed for random numbers" ) + | Opt( setColourUsage, "yes|no" ) + ["--use-colour"] + ( "should output be colourised" ) + | Opt( config.libIdentify ) + ["--libidentify"] + ( "report name and version according to libidentify standard" ) + | Opt( setWaitForKeypress, "start|exit|both" ) + ["--wait-for-keypress"] + ( "waits for a keypress before exiting" ) + | Opt( config.benchmarkResolutionMultiple, "multiplier" ) + ["--benchmark-resolution-multiple"] + ( "multiple of clock resolution to run benchmarks" ) + + | Arg( config.testsOrTags, "test name|pattern|tags" ) + ( "which test or tests to use" ); + + return cli; + } + +} // end namespace Catch +// end catch_commandline.cpp +// start catch_common.cpp + +#include +#include + +namespace Catch { + + bool SourceLineInfo::empty() const noexcept { + return file[0] == '\0'; + } + bool SourceLineInfo::operator == ( SourceLineInfo const& other ) const noexcept { + return line == other.line && (file == other.file || std::strcmp(file, other.file) == 0); + } + bool SourceLineInfo::operator < ( SourceLineInfo const& other ) const noexcept { + return line < other.line || ( line == other.line && (std::strcmp(file, other.file) < 0)); + } + + std::ostream& operator << ( std::ostream& os, SourceLineInfo const& info ) { +#ifndef __GNUG__ + os << info.file << '(' << info.line << ')'; +#else + os << info.file << ':' << info.line; +#endif + return os; + } + + std::string StreamEndStop::operator+() const { + return std::string(); + } + + NonCopyable::NonCopyable() = default; + NonCopyable::~NonCopyable() = default; + +} +// end catch_common.cpp +// start catch_config.cpp + +// start catch_enforce.h + +#include + +#define CATCH_PREPARE_EXCEPTION( type, msg ) \ + type( ( Catch::ReusableStringStream() << msg ).str() ) +#define CATCH_INTERNAL_ERROR( msg ) \ + throw CATCH_PREPARE_EXCEPTION( std::logic_error, CATCH_INTERNAL_LINEINFO << ": Internal Catch error: " << msg); +#define CATCH_ERROR( msg ) \ + throw CATCH_PREPARE_EXCEPTION( std::domain_error, msg ) +#define CATCH_ENFORCE( condition, msg ) \ + do{ if( !(condition) ) CATCH_ERROR( msg ); } while(false) + +// end catch_enforce.h +namespace Catch { + + Config::Config( ConfigData const& data ) + : m_data( data ), + m_stream( openStream() ) + { + TestSpecParser parser(ITagAliasRegistry::get()); + if (data.testsOrTags.empty()) { + parser.parse("~[.]"); // All not hidden tests + } + else { + m_hasTestFilters = true; + for( auto const& testOrTags : data.testsOrTags ) + parser.parse( testOrTags ); + } + m_testSpec = parser.testSpec(); + } + + std::string const& Config::getFilename() const { + return m_data.outputFilename ; + } + + bool Config::listTests() const { return m_data.listTests; } + bool Config::listTestNamesOnly() const { return m_data.listTestNamesOnly; } + bool Config::listTags() const { return m_data.listTags; } + bool Config::listReporters() const { return m_data.listReporters; } + + std::string Config::getProcessName() const { return m_data.processName; } + std::string const& Config::getReporterName() const { return m_data.reporterName; } + + std::vector const& Config::getTestsOrTags() const { return m_data.testsOrTags; } + std::vector const& Config::getSectionsToRun() const { return m_data.sectionsToRun; } + + TestSpec const& Config::testSpec() const { return m_testSpec; } + bool Config::hasTestFilters() const { return m_hasTestFilters; } + + bool Config::showHelp() const { return m_data.showHelp; } + + // IConfig interface + bool Config::allowThrows() const { return !m_data.noThrow; } + std::ostream& Config::stream() const { return m_stream->stream(); } + std::string Config::name() const { return m_data.name.empty() ? m_data.processName : m_data.name; } + bool Config::includeSuccessfulResults() const { return m_data.showSuccessfulTests; } + bool Config::warnAboutMissingAssertions() const { return !!(m_data.warnings & WarnAbout::NoAssertions); } + bool Config::warnAboutNoTests() const { return !!(m_data.warnings & WarnAbout::NoTests); } + ShowDurations::OrNot Config::showDurations() const { return m_data.showDurations; } + RunTests::InWhatOrder Config::runOrder() const { return m_data.runOrder; } + unsigned int Config::rngSeed() const { return m_data.rngSeed; } + int Config::benchmarkResolutionMultiple() const { return m_data.benchmarkResolutionMultiple; } + UseColour::YesOrNo Config::useColour() const { return m_data.useColour; } + bool Config::shouldDebugBreak() const { return m_data.shouldDebugBreak; } + int Config::abortAfter() const { return m_data.abortAfter; } + bool Config::showInvisibles() const { return m_data.showInvisibles; } + Verbosity Config::verbosity() const { return m_data.verbosity; } + + IStream const* Config::openStream() { + return Catch::makeStream(m_data.outputFilename); + } + +} // end namespace Catch +// end catch_config.cpp +// start catch_console_colour.cpp + +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wexit-time-destructors" +#endif + +// start catch_errno_guard.h + +namespace Catch { + + class ErrnoGuard { + public: + ErrnoGuard(); + ~ErrnoGuard(); + private: + int m_oldErrno; + }; + +} + +// end catch_errno_guard.h +#include + +namespace Catch { + namespace { + + struct IColourImpl { + virtual ~IColourImpl() = default; + virtual void use( Colour::Code _colourCode ) = 0; + }; + + struct NoColourImpl : IColourImpl { + void use( Colour::Code ) {} + + static IColourImpl* instance() { + static NoColourImpl s_instance; + return &s_instance; + } + }; + + } // anon namespace +} // namespace Catch + +#if !defined( CATCH_CONFIG_COLOUR_NONE ) && !defined( CATCH_CONFIG_COLOUR_WINDOWS ) && !defined( CATCH_CONFIG_COLOUR_ANSI ) +# ifdef CATCH_PLATFORM_WINDOWS +# define CATCH_CONFIG_COLOUR_WINDOWS +# else +# define CATCH_CONFIG_COLOUR_ANSI +# endif +#endif + +#if defined ( CATCH_CONFIG_COLOUR_WINDOWS ) ///////////////////////////////////////// + +namespace Catch { +namespace { + + class Win32ColourImpl : public IColourImpl { + public: + Win32ColourImpl() : stdoutHandle( GetStdHandle(STD_OUTPUT_HANDLE) ) + { + CONSOLE_SCREEN_BUFFER_INFO csbiInfo; + GetConsoleScreenBufferInfo( stdoutHandle, &csbiInfo ); + originalForegroundAttributes = csbiInfo.wAttributes & ~( BACKGROUND_GREEN | BACKGROUND_RED | BACKGROUND_BLUE | BACKGROUND_INTENSITY ); + originalBackgroundAttributes = csbiInfo.wAttributes & ~( FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE | FOREGROUND_INTENSITY ); + } + + virtual void use( Colour::Code _colourCode ) override { + switch( _colourCode ) { + case Colour::None: return setTextAttribute( originalForegroundAttributes ); + case Colour::White: return setTextAttribute( FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE ); + case Colour::Red: return setTextAttribute( FOREGROUND_RED ); + case Colour::Green: return setTextAttribute( FOREGROUND_GREEN ); + case Colour::Blue: return setTextAttribute( FOREGROUND_BLUE ); + case Colour::Cyan: return setTextAttribute( FOREGROUND_BLUE | FOREGROUND_GREEN ); + case Colour::Yellow: return setTextAttribute( FOREGROUND_RED | FOREGROUND_GREEN ); + case Colour::Grey: return setTextAttribute( 0 ); + + case Colour::LightGrey: return setTextAttribute( FOREGROUND_INTENSITY ); + case Colour::BrightRed: return setTextAttribute( FOREGROUND_INTENSITY | FOREGROUND_RED ); + case Colour::BrightGreen: return setTextAttribute( FOREGROUND_INTENSITY | FOREGROUND_GREEN ); + case Colour::BrightWhite: return setTextAttribute( FOREGROUND_INTENSITY | FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE ); + case Colour::BrightYellow: return setTextAttribute( FOREGROUND_INTENSITY | FOREGROUND_RED | FOREGROUND_GREEN ); + + case Colour::Bright: CATCH_INTERNAL_ERROR( "not a colour" ); + + default: + CATCH_ERROR( "Unknown colour requested" ); + } + } + + private: + void setTextAttribute( WORD _textAttribute ) { + SetConsoleTextAttribute( stdoutHandle, _textAttribute | originalBackgroundAttributes ); + } + HANDLE stdoutHandle; + WORD originalForegroundAttributes; + WORD originalBackgroundAttributes; + }; + + IColourImpl* platformColourInstance() { + static Win32ColourImpl s_instance; + + IConfigPtr config = getCurrentContext().getConfig(); + UseColour::YesOrNo colourMode = config + ? config->useColour() + : UseColour::Auto; + if( colourMode == UseColour::Auto ) + colourMode = UseColour::Yes; + return colourMode == UseColour::Yes + ? &s_instance + : NoColourImpl::instance(); + } + +} // end anon namespace +} // end namespace Catch + +#elif defined( CATCH_CONFIG_COLOUR_ANSI ) ////////////////////////////////////// + +#include + +namespace Catch { +namespace { + + // use POSIX/ ANSI console terminal codes + // Thanks to Adam Strzelecki for original contribution + // (http://github.com/nanoant) + // https://github.com/philsquared/Catch/pull/131 + class PosixColourImpl : public IColourImpl { + public: + virtual void use( Colour::Code _colourCode ) override { + switch( _colourCode ) { + case Colour::None: + case Colour::White: return setColour( "[0m" ); + case Colour::Red: return setColour( "[0;31m" ); + case Colour::Green: return setColour( "[0;32m" ); + case Colour::Blue: return setColour( "[0;34m" ); + case Colour::Cyan: return setColour( "[0;36m" ); + case Colour::Yellow: return setColour( "[0;33m" ); + case Colour::Grey: return setColour( "[1;30m" ); + + case Colour::LightGrey: return setColour( "[0;37m" ); + case Colour::BrightRed: return setColour( "[1;31m" ); + case Colour::BrightGreen: return setColour( "[1;32m" ); + case Colour::BrightWhite: return setColour( "[1;37m" ); + case Colour::BrightYellow: return setColour( "[1;33m" ); + + case Colour::Bright: CATCH_INTERNAL_ERROR( "not a colour" ); + default: CATCH_INTERNAL_ERROR( "Unknown colour requested" ); + } + } + static IColourImpl* instance() { + static PosixColourImpl s_instance; + return &s_instance; + } + + private: + void setColour( const char* _escapeCode ) { + Catch::cout() << '\033' << _escapeCode; + } + }; + + bool useColourOnPlatform() { + return +#ifdef CATCH_PLATFORM_MAC + !isDebuggerActive() && +#endif +#if !(defined(__DJGPP__) && defined(__STRICT_ANSI__)) + isatty(STDOUT_FILENO) +#else + false +#endif + ; + } + IColourImpl* platformColourInstance() { + ErrnoGuard guard; + IConfigPtr config = getCurrentContext().getConfig(); + UseColour::YesOrNo colourMode = config + ? config->useColour() + : UseColour::Auto; + if( colourMode == UseColour::Auto ) + colourMode = useColourOnPlatform() + ? UseColour::Yes + : UseColour::No; + return colourMode == UseColour::Yes + ? PosixColourImpl::instance() + : NoColourImpl::instance(); + } + +} // end anon namespace +} // end namespace Catch + +#else // not Windows or ANSI /////////////////////////////////////////////// + +namespace Catch { + + static IColourImpl* platformColourInstance() { return NoColourImpl::instance(); } + +} // end namespace Catch + +#endif // Windows/ ANSI/ None + +namespace Catch { + + Colour::Colour( Code _colourCode ) { use( _colourCode ); } + Colour::Colour( Colour&& rhs ) noexcept { + m_moved = rhs.m_moved; + rhs.m_moved = true; + } + Colour& Colour::operator=( Colour&& rhs ) noexcept { + m_moved = rhs.m_moved; + rhs.m_moved = true; + return *this; + } + + Colour::~Colour(){ if( !m_moved ) use( None ); } + + void Colour::use( Code _colourCode ) { + static IColourImpl* impl = platformColourInstance(); + impl->use( _colourCode ); + } + + std::ostream& operator << ( std::ostream& os, Colour const& ) { + return os; + } + +} // end namespace Catch + +#if defined(__clang__) +# pragma clang diagnostic pop +#endif + +// end catch_console_colour.cpp +// start catch_context.cpp + +namespace Catch { + + class Context : public IMutableContext, NonCopyable { + + public: // IContext + virtual IResultCapture* getResultCapture() override { + return m_resultCapture; + } + virtual IRunner* getRunner() override { + return m_runner; + } + + virtual IConfigPtr const& getConfig() const override { + return m_config; + } + + virtual ~Context() override; + + public: // IMutableContext + virtual void setResultCapture( IResultCapture* resultCapture ) override { + m_resultCapture = resultCapture; + } + virtual void setRunner( IRunner* runner ) override { + m_runner = runner; + } + virtual void setConfig( IConfigPtr const& config ) override { + m_config = config; + } + + friend IMutableContext& getCurrentMutableContext(); + + private: + IConfigPtr m_config; + IRunner* m_runner = nullptr; + IResultCapture* m_resultCapture = nullptr; + }; + + IMutableContext *IMutableContext::currentContext = nullptr; + + void IMutableContext::createContext() + { + currentContext = new Context(); + } + + void cleanUpContext() { + delete IMutableContext::currentContext; + IMutableContext::currentContext = nullptr; + } + IContext::~IContext() = default; + IMutableContext::~IMutableContext() = default; + Context::~Context() = default; +} +// end catch_context.cpp +// start catch_debug_console.cpp + +// start catch_debug_console.h + +#include + +namespace Catch { + void writeToDebugConsole( std::string const& text ); +} + +// end catch_debug_console.h +#ifdef CATCH_PLATFORM_WINDOWS + + namespace Catch { + void writeToDebugConsole( std::string const& text ) { + ::OutputDebugStringA( text.c_str() ); + } + } + +#else + + namespace Catch { + void writeToDebugConsole( std::string const& text ) { + // !TBD: Need a version for Mac/ XCode and other IDEs + Catch::cout() << text; + } + } + +#endif // Platform +// end catch_debug_console.cpp +// start catch_debugger.cpp + +#ifdef CATCH_PLATFORM_MAC + +# include +# include +# include +# include +# include +# include +# include + +namespace Catch { + + // The following function is taken directly from the following technical note: + // http://developer.apple.com/library/mac/#qa/qa2004/qa1361.html + + // Returns true if the current process is being debugged (either + // running under the debugger or has a debugger attached post facto). + bool isDebuggerActive(){ + + int mib[4]; + struct kinfo_proc info; + std::size_t size; + + // Initialize the flags so that, if sysctl fails for some bizarre + // reason, we get a predictable result. + + info.kp_proc.p_flag = 0; + + // Initialize mib, which tells sysctl the info we want, in this case + // we're looking for information about a specific process ID. + + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_PID; + mib[3] = getpid(); + + // Call sysctl. + + size = sizeof(info); + if( sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &size, nullptr, 0) != 0 ) { + Catch::cerr() << "\n** Call to sysctl failed - unable to determine if debugger is active **\n" << std::endl; + return false; + } + + // We're being debugged if the P_TRACED flag is set. + + return ( (info.kp_proc.p_flag & P_TRACED) != 0 ); + } + } // namespace Catch + +#elif defined(CATCH_PLATFORM_LINUX) + #include + #include + + namespace Catch{ + // The standard POSIX way of detecting a debugger is to attempt to + // ptrace() the process, but this needs to be done from a child and not + // this process itself to still allow attaching to this process later + // if wanted, so is rather heavy. Under Linux we have the PID of the + // "debugger" (which doesn't need to be gdb, of course, it could also + // be strace, for example) in /proc/$PID/status, so just get it from + // there instead. + bool isDebuggerActive(){ + // Libstdc++ has a bug, where std::ifstream sets errno to 0 + // This way our users can properly assert over errno values + ErrnoGuard guard; + std::ifstream in("/proc/self/status"); + for( std::string line; std::getline(in, line); ) { + static const int PREFIX_LEN = 11; + if( line.compare(0, PREFIX_LEN, "TracerPid:\t") == 0 ) { + // We're traced if the PID is not 0 and no other PID starts + // with 0 digit, so it's enough to check for just a single + // character. + return line.length() > PREFIX_LEN && line[PREFIX_LEN] != '0'; + } + } + + return false; + } + } // namespace Catch +#elif defined(_MSC_VER) + extern "C" __declspec(dllimport) int __stdcall IsDebuggerPresent(); + namespace Catch { + bool isDebuggerActive() { + return IsDebuggerPresent() != 0; + } + } +#elif defined(__MINGW32__) + extern "C" __declspec(dllimport) int __stdcall IsDebuggerPresent(); + namespace Catch { + bool isDebuggerActive() { + return IsDebuggerPresent() != 0; + } + } +#else + namespace Catch { + bool isDebuggerActive() { return false; } + } +#endif // Platform +// end catch_debugger.cpp +// start catch_decomposer.cpp + +namespace Catch { + + ITransientExpression::~ITransientExpression() = default; + + void formatReconstructedExpression( std::ostream &os, std::string const& lhs, StringRef op, std::string const& rhs ) { + if( lhs.size() + rhs.size() < 40 && + lhs.find('\n') == std::string::npos && + rhs.find('\n') == std::string::npos ) + os << lhs << " " << op << " " << rhs; + else + os << lhs << "\n" << op << "\n" << rhs; + } +} +// end catch_decomposer.cpp +// start catch_errno_guard.cpp + +#include + +namespace Catch { + ErrnoGuard::ErrnoGuard():m_oldErrno(errno){} + ErrnoGuard::~ErrnoGuard() { errno = m_oldErrno; } +} +// end catch_errno_guard.cpp +// start catch_exception_translator_registry.cpp + +// start catch_exception_translator_registry.h + +#include +#include +#include + +namespace Catch { + + class ExceptionTranslatorRegistry : public IExceptionTranslatorRegistry { + public: + ~ExceptionTranslatorRegistry(); + virtual void registerTranslator( const IExceptionTranslator* translator ); + virtual std::string translateActiveException() const override; + std::string tryTranslators() const; + + private: + std::vector> m_translators; + }; +} + +// end catch_exception_translator_registry.h +#ifdef __OBJC__ +#import "Foundation/Foundation.h" +#endif + +namespace Catch { + + ExceptionTranslatorRegistry::~ExceptionTranslatorRegistry() { + } + + void ExceptionTranslatorRegistry::registerTranslator( const IExceptionTranslator* translator ) { + m_translators.push_back( std::unique_ptr( translator ) ); + } + + std::string ExceptionTranslatorRegistry::translateActiveException() const { + try { +#ifdef __OBJC__ + // In Objective-C try objective-c exceptions first + @try { + return tryTranslators(); + } + @catch (NSException *exception) { + return Catch::Detail::stringify( [exception description] ); + } +#else + // Compiling a mixed mode project with MSVC means that CLR + // exceptions will be caught in (...) as well. However, these + // do not fill-in std::current_exception and thus lead to crash + // when attempting rethrow. + // /EHa switch also causes structured exceptions to be caught + // here, but they fill-in current_exception properly, so + // at worst the output should be a little weird, instead of + // causing a crash. + if (std::current_exception() == nullptr) { + return "Non C++ exception. Possibly a CLR exception."; + } + return tryTranslators(); +#endif + } + catch( TestFailureException& ) { + std::rethrow_exception(std::current_exception()); + } + catch( std::exception& ex ) { + return ex.what(); + } + catch( std::string& msg ) { + return msg; + } + catch( const char* msg ) { + return msg; + } + catch(...) { + return "Unknown exception"; + } + } + + std::string ExceptionTranslatorRegistry::tryTranslators() const { + if( m_translators.empty() ) + std::rethrow_exception(std::current_exception()); + else + return m_translators[0]->translate( m_translators.begin()+1, m_translators.end() ); + } +} +// end catch_exception_translator_registry.cpp +// start catch_fatal_condition.cpp + +#if defined(__GNUC__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wmissing-field-initializers" +#endif + +#if defined( CATCH_CONFIG_WINDOWS_SEH ) || defined( CATCH_CONFIG_POSIX_SIGNALS ) + +namespace { + // Report the error condition + void reportFatal( char const * const message ) { + Catch::getCurrentContext().getResultCapture()->handleFatalErrorCondition( message ); + } +} + +#endif // signals/SEH handling + +#if defined( CATCH_CONFIG_WINDOWS_SEH ) + +namespace Catch { + struct SignalDefs { DWORD id; const char* name; }; + + // There is no 1-1 mapping between signals and windows exceptions. + // Windows can easily distinguish between SO and SigSegV, + // but SigInt, SigTerm, etc are handled differently. + static SignalDefs signalDefs[] = { + { EXCEPTION_ILLEGAL_INSTRUCTION, "SIGILL - Illegal instruction signal" }, + { EXCEPTION_STACK_OVERFLOW, "SIGSEGV - Stack overflow" }, + { EXCEPTION_ACCESS_VIOLATION, "SIGSEGV - Segmentation violation signal" }, + { EXCEPTION_INT_DIVIDE_BY_ZERO, "Divide by zero error" }, + }; + + LONG CALLBACK FatalConditionHandler::handleVectoredException(PEXCEPTION_POINTERS ExceptionInfo) { + for (auto const& def : signalDefs) { + if (ExceptionInfo->ExceptionRecord->ExceptionCode == def.id) { + reportFatal(def.name); + } + } + // If its not an exception we care about, pass it along. + // This stops us from eating debugger breaks etc. + return EXCEPTION_CONTINUE_SEARCH; + } + + FatalConditionHandler::FatalConditionHandler() { + isSet = true; + // 32k seems enough for Catch to handle stack overflow, + // but the value was found experimentally, so there is no strong guarantee + guaranteeSize = 32 * 1024; + exceptionHandlerHandle = nullptr; + // Register as first handler in current chain + exceptionHandlerHandle = AddVectoredExceptionHandler(1, handleVectoredException); + // Pass in guarantee size to be filled + SetThreadStackGuarantee(&guaranteeSize); + } + + void FatalConditionHandler::reset() { + if (isSet) { + RemoveVectoredExceptionHandler(exceptionHandlerHandle); + SetThreadStackGuarantee(&guaranteeSize); + exceptionHandlerHandle = nullptr; + isSet = false; + } + } + + FatalConditionHandler::~FatalConditionHandler() { + reset(); + } + +bool FatalConditionHandler::isSet = false; +ULONG FatalConditionHandler::guaranteeSize = 0; +PVOID FatalConditionHandler::exceptionHandlerHandle = nullptr; + +} // namespace Catch + +#elif defined( CATCH_CONFIG_POSIX_SIGNALS ) + +namespace Catch { + + struct SignalDefs { + int id; + const char* name; + }; + + // 32kb for the alternate stack seems to be sufficient. However, this value + // is experimentally determined, so that's not guaranteed. + constexpr static std::size_t sigStackSize = 32768 >= MINSIGSTKSZ ? 32768 : MINSIGSTKSZ; + + static SignalDefs signalDefs[] = { + { SIGINT, "SIGINT - Terminal interrupt signal" }, + { SIGILL, "SIGILL - Illegal instruction signal" }, + { SIGFPE, "SIGFPE - Floating point error signal" }, + { SIGSEGV, "SIGSEGV - Segmentation violation signal" }, + { SIGTERM, "SIGTERM - Termination request signal" }, + { SIGABRT, "SIGABRT - Abort (abnormal termination) signal" } + }; + + void FatalConditionHandler::handleSignal( int sig ) { + char const * name = ""; + for (auto const& def : signalDefs) { + if (sig == def.id) { + name = def.name; + break; + } + } + reset(); + reportFatal(name); + raise( sig ); + } + + FatalConditionHandler::FatalConditionHandler() { + isSet = true; + stack_t sigStack; + sigStack.ss_sp = altStackMem; + sigStack.ss_size = sigStackSize; + sigStack.ss_flags = 0; + sigaltstack(&sigStack, &oldSigStack); + struct sigaction sa = { }; + + sa.sa_handler = handleSignal; + sa.sa_flags = SA_ONSTACK; + for (std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i) { + sigaction(signalDefs[i].id, &sa, &oldSigActions[i]); + } + } + + FatalConditionHandler::~FatalConditionHandler() { + reset(); + } + + void FatalConditionHandler::reset() { + if( isSet ) { + // Set signals back to previous values -- hopefully nobody overwrote them in the meantime + for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) { + sigaction(signalDefs[i].id, &oldSigActions[i], nullptr); + } + // Return the old stack + sigaltstack(&oldSigStack, nullptr); + isSet = false; + } + } + + bool FatalConditionHandler::isSet = false; + struct sigaction FatalConditionHandler::oldSigActions[sizeof(signalDefs)/sizeof(SignalDefs)] = {}; + stack_t FatalConditionHandler::oldSigStack = {}; + char FatalConditionHandler::altStackMem[sigStackSize] = {}; + +} // namespace Catch + +#else + +namespace Catch { + void FatalConditionHandler::reset() {} +} + +#endif // signals/SEH handling + +#if defined(__GNUC__) +# pragma GCC diagnostic pop +#endif +// end catch_fatal_condition.cpp +// start catch_interfaces_capture.cpp + +namespace Catch { + IResultCapture::~IResultCapture() = default; +} +// end catch_interfaces_capture.cpp +// start catch_interfaces_config.cpp + +namespace Catch { + IConfig::~IConfig() = default; +} +// end catch_interfaces_config.cpp +// start catch_interfaces_exception.cpp + +namespace Catch { + IExceptionTranslator::~IExceptionTranslator() = default; + IExceptionTranslatorRegistry::~IExceptionTranslatorRegistry() = default; +} +// end catch_interfaces_exception.cpp +// start catch_interfaces_registry_hub.cpp + +namespace Catch { + IRegistryHub::~IRegistryHub() = default; + IMutableRegistryHub::~IMutableRegistryHub() = default; +} +// end catch_interfaces_registry_hub.cpp +// start catch_interfaces_reporter.cpp + +// start catch_reporter_listening.h + +namespace Catch { + + class ListeningReporter : public IStreamingReporter { + using Reporters = std::vector; + Reporters m_listeners; + IStreamingReporterPtr m_reporter = nullptr; + ReporterPreferences m_preferences; + + public: + ListeningReporter(); + + void addListener( IStreamingReporterPtr&& listener ); + void addReporter( IStreamingReporterPtr&& reporter ); + + public: // IStreamingReporter + + ReporterPreferences getPreferences() const override; + + void noMatchingTestCases( std::string const& spec ) override; + + static std::set getSupportedVerbosities(); + + void benchmarkStarting( BenchmarkInfo const& benchmarkInfo ) override; + void benchmarkEnded( BenchmarkStats const& benchmarkStats ) override; + + void testRunStarting( TestRunInfo const& testRunInfo ) override; + void testGroupStarting( GroupInfo const& groupInfo ) override; + void testCaseStarting( TestCaseInfo const& testInfo ) override; + void sectionStarting( SectionInfo const& sectionInfo ) override; + void assertionStarting( AssertionInfo const& assertionInfo ) override; + + // The return value indicates if the messages buffer should be cleared: + bool assertionEnded( AssertionStats const& assertionStats ) override; + void sectionEnded( SectionStats const& sectionStats ) override; + void testCaseEnded( TestCaseStats const& testCaseStats ) override; + void testGroupEnded( TestGroupStats const& testGroupStats ) override; + void testRunEnded( TestRunStats const& testRunStats ) override; + + void skipTest( TestCaseInfo const& testInfo ) override; + bool isMulti() const override; + + }; + +} // end namespace Catch + +// end catch_reporter_listening.h +namespace Catch { + + ReporterConfig::ReporterConfig( IConfigPtr const& _fullConfig ) + : m_stream( &_fullConfig->stream() ), m_fullConfig( _fullConfig ) {} + + ReporterConfig::ReporterConfig( IConfigPtr const& _fullConfig, std::ostream& _stream ) + : m_stream( &_stream ), m_fullConfig( _fullConfig ) {} + + std::ostream& ReporterConfig::stream() const { return *m_stream; } + IConfigPtr ReporterConfig::fullConfig() const { return m_fullConfig; } + + TestRunInfo::TestRunInfo( std::string const& _name ) : name( _name ) {} + + GroupInfo::GroupInfo( std::string const& _name, + std::size_t _groupIndex, + std::size_t _groupsCount ) + : name( _name ), + groupIndex( _groupIndex ), + groupsCounts( _groupsCount ) + {} + + AssertionStats::AssertionStats( AssertionResult const& _assertionResult, + std::vector const& _infoMessages, + Totals const& _totals ) + : assertionResult( _assertionResult ), + infoMessages( _infoMessages ), + totals( _totals ) + { + assertionResult.m_resultData.lazyExpression.m_transientExpression = _assertionResult.m_resultData.lazyExpression.m_transientExpression; + + if( assertionResult.hasMessage() ) { + // Copy message into messages list. + // !TBD This should have been done earlier, somewhere + MessageBuilder builder( assertionResult.getTestMacroName(), assertionResult.getSourceInfo(), assertionResult.getResultType() ); + builder << assertionResult.getMessage(); + builder.m_info.message = builder.m_stream.str(); + + infoMessages.push_back( builder.m_info ); + } + } + + AssertionStats::~AssertionStats() = default; + + SectionStats::SectionStats( SectionInfo const& _sectionInfo, + Counts const& _assertions, + double _durationInSeconds, + bool _missingAssertions ) + : sectionInfo( _sectionInfo ), + assertions( _assertions ), + durationInSeconds( _durationInSeconds ), + missingAssertions( _missingAssertions ) + {} + + SectionStats::~SectionStats() = default; + + TestCaseStats::TestCaseStats( TestCaseInfo const& _testInfo, + Totals const& _totals, + std::string const& _stdOut, + std::string const& _stdErr, + bool _aborting ) + : testInfo( _testInfo ), + totals( _totals ), + stdOut( _stdOut ), + stdErr( _stdErr ), + aborting( _aborting ) + {} + + TestCaseStats::~TestCaseStats() = default; + + TestGroupStats::TestGroupStats( GroupInfo const& _groupInfo, + Totals const& _totals, + bool _aborting ) + : groupInfo( _groupInfo ), + totals( _totals ), + aborting( _aborting ) + {} + + TestGroupStats::TestGroupStats( GroupInfo const& _groupInfo ) + : groupInfo( _groupInfo ), + aborting( false ) + {} + + TestGroupStats::~TestGroupStats() = default; + + TestRunStats::TestRunStats( TestRunInfo const& _runInfo, + Totals const& _totals, + bool _aborting ) + : runInfo( _runInfo ), + totals( _totals ), + aborting( _aborting ) + {} + + TestRunStats::~TestRunStats() = default; + + void IStreamingReporter::fatalErrorEncountered( StringRef ) {} + bool IStreamingReporter::isMulti() const { return false; } + + IReporterFactory::~IReporterFactory() = default; + IReporterRegistry::~IReporterRegistry() = default; + +} // end namespace Catch +// end catch_interfaces_reporter.cpp +// start catch_interfaces_runner.cpp + +namespace Catch { + IRunner::~IRunner() = default; +} +// end catch_interfaces_runner.cpp +// start catch_interfaces_testcase.cpp + +namespace Catch { + ITestInvoker::~ITestInvoker() = default; + ITestCaseRegistry::~ITestCaseRegistry() = default; +} +// end catch_interfaces_testcase.cpp +// start catch_leak_detector.cpp + +#ifdef CATCH_CONFIG_WINDOWS_CRTDBG +#include + +namespace Catch { + + LeakDetector::LeakDetector() { + int flag = _CrtSetDbgFlag(_CRTDBG_REPORT_FLAG); + flag |= _CRTDBG_LEAK_CHECK_DF; + flag |= _CRTDBG_ALLOC_MEM_DF; + _CrtSetDbgFlag(flag); + _CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG); + _CrtSetReportFile(_CRT_WARN, _CRTDBG_FILE_STDERR); + // Change this to leaking allocation's number to break there + _CrtSetBreakAlloc(-1); + } +} + +#else + + Catch::LeakDetector::LeakDetector() {} + +#endif +// end catch_leak_detector.cpp +// start catch_list.cpp + +// start catch_list.h + +#include + +namespace Catch { + + std::size_t listTests( Config const& config ); + + std::size_t listTestsNamesOnly( Config const& config ); + + struct TagInfo { + void add( std::string const& spelling ); + std::string all() const; + + std::set spellings; + std::size_t count = 0; + }; + + std::size_t listTags( Config const& config ); + + std::size_t listReporters( Config const& /*config*/ ); + + Option list( Config const& config ); + +} // end namespace Catch + +// end catch_list.h +// start catch_text.h + +namespace Catch { + using namespace clara::TextFlow; +} + +// end catch_text.h +#include +#include +#include + +namespace Catch { + + std::size_t listTests( Config const& config ) { + TestSpec testSpec = config.testSpec(); + if( config.hasTestFilters() ) + Catch::cout() << "Matching test cases:\n"; + else { + Catch::cout() << "All available test cases:\n"; + } + + auto matchedTestCases = filterTests( getAllTestCasesSorted( config ), testSpec, config ); + for( auto const& testCaseInfo : matchedTestCases ) { + Colour::Code colour = testCaseInfo.isHidden() + ? Colour::SecondaryText + : Colour::None; + Colour colourGuard( colour ); + + Catch::cout() << Column( testCaseInfo.name ).initialIndent( 2 ).indent( 4 ) << "\n"; + if( config.verbosity() >= Verbosity::High ) { + Catch::cout() << Column( Catch::Detail::stringify( testCaseInfo.lineInfo ) ).indent(4) << std::endl; + std::string description = testCaseInfo.description; + if( description.empty() ) + description = "(NO DESCRIPTION)"; + Catch::cout() << Column( description ).indent(4) << std::endl; + } + if( !testCaseInfo.tags.empty() ) + Catch::cout() << Column( testCaseInfo.tagsAsString() ).indent( 6 ) << "\n"; + } + + if( !config.hasTestFilters() ) + Catch::cout() << pluralise( matchedTestCases.size(), "test case" ) << '\n' << std::endl; + else + Catch::cout() << pluralise( matchedTestCases.size(), "matching test case" ) << '\n' << std::endl; + return matchedTestCases.size(); + } + + std::size_t listTestsNamesOnly( Config const& config ) { + TestSpec testSpec = config.testSpec(); + std::size_t matchedTests = 0; + std::vector matchedTestCases = filterTests( getAllTestCasesSorted( config ), testSpec, config ); + for( auto const& testCaseInfo : matchedTestCases ) { + matchedTests++; + if( startsWith( testCaseInfo.name, '#' ) ) + Catch::cout() << '"' << testCaseInfo.name << '"'; + else + Catch::cout() << testCaseInfo.name; + if ( config.verbosity() >= Verbosity::High ) + Catch::cout() << "\t@" << testCaseInfo.lineInfo; + Catch::cout() << std::endl; + } + return matchedTests; + } + + void TagInfo::add( std::string const& spelling ) { + ++count; + spellings.insert( spelling ); + } + + std::string TagInfo::all() const { + std::string out; + for( auto const& spelling : spellings ) + out += "[" + spelling + "]"; + return out; + } + + std::size_t listTags( Config const& config ) { + TestSpec testSpec = config.testSpec(); + if( config.hasTestFilters() ) + Catch::cout() << "Tags for matching test cases:\n"; + else { + Catch::cout() << "All available tags:\n"; + } + + std::map tagCounts; + + std::vector matchedTestCases = filterTests( getAllTestCasesSorted( config ), testSpec, config ); + for( auto const& testCase : matchedTestCases ) { + for( auto const& tagName : testCase.getTestCaseInfo().tags ) { + std::string lcaseTagName = toLower( tagName ); + auto countIt = tagCounts.find( lcaseTagName ); + if( countIt == tagCounts.end() ) + countIt = tagCounts.insert( std::make_pair( lcaseTagName, TagInfo() ) ).first; + countIt->second.add( tagName ); + } + } + + for( auto const& tagCount : tagCounts ) { + ReusableStringStream rss; + rss << " " << std::setw(2) << tagCount.second.count << " "; + auto str = rss.str(); + auto wrapper = Column( tagCount.second.all() ) + .initialIndent( 0 ) + .indent( str.size() ) + .width( CATCH_CONFIG_CONSOLE_WIDTH-10 ); + Catch::cout() << str << wrapper << '\n'; + } + Catch::cout() << pluralise( tagCounts.size(), "tag" ) << '\n' << std::endl; + return tagCounts.size(); + } + + std::size_t listReporters( Config const& /*config*/ ) { + Catch::cout() << "Available reporters:\n"; + IReporterRegistry::FactoryMap const& factories = getRegistryHub().getReporterRegistry().getFactories(); + std::size_t maxNameLen = 0; + for( auto const& factoryKvp : factories ) + maxNameLen = (std::max)( maxNameLen, factoryKvp.first.size() ); + + for( auto const& factoryKvp : factories ) { + Catch::cout() + << Column( factoryKvp.first + ":" ) + .indent(2) + .width( 5+maxNameLen ) + + Column( factoryKvp.second->getDescription() ) + .initialIndent(0) + .indent(2) + .width( CATCH_CONFIG_CONSOLE_WIDTH - maxNameLen-8 ) + << "\n"; + } + Catch::cout() << std::endl; + return factories.size(); + } + + Option list( Config const& config ) { + Option listedCount; + if( config.listTests() ) + listedCount = listedCount.valueOr(0) + listTests( config ); + if( config.listTestNamesOnly() ) + listedCount = listedCount.valueOr(0) + listTestsNamesOnly( config ); + if( config.listTags() ) + listedCount = listedCount.valueOr(0) + listTags( config ); + if( config.listReporters() ) + listedCount = listedCount.valueOr(0) + listReporters( config ); + return listedCount; + } + +} // end namespace Catch +// end catch_list.cpp +// start catch_matchers.cpp + +namespace Catch { +namespace Matchers { + namespace Impl { + + std::string MatcherUntypedBase::toString() const { + if( m_cachedToString.empty() ) + m_cachedToString = describe(); + return m_cachedToString; + } + + MatcherUntypedBase::~MatcherUntypedBase() = default; + + } // namespace Impl +} // namespace Matchers + +using namespace Matchers; +using Matchers::Impl::MatcherBase; + +} // namespace Catch +// end catch_matchers.cpp +// start catch_matchers_floating.cpp + +// start catch_to_string.hpp + +#include + +namespace Catch { + template + std::string to_string(T const& t) { +#if defined(CATCH_CONFIG_CPP11_TO_STRING) + return std::to_string(t); +#else + ReusableStringStream rss; + rss << t; + return rss.str(); +#endif + } +} // end namespace Catch + +// end catch_to_string.hpp +#include +#include +#include +#include + +namespace Catch { +namespace Matchers { +namespace Floating { +enum class FloatingPointKind : uint8_t { + Float, + Double +}; +} +} +} + +namespace { + +template +struct Converter; + +template <> +struct Converter { + static_assert(sizeof(float) == sizeof(int32_t), "Important ULP matcher assumption violated"); + Converter(float f) { + std::memcpy(&i, &f, sizeof(f)); + } + int32_t i; +}; + +template <> +struct Converter { + static_assert(sizeof(double) == sizeof(int64_t), "Important ULP matcher assumption violated"); + Converter(double d) { + std::memcpy(&i, &d, sizeof(d)); + } + int64_t i; +}; + +template +auto convert(T t) -> Converter { + return Converter(t); +} + +template +bool almostEqualUlps(FP lhs, FP rhs, int maxUlpDiff) { + // Comparison with NaN should always be false. + // This way we can rule it out before getting into the ugly details + if (std::isnan(lhs) || std::isnan(rhs)) { + return false; + } + + auto lc = convert(lhs); + auto rc = convert(rhs); + + if ((lc.i < 0) != (rc.i < 0)) { + // Potentially we can have +0 and -0 + return lhs == rhs; + } + + auto ulpDiff = std::abs(lc.i - rc.i); + return ulpDiff <= maxUlpDiff; +} + +} + +namespace Catch { +namespace Matchers { +namespace Floating { + WithinAbsMatcher::WithinAbsMatcher(double target, double margin) + :m_target{ target }, m_margin{ margin } { + if (m_margin < 0) { + throw std::domain_error("Allowed margin difference has to be >= 0"); + } + } + + // Performs equivalent check of std::fabs(lhs - rhs) <= margin + // But without the subtraction to allow for INFINITY in comparison + bool WithinAbsMatcher::match(double const& matchee) const { + return (matchee + m_margin >= m_target) && (m_target + m_margin >= matchee); + } + + std::string WithinAbsMatcher::describe() const { + return "is within " + ::Catch::Detail::stringify(m_margin) + " of " + ::Catch::Detail::stringify(m_target); + } + + WithinUlpsMatcher::WithinUlpsMatcher(double target, int ulps, FloatingPointKind baseType) + :m_target{ target }, m_ulps{ ulps }, m_type{ baseType } { + if (m_ulps < 0) { + throw std::domain_error("Allowed ulp difference has to be >= 0"); + } + } + + bool WithinUlpsMatcher::match(double const& matchee) const { + switch (m_type) { + case FloatingPointKind::Float: + return almostEqualUlps(static_cast(matchee), static_cast(m_target), m_ulps); + case FloatingPointKind::Double: + return almostEqualUlps(matchee, m_target, m_ulps); + default: + throw std::domain_error("Unknown FloatingPointKind value"); + } + } + + std::string WithinUlpsMatcher::describe() const { + return "is within " + Catch::to_string(m_ulps) + " ULPs of " + ::Catch::Detail::stringify(m_target) + ((m_type == FloatingPointKind::Float)? "f" : ""); + } + +}// namespace Floating + +Floating::WithinUlpsMatcher WithinULP(double target, int maxUlpDiff) { + return Floating::WithinUlpsMatcher(target, maxUlpDiff, Floating::FloatingPointKind::Double); +} + +Floating::WithinUlpsMatcher WithinULP(float target, int maxUlpDiff) { + return Floating::WithinUlpsMatcher(target, maxUlpDiff, Floating::FloatingPointKind::Float); +} + +Floating::WithinAbsMatcher WithinAbs(double target, double margin) { + return Floating::WithinAbsMatcher(target, margin); +} + +} // namespace Matchers +} // namespace Catch + +// end catch_matchers_floating.cpp +// start catch_matchers_generic.cpp + +std::string Catch::Matchers::Generic::Detail::finalizeDescription(const std::string& desc) { + if (desc.empty()) { + return "matches undescribed predicate"; + } else { + return "matches predicate: \"" + desc + '"'; + } +} +// end catch_matchers_generic.cpp +// start catch_matchers_string.cpp + +#include + +namespace Catch { +namespace Matchers { + + namespace StdString { + + CasedString::CasedString( std::string const& str, CaseSensitive::Choice caseSensitivity ) + : m_caseSensitivity( caseSensitivity ), + m_str( adjustString( str ) ) + {} + std::string CasedString::adjustString( std::string const& str ) const { + return m_caseSensitivity == CaseSensitive::No + ? toLower( str ) + : str; + } + std::string CasedString::caseSensitivitySuffix() const { + return m_caseSensitivity == CaseSensitive::No + ? " (case insensitive)" + : std::string(); + } + + StringMatcherBase::StringMatcherBase( std::string const& operation, CasedString const& comparator ) + : m_comparator( comparator ), + m_operation( operation ) { + } + + std::string StringMatcherBase::describe() const { + std::string description; + description.reserve(5 + m_operation.size() + m_comparator.m_str.size() + + m_comparator.caseSensitivitySuffix().size()); + description += m_operation; + description += ": \""; + description += m_comparator.m_str; + description += "\""; + description += m_comparator.caseSensitivitySuffix(); + return description; + } + + EqualsMatcher::EqualsMatcher( CasedString const& comparator ) : StringMatcherBase( "equals", comparator ) {} + + bool EqualsMatcher::match( std::string const& source ) const { + return m_comparator.adjustString( source ) == m_comparator.m_str; + } + + ContainsMatcher::ContainsMatcher( CasedString const& comparator ) : StringMatcherBase( "contains", comparator ) {} + + bool ContainsMatcher::match( std::string const& source ) const { + return contains( m_comparator.adjustString( source ), m_comparator.m_str ); + } + + StartsWithMatcher::StartsWithMatcher( CasedString const& comparator ) : StringMatcherBase( "starts with", comparator ) {} + + bool StartsWithMatcher::match( std::string const& source ) const { + return startsWith( m_comparator.adjustString( source ), m_comparator.m_str ); + } + + EndsWithMatcher::EndsWithMatcher( CasedString const& comparator ) : StringMatcherBase( "ends with", comparator ) {} + + bool EndsWithMatcher::match( std::string const& source ) const { + return endsWith( m_comparator.adjustString( source ), m_comparator.m_str ); + } + + RegexMatcher::RegexMatcher(std::string regex, CaseSensitive::Choice caseSensitivity): m_regex(std::move(regex)), m_caseSensitivity(caseSensitivity) {} + + bool RegexMatcher::match(std::string const& matchee) const { + auto flags = std::regex::ECMAScript; // ECMAScript is the default syntax option anyway + if (m_caseSensitivity == CaseSensitive::Choice::No) { + flags |= std::regex::icase; + } + auto reg = std::regex(m_regex, flags); + return std::regex_match(matchee, reg); + } + + std::string RegexMatcher::describe() const { + return "matches " + ::Catch::Detail::stringify(m_regex) + ((m_caseSensitivity == CaseSensitive::Choice::Yes)? " case sensitively" : " case insensitively"); + } + + } // namespace StdString + + StdString::EqualsMatcher Equals( std::string const& str, CaseSensitive::Choice caseSensitivity ) { + return StdString::EqualsMatcher( StdString::CasedString( str, caseSensitivity) ); + } + StdString::ContainsMatcher Contains( std::string const& str, CaseSensitive::Choice caseSensitivity ) { + return StdString::ContainsMatcher( StdString::CasedString( str, caseSensitivity) ); + } + StdString::EndsWithMatcher EndsWith( std::string const& str, CaseSensitive::Choice caseSensitivity ) { + return StdString::EndsWithMatcher( StdString::CasedString( str, caseSensitivity) ); + } + StdString::StartsWithMatcher StartsWith( std::string const& str, CaseSensitive::Choice caseSensitivity ) { + return StdString::StartsWithMatcher( StdString::CasedString( str, caseSensitivity) ); + } + + StdString::RegexMatcher Matches(std::string const& regex, CaseSensitive::Choice caseSensitivity) { + return StdString::RegexMatcher(regex, caseSensitivity); + } + +} // namespace Matchers +} // namespace Catch +// end catch_matchers_string.cpp +// start catch_message.cpp + +// start catch_uncaught_exceptions.h + +namespace Catch { + bool uncaught_exceptions(); +} // end namespace Catch + +// end catch_uncaught_exceptions.h +namespace Catch { + + MessageInfo::MessageInfo( std::string const& _macroName, + SourceLineInfo const& _lineInfo, + ResultWas::OfType _type ) + : macroName( _macroName ), + lineInfo( _lineInfo ), + type( _type ), + sequence( ++globalCount ) + {} + + bool MessageInfo::operator==( MessageInfo const& other ) const { + return sequence == other.sequence; + } + + bool MessageInfo::operator<( MessageInfo const& other ) const { + return sequence < other.sequence; + } + + // This may need protecting if threading support is added + unsigned int MessageInfo::globalCount = 0; + + //////////////////////////////////////////////////////////////////////////// + + Catch::MessageBuilder::MessageBuilder( std::string const& macroName, + SourceLineInfo const& lineInfo, + ResultWas::OfType type ) + :m_info(macroName, lineInfo, type) {} + + //////////////////////////////////////////////////////////////////////////// + + ScopedMessage::ScopedMessage( MessageBuilder const& builder ) + : m_info( builder.m_info ) + { + m_info.message = builder.m_stream.str(); + getResultCapture().pushScopedMessage( m_info ); + } + + ScopedMessage::~ScopedMessage() { + if ( !uncaught_exceptions() ){ + getResultCapture().popScopedMessage(m_info); + } + } +} // end namespace Catch +// end catch_message.cpp +// start catch_output_redirect.cpp + +// start catch_output_redirect.h +#ifndef TWOBLUECUBES_CATCH_OUTPUT_REDIRECT_H +#define TWOBLUECUBES_CATCH_OUTPUT_REDIRECT_H + +#include +#include +#include + +namespace Catch { + + class RedirectedStream { + std::ostream& m_originalStream; + std::ostream& m_redirectionStream; + std::streambuf* m_prevBuf; + + public: + RedirectedStream( std::ostream& originalStream, std::ostream& redirectionStream ); + ~RedirectedStream(); + }; + + class RedirectedStdOut { + ReusableStringStream m_rss; + RedirectedStream m_cout; + public: + RedirectedStdOut(); + auto str() const -> std::string; + }; + + // StdErr has two constituent streams in C++, std::cerr and std::clog + // This means that we need to redirect 2 streams into 1 to keep proper + // order of writes + class RedirectedStdErr { + ReusableStringStream m_rss; + RedirectedStream m_cerr; + RedirectedStream m_clog; + public: + RedirectedStdErr(); + auto str() const -> std::string; + }; + +#if defined(CATCH_CONFIG_NEW_CAPTURE) + + // Windows's implementation of std::tmpfile is terrible (it tries + // to create a file inside system folder, thus requiring elevated + // privileges for the binary), so we have to use tmpnam(_s) and + // create the file ourselves there. + class TempFile { + public: + TempFile(TempFile const&) = delete; + TempFile& operator=(TempFile const&) = delete; + TempFile(TempFile&&) = delete; + TempFile& operator=(TempFile&&) = delete; + + TempFile(); + ~TempFile(); + + std::FILE* getFile(); + std::string getContents(); + + private: + std::FILE* m_file = nullptr; + #if defined(_MSC_VER) + char m_buffer[L_tmpnam] = { 0 }; + #endif + }; + + class OutputRedirect { + public: + OutputRedirect(OutputRedirect const&) = delete; + OutputRedirect& operator=(OutputRedirect const&) = delete; + OutputRedirect(OutputRedirect&&) = delete; + OutputRedirect& operator=(OutputRedirect&&) = delete; + + OutputRedirect(std::string& stdout_dest, std::string& stderr_dest); + ~OutputRedirect(); + + private: + int m_originalStdout = -1; + int m_originalStderr = -1; + TempFile m_stdoutFile; + TempFile m_stderrFile; + std::string& m_stdoutDest; + std::string& m_stderrDest; + }; + +#endif + +} // end namespace Catch + +#endif // TWOBLUECUBES_CATCH_OUTPUT_REDIRECT_H +// end catch_output_redirect.h +#include +#include +#include +#include +#include + +#if defined(CATCH_CONFIG_NEW_CAPTURE) + #if defined(_MSC_VER) + #include //_dup and _dup2 + #define dup _dup + #define dup2 _dup2 + #define fileno _fileno + #else + #include // dup and dup2 + #endif +#endif + +namespace Catch { + + RedirectedStream::RedirectedStream( std::ostream& originalStream, std::ostream& redirectionStream ) + : m_originalStream( originalStream ), + m_redirectionStream( redirectionStream ), + m_prevBuf( m_originalStream.rdbuf() ) + { + m_originalStream.rdbuf( m_redirectionStream.rdbuf() ); + } + + RedirectedStream::~RedirectedStream() { + m_originalStream.rdbuf( m_prevBuf ); + } + + RedirectedStdOut::RedirectedStdOut() : m_cout( Catch::cout(), m_rss.get() ) {} + auto RedirectedStdOut::str() const -> std::string { return m_rss.str(); } + + RedirectedStdErr::RedirectedStdErr() + : m_cerr( Catch::cerr(), m_rss.get() ), + m_clog( Catch::clog(), m_rss.get() ) + {} + auto RedirectedStdErr::str() const -> std::string { return m_rss.str(); } + +#if defined(CATCH_CONFIG_NEW_CAPTURE) + +#if defined(_MSC_VER) + TempFile::TempFile() { + if (tmpnam_s(m_buffer)) { + throw std::runtime_error("Could not get a temp filename"); + } + if (fopen_s(&m_file, m_buffer, "w")) { + char buffer[100]; + if (strerror_s(buffer, errno)) { + throw std::runtime_error("Could not translate errno to string"); + } + throw std::runtime_error("Could not open the temp file: " + std::string(m_buffer) + buffer); + } + } +#else + TempFile::TempFile() { + m_file = std::tmpfile(); + if (!m_file) { + throw std::runtime_error("Could not create a temp file."); + } + } + +#endif + + TempFile::~TempFile() { + // TBD: What to do about errors here? + std::fclose(m_file); + // We manually create the file on Windows only, on Linux + // it will be autodeleted +#if defined(_MSC_VER) + std::remove(m_buffer); +#endif + } + + FILE* TempFile::getFile() { + return m_file; + } + + std::string TempFile::getContents() { + std::stringstream sstr; + char buffer[100] = {}; + std::rewind(m_file); + while (std::fgets(buffer, sizeof(buffer), m_file)) { + sstr << buffer; + } + return sstr.str(); + } + + OutputRedirect::OutputRedirect(std::string& stdout_dest, std::string& stderr_dest) : + m_originalStdout(dup(1)), + m_originalStderr(dup(2)), + m_stdoutDest(stdout_dest), + m_stderrDest(stderr_dest) { + dup2(fileno(m_stdoutFile.getFile()), 1); + dup2(fileno(m_stderrFile.getFile()), 2); + } + + OutputRedirect::~OutputRedirect() { + Catch::cout() << std::flush; + fflush(stdout); + // Since we support overriding these streams, we flush cerr + // even though std::cerr is unbuffered + Catch::cerr() << std::flush; + Catch::clog() << std::flush; + fflush(stderr); + + dup2(m_originalStdout, 1); + dup2(m_originalStderr, 2); + + m_stdoutDest += m_stdoutFile.getContents(); + m_stderrDest += m_stderrFile.getContents(); + } + +#endif // CATCH_CONFIG_NEW_CAPTURE + +} // namespace Catch + +#if defined(CATCH_CONFIG_NEW_CAPTURE) + #if defined(_MSC_VER) + #undef dup + #undef dup2 + #undef fileno + #endif +#endif +// end catch_output_redirect.cpp +// start catch_random_number_generator.cpp + +// start catch_random_number_generator.h + +#include +#include + +namespace Catch { + + struct IConfig; + + std::mt19937& rng(); + void seedRng( IConfig const& config ); + unsigned int rngSeed(); + +} + +// end catch_random_number_generator.h +namespace Catch { + + std::mt19937& rng() { + static std::mt19937 s_rng; + return s_rng; + } + + void seedRng( IConfig const& config ) { + if( config.rngSeed() != 0 ) { + std::srand( config.rngSeed() ); + rng().seed( config.rngSeed() ); + } + } + + unsigned int rngSeed() { + return getCurrentContext().getConfig()->rngSeed(); + } +} +// end catch_random_number_generator.cpp +// start catch_registry_hub.cpp + +// start catch_test_case_registry_impl.h + +#include +#include +#include +#include + +namespace Catch { + + class TestCase; + struct IConfig; + + std::vector sortTests( IConfig const& config, std::vector const& unsortedTestCases ); + bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ); + + void enforceNoDuplicateTestCases( std::vector const& functions ); + + std::vector filterTests( std::vector const& testCases, TestSpec const& testSpec, IConfig const& config ); + std::vector const& getAllTestCasesSorted( IConfig const& config ); + + class TestRegistry : public ITestCaseRegistry { + public: + virtual ~TestRegistry() = default; + + virtual void registerTest( TestCase const& testCase ); + + std::vector const& getAllTests() const override; + std::vector const& getAllTestsSorted( IConfig const& config ) const override; + + private: + std::vector m_functions; + mutable RunTests::InWhatOrder m_currentSortOrder = RunTests::InDeclarationOrder; + mutable std::vector m_sortedFunctions; + std::size_t m_unnamedCount = 0; + std::ios_base::Init m_ostreamInit; // Forces cout/ cerr to be initialised + }; + + /////////////////////////////////////////////////////////////////////////// + + class TestInvokerAsFunction : public ITestInvoker { + void(*m_testAsFunction)(); + public: + TestInvokerAsFunction( void(*testAsFunction)() ) noexcept; + + void invoke() const override; + }; + + std::string extractClassName( StringRef const& classOrQualifiedMethodName ); + + /////////////////////////////////////////////////////////////////////////// + +} // end namespace Catch + +// end catch_test_case_registry_impl.h +// start catch_reporter_registry.h + +#include + +namespace Catch { + + class ReporterRegistry : public IReporterRegistry { + + public: + + ~ReporterRegistry() override; + + IStreamingReporterPtr create( std::string const& name, IConfigPtr const& config ) const override; + + void registerReporter( std::string const& name, IReporterFactoryPtr const& factory ); + void registerListener( IReporterFactoryPtr const& factory ); + + FactoryMap const& getFactories() const override; + Listeners const& getListeners() const override; + + private: + FactoryMap m_factories; + Listeners m_listeners; + }; +} + +// end catch_reporter_registry.h +// start catch_tag_alias_registry.h + +// start catch_tag_alias.h + +#include + +namespace Catch { + + struct TagAlias { + TagAlias(std::string const& _tag, SourceLineInfo _lineInfo); + + std::string tag; + SourceLineInfo lineInfo; + }; + +} // end namespace Catch + +// end catch_tag_alias.h +#include + +namespace Catch { + + class TagAliasRegistry : public ITagAliasRegistry { + public: + ~TagAliasRegistry() override; + TagAlias const* find( std::string const& alias ) const override; + std::string expandAliases( std::string const& unexpandedTestSpec ) const override; + void add( std::string const& alias, std::string const& tag, SourceLineInfo const& lineInfo ); + + private: + std::map m_registry; + }; + +} // end namespace Catch + +// end catch_tag_alias_registry.h +// start catch_startup_exception_registry.h + +#include +#include + +namespace Catch { + + class StartupExceptionRegistry { + public: + void add(std::exception_ptr const& exception) noexcept; + std::vector const& getExceptions() const noexcept; + private: + std::vector m_exceptions; + }; + +} // end namespace Catch + +// end catch_startup_exception_registry.h +namespace Catch { + + namespace { + + class RegistryHub : public IRegistryHub, public IMutableRegistryHub, + private NonCopyable { + + public: // IRegistryHub + RegistryHub() = default; + IReporterRegistry const& getReporterRegistry() const override { + return m_reporterRegistry; + } + ITestCaseRegistry const& getTestCaseRegistry() const override { + return m_testCaseRegistry; + } + IExceptionTranslatorRegistry& getExceptionTranslatorRegistry() override { + return m_exceptionTranslatorRegistry; + } + ITagAliasRegistry const& getTagAliasRegistry() const override { + return m_tagAliasRegistry; + } + StartupExceptionRegistry const& getStartupExceptionRegistry() const override { + return m_exceptionRegistry; + } + + public: // IMutableRegistryHub + void registerReporter( std::string const& name, IReporterFactoryPtr const& factory ) override { + m_reporterRegistry.registerReporter( name, factory ); + } + void registerListener( IReporterFactoryPtr const& factory ) override { + m_reporterRegistry.registerListener( factory ); + } + void registerTest( TestCase const& testInfo ) override { + m_testCaseRegistry.registerTest( testInfo ); + } + void registerTranslator( const IExceptionTranslator* translator ) override { + m_exceptionTranslatorRegistry.registerTranslator( translator ); + } + void registerTagAlias( std::string const& alias, std::string const& tag, SourceLineInfo const& lineInfo ) override { + m_tagAliasRegistry.add( alias, tag, lineInfo ); + } + void registerStartupException() noexcept override { + m_exceptionRegistry.add(std::current_exception()); + } + + private: + TestRegistry m_testCaseRegistry; + ReporterRegistry m_reporterRegistry; + ExceptionTranslatorRegistry m_exceptionTranslatorRegistry; + TagAliasRegistry m_tagAliasRegistry; + StartupExceptionRegistry m_exceptionRegistry; + }; + + // Single, global, instance + RegistryHub*& getTheRegistryHub() { + static RegistryHub* theRegistryHub = nullptr; + if( !theRegistryHub ) + theRegistryHub = new RegistryHub(); + return theRegistryHub; + } + } + + IRegistryHub& getRegistryHub() { + return *getTheRegistryHub(); + } + IMutableRegistryHub& getMutableRegistryHub() { + return *getTheRegistryHub(); + } + void cleanUp() { + delete getTheRegistryHub(); + getTheRegistryHub() = nullptr; + cleanUpContext(); + ReusableStringStream::cleanup(); + } + std::string translateActiveException() { + return getRegistryHub().getExceptionTranslatorRegistry().translateActiveException(); + } + +} // end namespace Catch +// end catch_registry_hub.cpp +// start catch_reporter_registry.cpp + +namespace Catch { + + ReporterRegistry::~ReporterRegistry() = default; + + IStreamingReporterPtr ReporterRegistry::create( std::string const& name, IConfigPtr const& config ) const { + auto it = m_factories.find( name ); + if( it == m_factories.end() ) + return nullptr; + return it->second->create( ReporterConfig( config ) ); + } + + void ReporterRegistry::registerReporter( std::string const& name, IReporterFactoryPtr const& factory ) { + m_factories.emplace(name, factory); + } + void ReporterRegistry::registerListener( IReporterFactoryPtr const& factory ) { + m_listeners.push_back( factory ); + } + + IReporterRegistry::FactoryMap const& ReporterRegistry::getFactories() const { + return m_factories; + } + IReporterRegistry::Listeners const& ReporterRegistry::getListeners() const { + return m_listeners; + } + +} +// end catch_reporter_registry.cpp +// start catch_result_type.cpp + +namespace Catch { + + bool isOk( ResultWas::OfType resultType ) { + return ( resultType & ResultWas::FailureBit ) == 0; + } + bool isJustInfo( int flags ) { + return flags == ResultWas::Info; + } + + ResultDisposition::Flags operator | ( ResultDisposition::Flags lhs, ResultDisposition::Flags rhs ) { + return static_cast( static_cast( lhs ) | static_cast( rhs ) ); + } + + bool shouldContinueOnFailure( int flags ) { return ( flags & ResultDisposition::ContinueOnFailure ) != 0; } + bool shouldSuppressFailure( int flags ) { return ( flags & ResultDisposition::SuppressFail ) != 0; } + +} // end namespace Catch +// end catch_result_type.cpp +// start catch_run_context.cpp + +#include +#include +#include + +namespace Catch { + + RunContext::RunContext(IConfigPtr const& _config, IStreamingReporterPtr&& reporter) + : m_runInfo(_config->name()), + m_context(getCurrentMutableContext()), + m_config(_config), + m_reporter(std::move(reporter)), + m_lastAssertionInfo{ StringRef(), SourceLineInfo("",0), StringRef(), ResultDisposition::Normal }, + m_includeSuccessfulResults( m_config->includeSuccessfulResults() || m_reporter->getPreferences().shouldReportAllAssertions ) + { + m_context.setRunner(this); + m_context.setConfig(m_config); + m_context.setResultCapture(this); + m_reporter->testRunStarting(m_runInfo); + } + + RunContext::~RunContext() { + m_reporter->testRunEnded(TestRunStats(m_runInfo, m_totals, aborting())); + } + + void RunContext::testGroupStarting(std::string const& testSpec, std::size_t groupIndex, std::size_t groupsCount) { + m_reporter->testGroupStarting(GroupInfo(testSpec, groupIndex, groupsCount)); + } + + void RunContext::testGroupEnded(std::string const& testSpec, Totals const& totals, std::size_t groupIndex, std::size_t groupsCount) { + m_reporter->testGroupEnded(TestGroupStats(GroupInfo(testSpec, groupIndex, groupsCount), totals, aborting())); + } + + Totals RunContext::runTest(TestCase const& testCase) { + Totals prevTotals = m_totals; + + std::string redirectedCout; + std::string redirectedCerr; + + auto const& testInfo = testCase.getTestCaseInfo(); + + m_reporter->testCaseStarting(testInfo); + + m_activeTestCase = &testCase; + + ITracker& rootTracker = m_trackerContext.startRun(); + assert(rootTracker.isSectionTracker()); + static_cast(rootTracker).addInitialFilters(m_config->getSectionsToRun()); + do { + m_trackerContext.startCycle(); + m_testCaseTracker = &SectionTracker::acquire(m_trackerContext, TestCaseTracking::NameAndLocation(testInfo.name, testInfo.lineInfo)); + runCurrentTest(redirectedCout, redirectedCerr); + } while (!m_testCaseTracker->isSuccessfullyCompleted() && !aborting()); + + Totals deltaTotals = m_totals.delta(prevTotals); + if (testInfo.expectedToFail() && deltaTotals.testCases.passed > 0) { + deltaTotals.assertions.failed++; + deltaTotals.testCases.passed--; + deltaTotals.testCases.failed++; + } + m_totals.testCases += deltaTotals.testCases; + m_reporter->testCaseEnded(TestCaseStats(testInfo, + deltaTotals, + redirectedCout, + redirectedCerr, + aborting())); + + m_activeTestCase = nullptr; + m_testCaseTracker = nullptr; + + return deltaTotals; + } + + IConfigPtr RunContext::config() const { + return m_config; + } + + IStreamingReporter& RunContext::reporter() const { + return *m_reporter; + } + + void RunContext::assertionEnded(AssertionResult const & result) { + if (result.getResultType() == ResultWas::Ok) { + m_totals.assertions.passed++; + m_lastAssertionPassed = true; + } else if (!result.isOk()) { + m_lastAssertionPassed = false; + if( m_activeTestCase->getTestCaseInfo().okToFail() ) + m_totals.assertions.failedButOk++; + else + m_totals.assertions.failed++; + } + else { + m_lastAssertionPassed = true; + } + + // We have no use for the return value (whether messages should be cleared), because messages were made scoped + // and should be let to clear themselves out. + static_cast(m_reporter->assertionEnded(AssertionStats(result, m_messages, m_totals))); + + // Reset working state + resetAssertionInfo(); + m_lastResult = result; + } + void RunContext::resetAssertionInfo() { + m_lastAssertionInfo.macroName = StringRef(); + m_lastAssertionInfo.capturedExpression = "{Unknown expression after the reported line}"_sr; + } + + bool RunContext::sectionStarted(SectionInfo const & sectionInfo, Counts & assertions) { + ITracker& sectionTracker = SectionTracker::acquire(m_trackerContext, TestCaseTracking::NameAndLocation(sectionInfo.name, sectionInfo.lineInfo)); + if (!sectionTracker.isOpen()) + return false; + m_activeSections.push_back(§ionTracker); + + m_lastAssertionInfo.lineInfo = sectionInfo.lineInfo; + + m_reporter->sectionStarting(sectionInfo); + + assertions = m_totals.assertions; + + return true; + } + + bool RunContext::testForMissingAssertions(Counts& assertions) { + if (assertions.total() != 0) + return false; + if (!m_config->warnAboutMissingAssertions()) + return false; + if (m_trackerContext.currentTracker().hasChildren()) + return false; + m_totals.assertions.failed++; + assertions.failed++; + return true; + } + + void RunContext::sectionEnded(SectionEndInfo const & endInfo) { + Counts assertions = m_totals.assertions - endInfo.prevAssertions; + bool missingAssertions = testForMissingAssertions(assertions); + + if (!m_activeSections.empty()) { + m_activeSections.back()->close(); + m_activeSections.pop_back(); + } + + m_reporter->sectionEnded(SectionStats(endInfo.sectionInfo, assertions, endInfo.durationInSeconds, missingAssertions)); + m_messages.clear(); + } + + void RunContext::sectionEndedEarly(SectionEndInfo const & endInfo) { + if (m_unfinishedSections.empty()) + m_activeSections.back()->fail(); + else + m_activeSections.back()->close(); + m_activeSections.pop_back(); + + m_unfinishedSections.push_back(endInfo); + } + void RunContext::benchmarkStarting( BenchmarkInfo const& info ) { + m_reporter->benchmarkStarting( info ); + } + void RunContext::benchmarkEnded( BenchmarkStats const& stats ) { + m_reporter->benchmarkEnded( stats ); + } + + void RunContext::pushScopedMessage(MessageInfo const & message) { + m_messages.push_back(message); + } + + void RunContext::popScopedMessage(MessageInfo const & message) { + m_messages.erase(std::remove(m_messages.begin(), m_messages.end(), message), m_messages.end()); + } + + std::string RunContext::getCurrentTestName() const { + return m_activeTestCase + ? m_activeTestCase->getTestCaseInfo().name + : std::string(); + } + + const AssertionResult * RunContext::getLastResult() const { + return &(*m_lastResult); + } + + void RunContext::exceptionEarlyReported() { + m_shouldReportUnexpected = false; + } + + void RunContext::handleFatalErrorCondition( StringRef message ) { + // First notify reporter that bad things happened + m_reporter->fatalErrorEncountered(message); + + // Don't rebuild the result -- the stringification itself can cause more fatal errors + // Instead, fake a result data. + AssertionResultData tempResult( ResultWas::FatalErrorCondition, { false } ); + tempResult.message = message; + AssertionResult result(m_lastAssertionInfo, tempResult); + + assertionEnded(result); + + handleUnfinishedSections(); + + // Recreate section for test case (as we will lose the one that was in scope) + auto const& testCaseInfo = m_activeTestCase->getTestCaseInfo(); + SectionInfo testCaseSection(testCaseInfo.lineInfo, testCaseInfo.name); + + Counts assertions; + assertions.failed = 1; + SectionStats testCaseSectionStats(testCaseSection, assertions, 0, false); + m_reporter->sectionEnded(testCaseSectionStats); + + auto const& testInfo = m_activeTestCase->getTestCaseInfo(); + + Totals deltaTotals; + deltaTotals.testCases.failed = 1; + deltaTotals.assertions.failed = 1; + m_reporter->testCaseEnded(TestCaseStats(testInfo, + deltaTotals, + std::string(), + std::string(), + false)); + m_totals.testCases.failed++; + testGroupEnded(std::string(), m_totals, 1, 1); + m_reporter->testRunEnded(TestRunStats(m_runInfo, m_totals, false)); + } + + bool RunContext::lastAssertionPassed() { + return m_lastAssertionPassed; + } + + void RunContext::assertionPassed() { + m_lastAssertionPassed = true; + ++m_totals.assertions.passed; + resetAssertionInfo(); + } + + bool RunContext::aborting() const { + return m_totals.assertions.failed == static_cast(m_config->abortAfter()); + } + + void RunContext::runCurrentTest(std::string & redirectedCout, std::string & redirectedCerr) { + auto const& testCaseInfo = m_activeTestCase->getTestCaseInfo(); + SectionInfo testCaseSection(testCaseInfo.lineInfo, testCaseInfo.name); + m_reporter->sectionStarting(testCaseSection); + Counts prevAssertions = m_totals.assertions; + double duration = 0; + m_shouldReportUnexpected = true; + m_lastAssertionInfo = { "TEST_CASE"_sr, testCaseInfo.lineInfo, StringRef(), ResultDisposition::Normal }; + + seedRng(*m_config); + + Timer timer; + try { + if (m_reporter->getPreferences().shouldRedirectStdOut) { +#if !defined(CATCH_CONFIG_EXPERIMENTAL_REDIRECT) + RedirectedStdOut redirectedStdOut; + RedirectedStdErr redirectedStdErr; + + timer.start(); + invokeActiveTestCase(); + redirectedCout += redirectedStdOut.str(); + redirectedCerr += redirectedStdErr.str(); +#else + OutputRedirect r(redirectedCout, redirectedCerr); + timer.start(); + invokeActiveTestCase(); +#endif + } else { + timer.start(); + invokeActiveTestCase(); + } + duration = timer.getElapsedSeconds(); + } catch (TestFailureException&) { + // This just means the test was aborted due to failure + } catch (...) { + // Under CATCH_CONFIG_FAST_COMPILE, unexpected exceptions under REQUIRE assertions + // are reported without translation at the point of origin. + if( m_shouldReportUnexpected ) { + AssertionReaction dummyReaction; + handleUnexpectedInflightException( m_lastAssertionInfo, translateActiveException(), dummyReaction ); + } + } + Counts assertions = m_totals.assertions - prevAssertions; + bool missingAssertions = testForMissingAssertions(assertions); + + m_testCaseTracker->close(); + handleUnfinishedSections(); + m_messages.clear(); + + SectionStats testCaseSectionStats(testCaseSection, assertions, duration, missingAssertions); + m_reporter->sectionEnded(testCaseSectionStats); + } + + void RunContext::invokeActiveTestCase() { + FatalConditionHandler fatalConditionHandler; // Handle signals + m_activeTestCase->invoke(); + fatalConditionHandler.reset(); + } + + void RunContext::handleUnfinishedSections() { + // If sections ended prematurely due to an exception we stored their + // infos here so we can tear them down outside the unwind process. + for (auto it = m_unfinishedSections.rbegin(), + itEnd = m_unfinishedSections.rend(); + it != itEnd; + ++it) + sectionEnded(*it); + m_unfinishedSections.clear(); + } + + void RunContext::handleExpr( + AssertionInfo const& info, + ITransientExpression const& expr, + AssertionReaction& reaction + ) { + m_reporter->assertionStarting( info ); + + bool negated = isFalseTest( info.resultDisposition ); + bool result = expr.getResult() != negated; + + if( result ) { + if (!m_includeSuccessfulResults) { + assertionPassed(); + } + else { + reportExpr(info, ResultWas::Ok, &expr, negated); + } + } + else { + reportExpr(info, ResultWas::ExpressionFailed, &expr, negated ); + populateReaction( reaction ); + } + } + void RunContext::reportExpr( + AssertionInfo const &info, + ResultWas::OfType resultType, + ITransientExpression const *expr, + bool negated ) { + + m_lastAssertionInfo = info; + AssertionResultData data( resultType, LazyExpression( negated ) ); + + AssertionResult assertionResult{ info, data }; + assertionResult.m_resultData.lazyExpression.m_transientExpression = expr; + + assertionEnded( assertionResult ); + } + + void RunContext::handleMessage( + AssertionInfo const& info, + ResultWas::OfType resultType, + StringRef const& message, + AssertionReaction& reaction + ) { + m_reporter->assertionStarting( info ); + + m_lastAssertionInfo = info; + + AssertionResultData data( resultType, LazyExpression( false ) ); + data.message = message; + AssertionResult assertionResult{ m_lastAssertionInfo, data }; + assertionEnded( assertionResult ); + if( !assertionResult.isOk() ) + populateReaction( reaction ); + } + void RunContext::handleUnexpectedExceptionNotThrown( + AssertionInfo const& info, + AssertionReaction& reaction + ) { + handleNonExpr(info, Catch::ResultWas::DidntThrowException, reaction); + } + + void RunContext::handleUnexpectedInflightException( + AssertionInfo const& info, + std::string const& message, + AssertionReaction& reaction + ) { + m_lastAssertionInfo = info; + + AssertionResultData data( ResultWas::ThrewException, LazyExpression( false ) ); + data.message = message; + AssertionResult assertionResult{ info, data }; + assertionEnded( assertionResult ); + populateReaction( reaction ); + } + + void RunContext::populateReaction( AssertionReaction& reaction ) { + reaction.shouldDebugBreak = m_config->shouldDebugBreak(); + reaction.shouldThrow = aborting() || (m_lastAssertionInfo.resultDisposition & ResultDisposition::Normal); + } + + void RunContext::handleIncomplete( + AssertionInfo const& info + ) { + m_lastAssertionInfo = info; + + AssertionResultData data( ResultWas::ThrewException, LazyExpression( false ) ); + data.message = "Exception translation was disabled by CATCH_CONFIG_FAST_COMPILE"; + AssertionResult assertionResult{ info, data }; + assertionEnded( assertionResult ); + } + void RunContext::handleNonExpr( + AssertionInfo const &info, + ResultWas::OfType resultType, + AssertionReaction &reaction + ) { + m_lastAssertionInfo = info; + + AssertionResultData data( resultType, LazyExpression( false ) ); + AssertionResult assertionResult{ info, data }; + assertionEnded( assertionResult ); + + if( !assertionResult.isOk() ) + populateReaction( reaction ); + } + + IResultCapture& getResultCapture() { + if (auto* capture = getCurrentContext().getResultCapture()) + return *capture; + else + CATCH_INTERNAL_ERROR("No result capture instance"); + } +} +// end catch_run_context.cpp +// start catch_section.cpp + +namespace Catch { + + Section::Section( SectionInfo const& info ) + : m_info( info ), + m_sectionIncluded( getResultCapture().sectionStarted( m_info, m_assertions ) ) + { + m_timer.start(); + } + + Section::~Section() { + if( m_sectionIncluded ) { + SectionEndInfo endInfo{ m_info, m_assertions, m_timer.getElapsedSeconds() }; + if( uncaught_exceptions() ) + getResultCapture().sectionEndedEarly( endInfo ); + else + getResultCapture().sectionEnded( endInfo ); + } + } + + // This indicates whether the section should be executed or not + Section::operator bool() const { + return m_sectionIncluded; + } + +} // end namespace Catch +// end catch_section.cpp +// start catch_section_info.cpp + +namespace Catch { + + SectionInfo::SectionInfo + ( SourceLineInfo const& _lineInfo, + std::string const& _name ) + : name( _name ), + lineInfo( _lineInfo ) + {} + +} // end namespace Catch +// end catch_section_info.cpp +// start catch_session.cpp + +// start catch_session.h + +#include + +namespace Catch { + + class Session : NonCopyable { + public: + + Session(); + ~Session() override; + + void showHelp() const; + void libIdentify(); + + int applyCommandLine( int argc, char const * const * argv ); + + void useConfigData( ConfigData const& configData ); + + int run( int argc, char* argv[] ); + #if defined(CATCH_CONFIG_WCHAR) && defined(WIN32) && defined(UNICODE) + int run( int argc, wchar_t* const argv[] ); + #endif + int run(); + + clara::Parser const& cli() const; + void cli( clara::Parser const& newParser ); + ConfigData& configData(); + Config& config(); + private: + int runInternal(); + + clara::Parser m_cli; + ConfigData m_configData; + std::shared_ptr m_config; + bool m_startupExceptions = false; + }; + +} // end namespace Catch + +// end catch_session.h +// start catch_version.h + +#include + +namespace Catch { + + // Versioning information + struct Version { + Version( Version const& ) = delete; + Version& operator=( Version const& ) = delete; + Version( unsigned int _majorVersion, + unsigned int _minorVersion, + unsigned int _patchNumber, + char const * const _branchName, + unsigned int _buildNumber ); + + unsigned int const majorVersion; + unsigned int const minorVersion; + unsigned int const patchNumber; + + // buildNumber is only used if branchName is not null + char const * const branchName; + unsigned int const buildNumber; + + friend std::ostream& operator << ( std::ostream& os, Version const& version ); + }; + + Version const& libraryVersion(); +} + +// end catch_version.h +#include +#include + +namespace Catch { + + namespace { + const int MaxExitCode = 255; + + IStreamingReporterPtr createReporter(std::string const& reporterName, IConfigPtr const& config) { + auto reporter = Catch::getRegistryHub().getReporterRegistry().create(reporterName, config); + CATCH_ENFORCE(reporter, "No reporter registered with name: '" << reporterName << "'"); + + return reporter; + } + + IStreamingReporterPtr makeReporter(std::shared_ptr const& config) { + if (Catch::getRegistryHub().getReporterRegistry().getListeners().empty()) { + return createReporter(config->getReporterName(), config); + } + + auto multi = std::unique_ptr(new ListeningReporter); + + auto const& listeners = Catch::getRegistryHub().getReporterRegistry().getListeners(); + for (auto const& listener : listeners) { + multi->addListener(listener->create(Catch::ReporterConfig(config))); + } + multi->addReporter(createReporter(config->getReporterName(), config)); + return std::move(multi); + } + + Catch::Totals runTests(std::shared_ptr const& config) { + // FixMe: Add listeners in order first, then add reporters. + + auto reporter = makeReporter(config); + + RunContext context(config, std::move(reporter)); + + Totals totals; + + context.testGroupStarting(config->name(), 1, 1); + + TestSpec testSpec = config->testSpec(); + + auto const& allTestCases = getAllTestCasesSorted(*config); + for (auto const& testCase : allTestCases) { + if (!context.aborting() && matchTest(testCase, testSpec, *config)) + totals += context.runTest(testCase); + else + context.reporter().skipTest(testCase); + } + + if (config->warnAboutNoTests() && totals.testCases.total() == 0) { + ReusableStringStream testConfig; + + bool first = true; + for (const auto& input : config->getTestsOrTags()) { + if (!first) { testConfig << ' '; } + first = false; + testConfig << input; + } + + context.reporter().noMatchingTestCases(testConfig.str()); + totals.error = -1; + } + + context.testGroupEnded(config->name(), totals, 1, 1); + return totals; + } + + void applyFilenamesAsTags(Catch::IConfig const& config) { + auto& tests = const_cast&>(getAllTestCasesSorted(config)); + for (auto& testCase : tests) { + auto tags = testCase.tags; + + std::string filename = testCase.lineInfo.file; + auto lastSlash = filename.find_last_of("\\/"); + if (lastSlash != std::string::npos) { + filename.erase(0, lastSlash); + filename[0] = '#'; + } + + auto lastDot = filename.find_last_of('.'); + if (lastDot != std::string::npos) { + filename.erase(lastDot); + } + + tags.push_back(std::move(filename)); + setTags(testCase, tags); + } + } + + } // anon namespace + + Session::Session() { + static bool alreadyInstantiated = false; + if( alreadyInstantiated ) { + try { CATCH_INTERNAL_ERROR( "Only one instance of Catch::Session can ever be used" ); } + catch(...) { getMutableRegistryHub().registerStartupException(); } + } + + const auto& exceptions = getRegistryHub().getStartupExceptionRegistry().getExceptions(); + if ( !exceptions.empty() ) { + m_startupExceptions = true; + Colour colourGuard( Colour::Red ); + Catch::cerr() << "Errors occurred during startup!" << '\n'; + // iterate over all exceptions and notify user + for ( const auto& ex_ptr : exceptions ) { + try { + std::rethrow_exception(ex_ptr); + } catch ( std::exception const& ex ) { + Catch::cerr() << Column( ex.what() ).indent(2) << '\n'; + } + } + } + + alreadyInstantiated = true; + m_cli = makeCommandLineParser( m_configData ); + } + Session::~Session() { + Catch::cleanUp(); + } + + void Session::showHelp() const { + Catch::cout() + << "\nCatch v" << libraryVersion() << "\n" + << m_cli << std::endl + << "For more detailed usage please see the project docs\n" << std::endl; + } + void Session::libIdentify() { + Catch::cout() + << std::left << std::setw(16) << "description: " << "A Catch test executable\n" + << std::left << std::setw(16) << "category: " << "testframework\n" + << std::left << std::setw(16) << "framework: " << "Catch Test\n" + << std::left << std::setw(16) << "version: " << libraryVersion() << std::endl; + } + + int Session::applyCommandLine( int argc, char const * const * argv ) { + if( m_startupExceptions ) + return 1; + + auto result = m_cli.parse( clara::Args( argc, argv ) ); + if( !result ) { + Catch::cerr() + << Colour( Colour::Red ) + << "\nError(s) in input:\n" + << Column( result.errorMessage() ).indent( 2 ) + << "\n\n"; + Catch::cerr() << "Run with -? for usage\n" << std::endl; + return MaxExitCode; + } + + if( m_configData.showHelp ) + showHelp(); + if( m_configData.libIdentify ) + libIdentify(); + m_config.reset(); + return 0; + } + + void Session::useConfigData( ConfigData const& configData ) { + m_configData = configData; + m_config.reset(); + } + + int Session::run( int argc, char* argv[] ) { + if( m_startupExceptions ) + return 1; + int returnCode = applyCommandLine( argc, argv ); + if( returnCode == 0 ) + returnCode = run(); + return returnCode; + } + +#if defined(CATCH_CONFIG_WCHAR) && defined(WIN32) && defined(UNICODE) + int Session::run( int argc, wchar_t* const argv[] ) { + + char **utf8Argv = new char *[ argc ]; + + for ( int i = 0; i < argc; ++i ) { + int bufSize = WideCharToMultiByte( CP_UTF8, 0, argv[i], -1, NULL, 0, NULL, NULL ); + + utf8Argv[ i ] = new char[ bufSize ]; + + WideCharToMultiByte( CP_UTF8, 0, argv[i], -1, utf8Argv[i], bufSize, NULL, NULL ); + } + + int returnCode = run( argc, utf8Argv ); + + for ( int i = 0; i < argc; ++i ) + delete [] utf8Argv[ i ]; + + delete [] utf8Argv; + + return returnCode; + } +#endif + int Session::run() { + if( ( m_configData.waitForKeypress & WaitForKeypress::BeforeStart ) != 0 ) { + Catch::cout() << "...waiting for enter/ return before starting" << std::endl; + static_cast(std::getchar()); + } + int exitCode = runInternal(); + if( ( m_configData.waitForKeypress & WaitForKeypress::BeforeExit ) != 0 ) { + Catch::cout() << "...waiting for enter/ return before exiting, with code: " << exitCode << std::endl; + static_cast(std::getchar()); + } + return exitCode; + } + + clara::Parser const& Session::cli() const { + return m_cli; + } + void Session::cli( clara::Parser const& newParser ) { + m_cli = newParser; + } + ConfigData& Session::configData() { + return m_configData; + } + Config& Session::config() { + if( !m_config ) + m_config = std::make_shared( m_configData ); + return *m_config; + } + + int Session::runInternal() { + if( m_startupExceptions ) + return 1; + + if( m_configData.showHelp || m_configData.libIdentify ) + return 0; + + try + { + config(); // Force config to be constructed + + seedRng( *m_config ); + + if( m_configData.filenamesAsTags ) + applyFilenamesAsTags( *m_config ); + + // Handle list request + if( Option listed = list( config() ) ) + return static_cast( *listed ); + + auto totals = runTests( m_config ); + // Note that on unices only the lower 8 bits are usually used, clamping + // the return value to 255 prevents false negative when some multiple + // of 256 tests has failed + return (std::min) (MaxExitCode, (std::max) (totals.error, static_cast(totals.assertions.failed))); + } + catch( std::exception& ex ) { + Catch::cerr() << ex.what() << std::endl; + return MaxExitCode; + } + } + +} // end namespace Catch +// end catch_session.cpp +// start catch_startup_exception_registry.cpp + +namespace Catch { + void StartupExceptionRegistry::add( std::exception_ptr const& exception ) noexcept { + try { + m_exceptions.push_back(exception); + } + catch(...) { + // If we run out of memory during start-up there's really not a lot more we can do about it + std::terminate(); + } + } + + std::vector const& StartupExceptionRegistry::getExceptions() const noexcept { + return m_exceptions; + } + +} // end namespace Catch +// end catch_startup_exception_registry.cpp +// start catch_stream.cpp + +#include +#include +#include +#include +#include +#include + +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wexit-time-destructors" +#endif + +namespace Catch { + + Catch::IStream::~IStream() = default; + + namespace detail { namespace { + template + class StreamBufImpl : public std::streambuf { + char data[bufferSize]; + WriterF m_writer; + + public: + StreamBufImpl() { + setp( data, data + sizeof(data) ); + } + + ~StreamBufImpl() noexcept { + StreamBufImpl::sync(); + } + + private: + int overflow( int c ) override { + sync(); + + if( c != EOF ) { + if( pbase() == epptr() ) + m_writer( std::string( 1, static_cast( c ) ) ); + else + sputc( static_cast( c ) ); + } + return 0; + } + + int sync() override { + if( pbase() != pptr() ) { + m_writer( std::string( pbase(), static_cast( pptr() - pbase() ) ) ); + setp( pbase(), epptr() ); + } + return 0; + } + }; + + /////////////////////////////////////////////////////////////////////////// + + struct OutputDebugWriter { + + void operator()( std::string const&str ) { + writeToDebugConsole( str ); + } + }; + + /////////////////////////////////////////////////////////////////////////// + + class FileStream : public IStream { + mutable std::ofstream m_ofs; + public: + FileStream( StringRef filename ) { + m_ofs.open( filename.c_str() ); + CATCH_ENFORCE( !m_ofs.fail(), "Unable to open file: '" << filename << "'" ); + } + ~FileStream() override = default; + public: // IStream + std::ostream& stream() const override { + return m_ofs; + } + }; + + /////////////////////////////////////////////////////////////////////////// + + class CoutStream : public IStream { + mutable std::ostream m_os; + public: + // Store the streambuf from cout up-front because + // cout may get redirected when running tests + CoutStream() : m_os( Catch::cout().rdbuf() ) {} + ~CoutStream() override = default; + + public: // IStream + std::ostream& stream() const override { return m_os; } + }; + + /////////////////////////////////////////////////////////////////////////// + + class DebugOutStream : public IStream { + std::unique_ptr> m_streamBuf; + mutable std::ostream m_os; + public: + DebugOutStream() + : m_streamBuf( new StreamBufImpl() ), + m_os( m_streamBuf.get() ) + {} + + ~DebugOutStream() override = default; + + public: // IStream + std::ostream& stream() const override { return m_os; } + }; + + }} // namespace anon::detail + + /////////////////////////////////////////////////////////////////////////// + + auto makeStream( StringRef const &filename ) -> IStream const* { + if( filename.empty() ) + return new detail::CoutStream(); + else if( filename[0] == '%' ) { + if( filename == "%debug" ) + return new detail::DebugOutStream(); + else + CATCH_ERROR( "Unrecognised stream: '" << filename << "'" ); + } + else + return new detail::FileStream( filename ); + } + + // This class encapsulates the idea of a pool of ostringstreams that can be reused. + struct StringStreams { + std::vector> m_streams; + std::vector m_unused; + std::ostringstream m_referenceStream; // Used for copy state/ flags from + static StringStreams* s_instance; + + auto add() -> std::size_t { + if( m_unused.empty() ) { + m_streams.push_back( std::unique_ptr( new std::ostringstream ) ); + return m_streams.size()-1; + } + else { + auto index = m_unused.back(); + m_unused.pop_back(); + return index; + } + } + + void release( std::size_t index ) { + m_streams[index]->copyfmt( m_referenceStream ); // Restore initial flags and other state + m_unused.push_back(index); + } + + // !TBD: put in TLS + static auto instance() -> StringStreams& { + if( !s_instance ) + s_instance = new StringStreams(); + return *s_instance; + } + static void cleanup() { + delete s_instance; + s_instance = nullptr; + } + }; + + StringStreams* StringStreams::s_instance = nullptr; + + void ReusableStringStream::cleanup() { + StringStreams::cleanup(); + } + + ReusableStringStream::ReusableStringStream() + : m_index( StringStreams::instance().add() ), + m_oss( StringStreams::instance().m_streams[m_index].get() ) + {} + + ReusableStringStream::~ReusableStringStream() { + static_cast( m_oss )->str(""); + m_oss->clear(); + StringStreams::instance().release( m_index ); + } + + auto ReusableStringStream::str() const -> std::string { + return static_cast( m_oss )->str(); + } + + /////////////////////////////////////////////////////////////////////////// + +#ifndef CATCH_CONFIG_NOSTDOUT // If you #define this you must implement these functions + std::ostream& cout() { return std::cout; } + std::ostream& cerr() { return std::cerr; } + std::ostream& clog() { return std::clog; } +#endif +} + +#if defined(__clang__) +# pragma clang diagnostic pop +#endif +// end catch_stream.cpp +// start catch_string_manip.cpp + +#include +#include +#include +#include + +namespace Catch { + + namespace { + char toLowerCh(char c) { + return static_cast( std::tolower( c ) ); + } + } + + bool startsWith( std::string const& s, std::string const& prefix ) { + return s.size() >= prefix.size() && std::equal(prefix.begin(), prefix.end(), s.begin()); + } + bool startsWith( std::string const& s, char prefix ) { + return !s.empty() && s[0] == prefix; + } + bool endsWith( std::string const& s, std::string const& suffix ) { + return s.size() >= suffix.size() && std::equal(suffix.rbegin(), suffix.rend(), s.rbegin()); + } + bool endsWith( std::string const& s, char suffix ) { + return !s.empty() && s[s.size()-1] == suffix; + } + bool contains( std::string const& s, std::string const& infix ) { + return s.find( infix ) != std::string::npos; + } + void toLowerInPlace( std::string& s ) { + std::transform( s.begin(), s.end(), s.begin(), toLowerCh ); + } + std::string toLower( std::string const& s ) { + std::string lc = s; + toLowerInPlace( lc ); + return lc; + } + std::string trim( std::string const& str ) { + static char const* whitespaceChars = "\n\r\t "; + std::string::size_type start = str.find_first_not_of( whitespaceChars ); + std::string::size_type end = str.find_last_not_of( whitespaceChars ); + + return start != std::string::npos ? str.substr( start, 1+end-start ) : std::string(); + } + + bool replaceInPlace( std::string& str, std::string const& replaceThis, std::string const& withThis ) { + bool replaced = false; + std::size_t i = str.find( replaceThis ); + while( i != std::string::npos ) { + replaced = true; + str = str.substr( 0, i ) + withThis + str.substr( i+replaceThis.size() ); + if( i < str.size()-withThis.size() ) + i = str.find( replaceThis, i+withThis.size() ); + else + i = std::string::npos; + } + return replaced; + } + + pluralise::pluralise( std::size_t count, std::string const& label ) + : m_count( count ), + m_label( label ) + {} + + std::ostream& operator << ( std::ostream& os, pluralise const& pluraliser ) { + os << pluraliser.m_count << ' ' << pluraliser.m_label; + if( pluraliser.m_count != 1 ) + os << 's'; + return os; + } + +} +// end catch_string_manip.cpp +// start catch_stringref.cpp + +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wexit-time-destructors" +#endif + +#include +#include +#include + +namespace { + const uint32_t byte_2_lead = 0xC0; + const uint32_t byte_3_lead = 0xE0; + const uint32_t byte_4_lead = 0xF0; +} + +namespace Catch { + StringRef::StringRef( char const* rawChars ) noexcept + : StringRef( rawChars, static_cast(std::strlen(rawChars) ) ) + {} + + StringRef::operator std::string() const { + return std::string( m_start, m_size ); + } + + void StringRef::swap( StringRef& other ) noexcept { + std::swap( m_start, other.m_start ); + std::swap( m_size, other.m_size ); + std::swap( m_data, other.m_data ); + } + + auto StringRef::c_str() const -> char const* { + if( isSubstring() ) + const_cast( this )->takeOwnership(); + return m_start; + } + auto StringRef::currentData() const noexcept -> char const* { + return m_start; + } + + auto StringRef::isOwned() const noexcept -> bool { + return m_data != nullptr; + } + auto StringRef::isSubstring() const noexcept -> bool { + return m_start[m_size] != '\0'; + } + + void StringRef::takeOwnership() { + if( !isOwned() ) { + m_data = new char[m_size+1]; + memcpy( m_data, m_start, m_size ); + m_data[m_size] = '\0'; + m_start = m_data; + } + } + auto StringRef::substr( size_type start, size_type size ) const noexcept -> StringRef { + if( start < m_size ) + return StringRef( m_start+start, size ); + else + return StringRef(); + } + auto StringRef::operator == ( StringRef const& other ) const noexcept -> bool { + return + size() == other.size() && + (std::strncmp( m_start, other.m_start, size() ) == 0); + } + auto StringRef::operator != ( StringRef const& other ) const noexcept -> bool { + return !operator==( other ); + } + + auto StringRef::operator[](size_type index) const noexcept -> char { + return m_start[index]; + } + + auto StringRef::numberOfCharacters() const noexcept -> size_type { + size_type noChars = m_size; + // Make adjustments for uft encodings + for( size_type i=0; i < m_size; ++i ) { + char c = m_start[i]; + if( ( c & byte_2_lead ) == byte_2_lead ) { + noChars--; + if (( c & byte_3_lead ) == byte_3_lead ) + noChars--; + if( ( c & byte_4_lead ) == byte_4_lead ) + noChars--; + } + } + return noChars; + } + + auto operator + ( StringRef const& lhs, StringRef const& rhs ) -> std::string { + std::string str; + str.reserve( lhs.size() + rhs.size() ); + str += lhs; + str += rhs; + return str; + } + auto operator + ( StringRef const& lhs, const char* rhs ) -> std::string { + return std::string( lhs ) + std::string( rhs ); + } + auto operator + ( char const* lhs, StringRef const& rhs ) -> std::string { + return std::string( lhs ) + std::string( rhs ); + } + + auto operator << ( std::ostream& os, StringRef const& str ) -> std::ostream& { + return os.write(str.currentData(), str.size()); + } + + auto operator+=( std::string& lhs, StringRef const& rhs ) -> std::string& { + lhs.append(rhs.currentData(), rhs.size()); + return lhs; + } + +} // namespace Catch + +#if defined(__clang__) +# pragma clang diagnostic pop +#endif +// end catch_stringref.cpp +// start catch_tag_alias.cpp + +namespace Catch { + TagAlias::TagAlias(std::string const & _tag, SourceLineInfo _lineInfo): tag(_tag), lineInfo(_lineInfo) {} +} +// end catch_tag_alias.cpp +// start catch_tag_alias_autoregistrar.cpp + +namespace Catch { + + RegistrarForTagAliases::RegistrarForTagAliases(char const* alias, char const* tag, SourceLineInfo const& lineInfo) { + try { + getMutableRegistryHub().registerTagAlias(alias, tag, lineInfo); + } catch (...) { + // Do not throw when constructing global objects, instead register the exception to be processed later + getMutableRegistryHub().registerStartupException(); + } + } + +} +// end catch_tag_alias_autoregistrar.cpp +// start catch_tag_alias_registry.cpp + +#include + +namespace Catch { + + TagAliasRegistry::~TagAliasRegistry() {} + + TagAlias const* TagAliasRegistry::find( std::string const& alias ) const { + auto it = m_registry.find( alias ); + if( it != m_registry.end() ) + return &(it->second); + else + return nullptr; + } + + std::string TagAliasRegistry::expandAliases( std::string const& unexpandedTestSpec ) const { + std::string expandedTestSpec = unexpandedTestSpec; + for( auto const& registryKvp : m_registry ) { + std::size_t pos = expandedTestSpec.find( registryKvp.first ); + if( pos != std::string::npos ) { + expandedTestSpec = expandedTestSpec.substr( 0, pos ) + + registryKvp.second.tag + + expandedTestSpec.substr( pos + registryKvp.first.size() ); + } + } + return expandedTestSpec; + } + + void TagAliasRegistry::add( std::string const& alias, std::string const& tag, SourceLineInfo const& lineInfo ) { + CATCH_ENFORCE( startsWith(alias, "[@") && endsWith(alias, ']'), + "error: tag alias, '" << alias << "' is not of the form [@alias name].\n" << lineInfo ); + + CATCH_ENFORCE( m_registry.insert(std::make_pair(alias, TagAlias(tag, lineInfo))).second, + "error: tag alias, '" << alias << "' already registered.\n" + << "\tFirst seen at: " << find(alias)->lineInfo << "\n" + << "\tRedefined at: " << lineInfo ); + } + + ITagAliasRegistry::~ITagAliasRegistry() {} + + ITagAliasRegistry const& ITagAliasRegistry::get() { + return getRegistryHub().getTagAliasRegistry(); + } + +} // end namespace Catch +// end catch_tag_alias_registry.cpp +// start catch_test_case_info.cpp + +#include +#include +#include +#include + +namespace Catch { + + namespace { + TestCaseInfo::SpecialProperties parseSpecialTag( std::string const& tag ) { + if( startsWith( tag, '.' ) || + tag == "!hide" ) + return TestCaseInfo::IsHidden; + else if( tag == "!throws" ) + return TestCaseInfo::Throws; + else if( tag == "!shouldfail" ) + return TestCaseInfo::ShouldFail; + else if( tag == "!mayfail" ) + return TestCaseInfo::MayFail; + else if( tag == "!nonportable" ) + return TestCaseInfo::NonPortable; + else if( tag == "!benchmark" ) + return static_cast( TestCaseInfo::Benchmark | TestCaseInfo::IsHidden ); + else + return TestCaseInfo::None; + } + bool isReservedTag( std::string const& tag ) { + return parseSpecialTag( tag ) == TestCaseInfo::None && tag.size() > 0 && !std::isalnum( static_cast(tag[0]) ); + } + void enforceNotReservedTag( std::string const& tag, SourceLineInfo const& _lineInfo ) { + CATCH_ENFORCE( !isReservedTag(tag), + "Tag name: [" << tag << "] is not allowed.\n" + << "Tag names starting with non alpha-numeric characters are reserved\n" + << _lineInfo ); + } + } + + TestCase makeTestCase( ITestInvoker* _testCase, + std::string const& _className, + NameAndTags const& nameAndTags, + SourceLineInfo const& _lineInfo ) + { + bool isHidden = false; + + // Parse out tags + std::vector tags; + std::string desc, tag; + bool inTag = false; + std::string _descOrTags = nameAndTags.tags; + for (char c : _descOrTags) { + if( !inTag ) { + if( c == '[' ) + inTag = true; + else + desc += c; + } + else { + if( c == ']' ) { + TestCaseInfo::SpecialProperties prop = parseSpecialTag( tag ); + if( ( prop & TestCaseInfo::IsHidden ) != 0 ) + isHidden = true; + else if( prop == TestCaseInfo::None ) + enforceNotReservedTag( tag, _lineInfo ); + + tags.push_back( tag ); + tag.clear(); + inTag = false; + } + else + tag += c; + } + } + if( isHidden ) { + tags.push_back( "." ); + } + + TestCaseInfo info( nameAndTags.name, _className, desc, tags, _lineInfo ); + return TestCase( _testCase, std::move(info) ); + } + + void setTags( TestCaseInfo& testCaseInfo, std::vector tags ) { + std::sort(begin(tags), end(tags)); + tags.erase(std::unique(begin(tags), end(tags)), end(tags)); + testCaseInfo.lcaseTags.clear(); + + for( auto const& tag : tags ) { + std::string lcaseTag = toLower( tag ); + testCaseInfo.properties = static_cast( testCaseInfo.properties | parseSpecialTag( lcaseTag ) ); + testCaseInfo.lcaseTags.push_back( lcaseTag ); + } + testCaseInfo.tags = std::move(tags); + } + + TestCaseInfo::TestCaseInfo( std::string const& _name, + std::string const& _className, + std::string const& _description, + std::vector const& _tags, + SourceLineInfo const& _lineInfo ) + : name( _name ), + className( _className ), + description( _description ), + lineInfo( _lineInfo ), + properties( None ) + { + setTags( *this, _tags ); + } + + bool TestCaseInfo::isHidden() const { + return ( properties & IsHidden ) != 0; + } + bool TestCaseInfo::throws() const { + return ( properties & Throws ) != 0; + } + bool TestCaseInfo::okToFail() const { + return ( properties & (ShouldFail | MayFail ) ) != 0; + } + bool TestCaseInfo::expectedToFail() const { + return ( properties & (ShouldFail ) ) != 0; + } + + std::string TestCaseInfo::tagsAsString() const { + std::string ret; + // '[' and ']' per tag + std::size_t full_size = 2 * tags.size(); + for (const auto& tag : tags) { + full_size += tag.size(); + } + ret.reserve(full_size); + for (const auto& tag : tags) { + ret.push_back('['); + ret.append(tag); + ret.push_back(']'); + } + + return ret; + } + + TestCase::TestCase( ITestInvoker* testCase, TestCaseInfo&& info ) : TestCaseInfo( std::move(info) ), test( testCase ) {} + + TestCase TestCase::withName( std::string const& _newName ) const { + TestCase other( *this ); + other.name = _newName; + return other; + } + + void TestCase::invoke() const { + test->invoke(); + } + + bool TestCase::operator == ( TestCase const& other ) const { + return test.get() == other.test.get() && + name == other.name && + className == other.className; + } + + bool TestCase::operator < ( TestCase const& other ) const { + return name < other.name; + } + + TestCaseInfo const& TestCase::getTestCaseInfo() const + { + return *this; + } + +} // end namespace Catch +// end catch_test_case_info.cpp +// start catch_test_case_registry_impl.cpp + +#include + +namespace Catch { + + std::vector sortTests( IConfig const& config, std::vector const& unsortedTestCases ) { + + std::vector sorted = unsortedTestCases; + + switch( config.runOrder() ) { + case RunTests::InLexicographicalOrder: + std::sort( sorted.begin(), sorted.end() ); + break; + case RunTests::InRandomOrder: + seedRng( config ); + std::shuffle( sorted.begin(), sorted.end(), rng() ); + break; + case RunTests::InDeclarationOrder: + // already in declaration order + break; + } + return sorted; + } + bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ) { + return testSpec.matches( testCase ) && ( config.allowThrows() || !testCase.throws() ); + } + + void enforceNoDuplicateTestCases( std::vector const& functions ) { + std::set seenFunctions; + for( auto const& function : functions ) { + auto prev = seenFunctions.insert( function ); + CATCH_ENFORCE( prev.second, + "error: TEST_CASE( \"" << function.name << "\" ) already defined.\n" + << "\tFirst seen at " << prev.first->getTestCaseInfo().lineInfo << "\n" + << "\tRedefined at " << function.getTestCaseInfo().lineInfo ); + } + } + + std::vector filterTests( std::vector const& testCases, TestSpec const& testSpec, IConfig const& config ) { + std::vector filtered; + filtered.reserve( testCases.size() ); + for( auto const& testCase : testCases ) + if( matchTest( testCase, testSpec, config ) ) + filtered.push_back( testCase ); + return filtered; + } + std::vector const& getAllTestCasesSorted( IConfig const& config ) { + return getRegistryHub().getTestCaseRegistry().getAllTestsSorted( config ); + } + + void TestRegistry::registerTest( TestCase const& testCase ) { + std::string name = testCase.getTestCaseInfo().name; + if( name.empty() ) { + ReusableStringStream rss; + rss << "Anonymous test case " << ++m_unnamedCount; + return registerTest( testCase.withName( rss.str() ) ); + } + m_functions.push_back( testCase ); + } + + std::vector const& TestRegistry::getAllTests() const { + return m_functions; + } + std::vector const& TestRegistry::getAllTestsSorted( IConfig const& config ) const { + if( m_sortedFunctions.empty() ) + enforceNoDuplicateTestCases( m_functions ); + + if( m_currentSortOrder != config.runOrder() || m_sortedFunctions.empty() ) { + m_sortedFunctions = sortTests( config, m_functions ); + m_currentSortOrder = config.runOrder(); + } + return m_sortedFunctions; + } + + /////////////////////////////////////////////////////////////////////////// + TestInvokerAsFunction::TestInvokerAsFunction( void(*testAsFunction)() ) noexcept : m_testAsFunction( testAsFunction ) {} + + void TestInvokerAsFunction::invoke() const { + m_testAsFunction(); + } + + std::string extractClassName( StringRef const& classOrQualifiedMethodName ) { + std::string className = classOrQualifiedMethodName; + if( startsWith( className, '&' ) ) + { + std::size_t lastColons = className.rfind( "::" ); + std::size_t penultimateColons = className.rfind( "::", lastColons-1 ); + if( penultimateColons == std::string::npos ) + penultimateColons = 1; + className = className.substr( penultimateColons, lastColons-penultimateColons ); + } + return className; + } + +} // end namespace Catch +// end catch_test_case_registry_impl.cpp +// start catch_test_case_tracker.cpp + +#include +#include +#include +#include +#include + +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wexit-time-destructors" +#endif + +namespace Catch { +namespace TestCaseTracking { + + NameAndLocation::NameAndLocation( std::string const& _name, SourceLineInfo const& _location ) + : name( _name ), + location( _location ) + {} + + ITracker::~ITracker() = default; + + TrackerContext& TrackerContext::instance() { + static TrackerContext s_instance; + return s_instance; + } + + ITracker& TrackerContext::startRun() { + m_rootTracker = std::make_shared( NameAndLocation( "{root}", CATCH_INTERNAL_LINEINFO ), *this, nullptr ); + m_currentTracker = nullptr; + m_runState = Executing; + return *m_rootTracker; + } + + void TrackerContext::endRun() { + m_rootTracker.reset(); + m_currentTracker = nullptr; + m_runState = NotStarted; + } + + void TrackerContext::startCycle() { + m_currentTracker = m_rootTracker.get(); + m_runState = Executing; + } + void TrackerContext::completeCycle() { + m_runState = CompletedCycle; + } + + bool TrackerContext::completedCycle() const { + return m_runState == CompletedCycle; + } + ITracker& TrackerContext::currentTracker() { + return *m_currentTracker; + } + void TrackerContext::setCurrentTracker( ITracker* tracker ) { + m_currentTracker = tracker; + } + + TrackerBase::TrackerHasName::TrackerHasName( NameAndLocation const& nameAndLocation ) : m_nameAndLocation( nameAndLocation ) {} + bool TrackerBase::TrackerHasName::operator ()( ITrackerPtr const& tracker ) const { + return + tracker->nameAndLocation().location == m_nameAndLocation.location && + tracker->nameAndLocation().name == m_nameAndLocation.name; + } + + TrackerBase::TrackerBase( NameAndLocation const& nameAndLocation, TrackerContext& ctx, ITracker* parent ) + : m_nameAndLocation( nameAndLocation ), + m_ctx( ctx ), + m_parent( parent ) + {} + + NameAndLocation const& TrackerBase::nameAndLocation() const { + return m_nameAndLocation; + } + bool TrackerBase::isComplete() const { + return m_runState == CompletedSuccessfully || m_runState == Failed; + } + bool TrackerBase::isSuccessfullyCompleted() const { + return m_runState == CompletedSuccessfully; + } + bool TrackerBase::isOpen() const { + return m_runState != NotStarted && !isComplete(); + } + bool TrackerBase::hasChildren() const { + return !m_children.empty(); + } + + void TrackerBase::addChild( ITrackerPtr const& child ) { + m_children.push_back( child ); + } + + ITrackerPtr TrackerBase::findChild( NameAndLocation const& nameAndLocation ) { + auto it = std::find_if( m_children.begin(), m_children.end(), TrackerHasName( nameAndLocation ) ); + return( it != m_children.end() ) + ? *it + : nullptr; + } + ITracker& TrackerBase::parent() { + assert( m_parent ); // Should always be non-null except for root + return *m_parent; + } + + void TrackerBase::openChild() { + if( m_runState != ExecutingChildren ) { + m_runState = ExecutingChildren; + if( m_parent ) + m_parent->openChild(); + } + } + + bool TrackerBase::isSectionTracker() const { return false; } + bool TrackerBase::isIndexTracker() const { return false; } + + void TrackerBase::open() { + m_runState = Executing; + moveToThis(); + if( m_parent ) + m_parent->openChild(); + } + + void TrackerBase::close() { + + // Close any still open children (e.g. generators) + while( &m_ctx.currentTracker() != this ) + m_ctx.currentTracker().close(); + + switch( m_runState ) { + case NeedsAnotherRun: + break; + + case Executing: + m_runState = CompletedSuccessfully; + break; + case ExecutingChildren: + if( m_children.empty() || m_children.back()->isComplete() ) + m_runState = CompletedSuccessfully; + break; + + case NotStarted: + case CompletedSuccessfully: + case Failed: + CATCH_INTERNAL_ERROR( "Illogical state: " << m_runState ); + + default: + CATCH_INTERNAL_ERROR( "Unknown state: " << m_runState ); + } + moveToParent(); + m_ctx.completeCycle(); + } + void TrackerBase::fail() { + m_runState = Failed; + if( m_parent ) + m_parent->markAsNeedingAnotherRun(); + moveToParent(); + m_ctx.completeCycle(); + } + void TrackerBase::markAsNeedingAnotherRun() { + m_runState = NeedsAnotherRun; + } + + void TrackerBase::moveToParent() { + assert( m_parent ); + m_ctx.setCurrentTracker( m_parent ); + } + void TrackerBase::moveToThis() { + m_ctx.setCurrentTracker( this ); + } + + SectionTracker::SectionTracker( NameAndLocation const& nameAndLocation, TrackerContext& ctx, ITracker* parent ) + : TrackerBase( nameAndLocation, ctx, parent ) + { + if( parent ) { + while( !parent->isSectionTracker() ) + parent = &parent->parent(); + + SectionTracker& parentSection = static_cast( *parent ); + addNextFilters( parentSection.m_filters ); + } + } + + bool SectionTracker::isSectionTracker() const { return true; } + + SectionTracker& SectionTracker::acquire( TrackerContext& ctx, NameAndLocation const& nameAndLocation ) { + std::shared_ptr section; + + ITracker& currentTracker = ctx.currentTracker(); + if( ITrackerPtr childTracker = currentTracker.findChild( nameAndLocation ) ) { + assert( childTracker ); + assert( childTracker->isSectionTracker() ); + section = std::static_pointer_cast( childTracker ); + } + else { + section = std::make_shared( nameAndLocation, ctx, ¤tTracker ); + currentTracker.addChild( section ); + } + if( !ctx.completedCycle() ) + section->tryOpen(); + return *section; + } + + void SectionTracker::tryOpen() { + if( !isComplete() && (m_filters.empty() || m_filters[0].empty() || m_filters[0] == m_nameAndLocation.name ) ) + open(); + } + + void SectionTracker::addInitialFilters( std::vector const& filters ) { + if( !filters.empty() ) { + m_filters.push_back(""); // Root - should never be consulted + m_filters.push_back(""); // Test Case - not a section filter + m_filters.insert( m_filters.end(), filters.begin(), filters.end() ); + } + } + void SectionTracker::addNextFilters( std::vector const& filters ) { + if( filters.size() > 1 ) + m_filters.insert( m_filters.end(), ++filters.begin(), filters.end() ); + } + + IndexTracker::IndexTracker( NameAndLocation const& nameAndLocation, TrackerContext& ctx, ITracker* parent, int size ) + : TrackerBase( nameAndLocation, ctx, parent ), + m_size( size ) + {} + + bool IndexTracker::isIndexTracker() const { return true; } + + IndexTracker& IndexTracker::acquire( TrackerContext& ctx, NameAndLocation const& nameAndLocation, int size ) { + std::shared_ptr tracker; + + ITracker& currentTracker = ctx.currentTracker(); + if( ITrackerPtr childTracker = currentTracker.findChild( nameAndLocation ) ) { + assert( childTracker ); + assert( childTracker->isIndexTracker() ); + tracker = std::static_pointer_cast( childTracker ); + } + else { + tracker = std::make_shared( nameAndLocation, ctx, ¤tTracker, size ); + currentTracker.addChild( tracker ); + } + + if( !ctx.completedCycle() && !tracker->isComplete() ) { + if( tracker->m_runState != ExecutingChildren && tracker->m_runState != NeedsAnotherRun ) + tracker->moveNext(); + tracker->open(); + } + + return *tracker; + } + + int IndexTracker::index() const { return m_index; } + + void IndexTracker::moveNext() { + m_index++; + m_children.clear(); + } + + void IndexTracker::close() { + TrackerBase::close(); + if( m_runState == CompletedSuccessfully && m_index < m_size-1 ) + m_runState = Executing; + } + +} // namespace TestCaseTracking + +using TestCaseTracking::ITracker; +using TestCaseTracking::TrackerContext; +using TestCaseTracking::SectionTracker; +using TestCaseTracking::IndexTracker; + +} // namespace Catch + +#if defined(__clang__) +# pragma clang diagnostic pop +#endif +// end catch_test_case_tracker.cpp +// start catch_test_registry.cpp + +namespace Catch { + + auto makeTestInvoker( void(*testAsFunction)() ) noexcept -> ITestInvoker* { + return new(std::nothrow) TestInvokerAsFunction( testAsFunction ); + } + + NameAndTags::NameAndTags( StringRef const& name_ , StringRef const& tags_ ) noexcept : name( name_ ), tags( tags_ ) {} + + AutoReg::AutoReg( ITestInvoker* invoker, SourceLineInfo const& lineInfo, StringRef const& classOrMethod, NameAndTags const& nameAndTags ) noexcept { + try { + getMutableRegistryHub() + .registerTest( + makeTestCase( + invoker, + extractClassName( classOrMethod ), + nameAndTags, + lineInfo)); + } catch (...) { + // Do not throw when constructing global objects, instead register the exception to be processed later + getMutableRegistryHub().registerStartupException(); + } + } + + AutoReg::~AutoReg() = default; +} +// end catch_test_registry.cpp +// start catch_test_spec.cpp + +#include +#include +#include +#include + +namespace Catch { + + TestSpec::Pattern::~Pattern() = default; + TestSpec::NamePattern::~NamePattern() = default; + TestSpec::TagPattern::~TagPattern() = default; + TestSpec::ExcludedPattern::~ExcludedPattern() = default; + + TestSpec::NamePattern::NamePattern( std::string const& name ) + : m_wildcardPattern( toLower( name ), CaseSensitive::No ) + {} + bool TestSpec::NamePattern::matches( TestCaseInfo const& testCase ) const { + return m_wildcardPattern.matches( toLower( testCase.name ) ); + } + + TestSpec::TagPattern::TagPattern( std::string const& tag ) : m_tag( toLower( tag ) ) {} + bool TestSpec::TagPattern::matches( TestCaseInfo const& testCase ) const { + return std::find(begin(testCase.lcaseTags), + end(testCase.lcaseTags), + m_tag) != end(testCase.lcaseTags); + } + + TestSpec::ExcludedPattern::ExcludedPattern( PatternPtr const& underlyingPattern ) : m_underlyingPattern( underlyingPattern ) {} + bool TestSpec::ExcludedPattern::matches( TestCaseInfo const& testCase ) const { return !m_underlyingPattern->matches( testCase ); } + + bool TestSpec::Filter::matches( TestCaseInfo const& testCase ) const { + // All patterns in a filter must match for the filter to be a match + for( auto const& pattern : m_patterns ) { + if( !pattern->matches( testCase ) ) + return false; + } + return true; + } + + bool TestSpec::hasFilters() const { + return !m_filters.empty(); + } + bool TestSpec::matches( TestCaseInfo const& testCase ) const { + // A TestSpec matches if any filter matches + for( auto const& filter : m_filters ) + if( filter.matches( testCase ) ) + return true; + return false; + } +} +// end catch_test_spec.cpp +// start catch_test_spec_parser.cpp + +namespace Catch { + + TestSpecParser::TestSpecParser( ITagAliasRegistry const& tagAliases ) : m_tagAliases( &tagAliases ) {} + + TestSpecParser& TestSpecParser::parse( std::string const& arg ) { + m_mode = None; + m_exclusion = false; + m_start = std::string::npos; + m_arg = m_tagAliases->expandAliases( arg ); + m_escapeChars.clear(); + for( m_pos = 0; m_pos < m_arg.size(); ++m_pos ) + visitChar( m_arg[m_pos] ); + if( m_mode == Name ) + addPattern(); + return *this; + } + TestSpec TestSpecParser::testSpec() { + addFilter(); + return m_testSpec; + } + + void TestSpecParser::visitChar( char c ) { + if( m_mode == None ) { + switch( c ) { + case ' ': return; + case '~': m_exclusion = true; return; + case '[': return startNewMode( Tag, ++m_pos ); + case '"': return startNewMode( QuotedName, ++m_pos ); + case '\\': return escape(); + default: startNewMode( Name, m_pos ); break; + } + } + if( m_mode == Name ) { + if( c == ',' ) { + addPattern(); + addFilter(); + } + else if( c == '[' ) { + if( subString() == "exclude:" ) + m_exclusion = true; + else + addPattern(); + startNewMode( Tag, ++m_pos ); + } + else if( c == '\\' ) + escape(); + } + else if( m_mode == EscapedName ) + m_mode = Name; + else if( m_mode == QuotedName && c == '"' ) + addPattern(); + else if( m_mode == Tag && c == ']' ) + addPattern(); + } + void TestSpecParser::startNewMode( Mode mode, std::size_t start ) { + m_mode = mode; + m_start = start; + } + void TestSpecParser::escape() { + if( m_mode == None ) + m_start = m_pos; + m_mode = EscapedName; + m_escapeChars.push_back( m_pos ); + } + std::string TestSpecParser::subString() const { return m_arg.substr( m_start, m_pos - m_start ); } + + void TestSpecParser::addFilter() { + if( !m_currentFilter.m_patterns.empty() ) { + m_testSpec.m_filters.push_back( m_currentFilter ); + m_currentFilter = TestSpec::Filter(); + } + } + + TestSpec parseTestSpec( std::string const& arg ) { + return TestSpecParser( ITagAliasRegistry::get() ).parse( arg ).testSpec(); + } + +} // namespace Catch +// end catch_test_spec_parser.cpp +// start catch_timer.cpp + +#include + +static const uint64_t nanosecondsInSecond = 1000000000; + +namespace Catch { + + auto getCurrentNanosecondsSinceEpoch() -> uint64_t { + return std::chrono::duration_cast( std::chrono::high_resolution_clock::now().time_since_epoch() ).count(); + } + + namespace { + auto estimateClockResolution() -> uint64_t { + uint64_t sum = 0; + static const uint64_t iterations = 1000000; + + auto startTime = getCurrentNanosecondsSinceEpoch(); + + for( std::size_t i = 0; i < iterations; ++i ) { + + uint64_t ticks; + uint64_t baseTicks = getCurrentNanosecondsSinceEpoch(); + do { + ticks = getCurrentNanosecondsSinceEpoch(); + } while( ticks == baseTicks ); + + auto delta = ticks - baseTicks; + sum += delta; + + // If we have been calibrating for over 3 seconds -- the clock + // is terrible and we should move on. + // TBD: How to signal that the measured resolution is probably wrong? + if (ticks > startTime + 3 * nanosecondsInSecond) { + return sum / i; + } + } + + // We're just taking the mean, here. To do better we could take the std. dev and exclude outliers + // - and potentially do more iterations if there's a high variance. + return sum/iterations; + } + } + auto getEstimatedClockResolution() -> uint64_t { + static auto s_resolution = estimateClockResolution(); + return s_resolution; + } + + void Timer::start() { + m_nanoseconds = getCurrentNanosecondsSinceEpoch(); + } + auto Timer::getElapsedNanoseconds() const -> uint64_t { + return getCurrentNanosecondsSinceEpoch() - m_nanoseconds; + } + auto Timer::getElapsedMicroseconds() const -> uint64_t { + return getElapsedNanoseconds()/1000; + } + auto Timer::getElapsedMilliseconds() const -> unsigned int { + return static_cast(getElapsedMicroseconds()/1000); + } + auto Timer::getElapsedSeconds() const -> double { + return getElapsedMicroseconds()/1000000.0; + } + +} // namespace Catch +// end catch_timer.cpp +// start catch_tostring.cpp + +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wexit-time-destructors" +# pragma clang diagnostic ignored "-Wglobal-constructors" +#endif + +// Enable specific decls locally +#if !defined(CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER) +#define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER +#endif + +#include +#include + +namespace Catch { + +namespace Detail { + + const std::string unprintableString = "{?}"; + + namespace { + const int hexThreshold = 255; + + struct Endianness { + enum Arch { Big, Little }; + + static Arch which() { + union _{ + int asInt; + char asChar[sizeof (int)]; + } u; + + u.asInt = 1; + return ( u.asChar[sizeof(int)-1] == 1 ) ? Big : Little; + } + }; + } + + std::string rawMemoryToString( const void *object, std::size_t size ) { + // Reverse order for little endian architectures + int i = 0, end = static_cast( size ), inc = 1; + if( Endianness::which() == Endianness::Little ) { + i = end-1; + end = inc = -1; + } + + unsigned char const *bytes = static_cast(object); + ReusableStringStream rss; + rss << "0x" << std::setfill('0') << std::hex; + for( ; i != end; i += inc ) + rss << std::setw(2) << static_cast(bytes[i]); + return rss.str(); + } +} + +template +std::string fpToString( T value, int precision ) { + if (std::isnan(value)) { + return "nan"; + } + + ReusableStringStream rss; + rss << std::setprecision( precision ) + << std::fixed + << value; + std::string d = rss.str(); + std::size_t i = d.find_last_not_of( '0' ); + if( i != std::string::npos && i != d.size()-1 ) { + if( d[i] == '.' ) + i++; + d = d.substr( 0, i+1 ); + } + return d; +} + +//// ======================================================= //// +// +// Out-of-line defs for full specialization of StringMaker +// +//// ======================================================= //// + +std::string StringMaker::convert(const std::string& str) { + if (!getCurrentContext().getConfig()->showInvisibles()) { + return '"' + str + '"'; + } + + std::string s("\""); + for (char c : str) { + switch (c) { + case '\n': + s.append("\\n"); + break; + case '\t': + s.append("\\t"); + break; + default: + s.push_back(c); + break; + } + } + s.append("\""); + return s; +} + +#ifdef CATCH_CONFIG_WCHAR +std::string StringMaker::convert(const std::wstring& wstr) { + std::string s; + s.reserve(wstr.size()); + for (auto c : wstr) { + s += (c <= 0xff) ? static_cast(c) : '?'; + } + return ::Catch::Detail::stringify(s); +} +#endif + +std::string StringMaker::convert(char const* str) { + if (str) { + return ::Catch::Detail::stringify(std::string{ str }); + } else { + return{ "{null string}" }; + } +} +std::string StringMaker::convert(char* str) { + if (str) { + return ::Catch::Detail::stringify(std::string{ str }); + } else { + return{ "{null string}" }; + } +} +#ifdef CATCH_CONFIG_WCHAR +std::string StringMaker::convert(wchar_t const * str) { + if (str) { + return ::Catch::Detail::stringify(std::wstring{ str }); + } else { + return{ "{null string}" }; + } +} +std::string StringMaker::convert(wchar_t * str) { + if (str) { + return ::Catch::Detail::stringify(std::wstring{ str }); + } else { + return{ "{null string}" }; + } +} +#endif + +std::string StringMaker::convert(int value) { + return ::Catch::Detail::stringify(static_cast(value)); +} +std::string StringMaker::convert(long value) { + return ::Catch::Detail::stringify(static_cast(value)); +} +std::string StringMaker::convert(long long value) { + ReusableStringStream rss; + rss << value; + if (value > Detail::hexThreshold) { + rss << " (0x" << std::hex << value << ')'; + } + return rss.str(); +} + +std::string StringMaker::convert(unsigned int value) { + return ::Catch::Detail::stringify(static_cast(value)); +} +std::string StringMaker::convert(unsigned long value) { + return ::Catch::Detail::stringify(static_cast(value)); +} +std::string StringMaker::convert(unsigned long long value) { + ReusableStringStream rss; + rss << value; + if (value > Detail::hexThreshold) { + rss << " (0x" << std::hex << value << ')'; + } + return rss.str(); +} + +std::string StringMaker::convert(bool b) { + return b ? "true" : "false"; +} + +std::string StringMaker::convert(char value) { + if (value == '\r') { + return "'\\r'"; + } else if (value == '\f') { + return "'\\f'"; + } else if (value == '\n') { + return "'\\n'"; + } else if (value == '\t') { + return "'\\t'"; + } else if ('\0' <= value && value < ' ') { + return ::Catch::Detail::stringify(static_cast(value)); + } else { + char chstr[] = "' '"; + chstr[1] = value; + return chstr; + } +} +std::string StringMaker::convert(signed char c) { + return ::Catch::Detail::stringify(static_cast(c)); +} +std::string StringMaker::convert(unsigned char c) { + return ::Catch::Detail::stringify(static_cast(c)); +} + +std::string StringMaker::convert(std::nullptr_t) { + return "nullptr"; +} + +std::string StringMaker::convert(float value) { + return fpToString(value, 5) + 'f'; +} +std::string StringMaker::convert(double value) { + return fpToString(value, 10); +} + +std::string ratio_string::symbol() { return "a"; } +std::string ratio_string::symbol() { return "f"; } +std::string ratio_string::symbol() { return "p"; } +std::string ratio_string::symbol() { return "n"; } +std::string ratio_string::symbol() { return "u"; } +std::string ratio_string::symbol() { return "m"; } + +} // end namespace Catch + +#if defined(__clang__) +# pragma clang diagnostic pop +#endif + +// end catch_tostring.cpp +// start catch_totals.cpp + +namespace Catch { + + Counts Counts::operator - ( Counts const& other ) const { + Counts diff; + diff.passed = passed - other.passed; + diff.failed = failed - other.failed; + diff.failedButOk = failedButOk - other.failedButOk; + return diff; + } + + Counts& Counts::operator += ( Counts const& other ) { + passed += other.passed; + failed += other.failed; + failedButOk += other.failedButOk; + return *this; + } + + std::size_t Counts::total() const { + return passed + failed + failedButOk; + } + bool Counts::allPassed() const { + return failed == 0 && failedButOk == 0; + } + bool Counts::allOk() const { + return failed == 0; + } + + Totals Totals::operator - ( Totals const& other ) const { + Totals diff; + diff.assertions = assertions - other.assertions; + diff.testCases = testCases - other.testCases; + return diff; + } + + Totals& Totals::operator += ( Totals const& other ) { + assertions += other.assertions; + testCases += other.testCases; + return *this; + } + + Totals Totals::delta( Totals const& prevTotals ) const { + Totals diff = *this - prevTotals; + if( diff.assertions.failed > 0 ) + ++diff.testCases.failed; + else if( diff.assertions.failedButOk > 0 ) + ++diff.testCases.failedButOk; + else + ++diff.testCases.passed; + return diff; + } + +} +// end catch_totals.cpp +// start catch_uncaught_exceptions.cpp + +#include + +namespace Catch { + bool uncaught_exceptions() { +#if defined(CATCH_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS) + return std::uncaught_exceptions() > 0; +#else + return std::uncaught_exception(); +#endif + } +} // end namespace Catch +// end catch_uncaught_exceptions.cpp +// start catch_version.cpp + +#include + +namespace Catch { + + Version::Version + ( unsigned int _majorVersion, + unsigned int _minorVersion, + unsigned int _patchNumber, + char const * const _branchName, + unsigned int _buildNumber ) + : majorVersion( _majorVersion ), + minorVersion( _minorVersion ), + patchNumber( _patchNumber ), + branchName( _branchName ), + buildNumber( _buildNumber ) + {} + + std::ostream& operator << ( std::ostream& os, Version const& version ) { + os << version.majorVersion << '.' + << version.minorVersion << '.' + << version.patchNumber; + // branchName is never null -> 0th char is \0 if it is empty + if (version.branchName[0]) { + os << '-' << version.branchName + << '.' << version.buildNumber; + } + return os; + } + + Version const& libraryVersion() { + static Version version( 2, 3, 0, "", 0 ); + return version; + } + +} +// end catch_version.cpp +// start catch_wildcard_pattern.cpp + +#include + +namespace Catch { + + WildcardPattern::WildcardPattern( std::string const& pattern, + CaseSensitive::Choice caseSensitivity ) + : m_caseSensitivity( caseSensitivity ), + m_pattern( adjustCase( pattern ) ) + { + if( startsWith( m_pattern, '*' ) ) { + m_pattern = m_pattern.substr( 1 ); + m_wildcard = WildcardAtStart; + } + if( endsWith( m_pattern, '*' ) ) { + m_pattern = m_pattern.substr( 0, m_pattern.size()-1 ); + m_wildcard = static_cast( m_wildcard | WildcardAtEnd ); + } + } + + bool WildcardPattern::matches( std::string const& str ) const { + switch( m_wildcard ) { + case NoWildcard: + return m_pattern == adjustCase( str ); + case WildcardAtStart: + return endsWith( adjustCase( str ), m_pattern ); + case WildcardAtEnd: + return startsWith( adjustCase( str ), m_pattern ); + case WildcardAtBothEnds: + return contains( adjustCase( str ), m_pattern ); + default: + CATCH_INTERNAL_ERROR( "Unknown enum" ); + } + } + + std::string WildcardPattern::adjustCase( std::string const& str ) const { + return m_caseSensitivity == CaseSensitive::No ? toLower( str ) : str; + } +} +// end catch_wildcard_pattern.cpp +// start catch_xmlwriter.cpp + +#include + +using uchar = unsigned char; + +namespace Catch { + +namespace { + + size_t trailingBytes(unsigned char c) { + if ((c & 0xE0) == 0xC0) { + return 2; + } + if ((c & 0xF0) == 0xE0) { + return 3; + } + if ((c & 0xF8) == 0xF0) { + return 4; + } + CATCH_INTERNAL_ERROR("Invalid multibyte utf-8 start byte encountered"); + } + + uint32_t headerValue(unsigned char c) { + if ((c & 0xE0) == 0xC0) { + return c & 0x1F; + } + if ((c & 0xF0) == 0xE0) { + return c & 0x0F; + } + if ((c & 0xF8) == 0xF0) { + return c & 0x07; + } + CATCH_INTERNAL_ERROR("Invalid multibyte utf-8 start byte encountered"); + } + + void hexEscapeChar(std::ostream& os, unsigned char c) { + os << "\\x" + << std::uppercase << std::hex << std::setfill('0') << std::setw(2) + << static_cast(c); + } + +} // anonymous namespace + + XmlEncode::XmlEncode( std::string const& str, ForWhat forWhat ) + : m_str( str ), + m_forWhat( forWhat ) + {} + + void XmlEncode::encodeTo( std::ostream& os ) const { + // Apostrophe escaping not necessary if we always use " to write attributes + // (see: http://www.w3.org/TR/xml/#syntax) + + for( std::size_t idx = 0; idx < m_str.size(); ++ idx ) { + uchar c = m_str[idx]; + switch (c) { + case '<': os << "<"; break; + case '&': os << "&"; break; + + case '>': + // See: http://www.w3.org/TR/xml/#syntax + if (idx > 2 && m_str[idx - 1] == ']' && m_str[idx - 2] == ']') + os << ">"; + else + os << c; + break; + + case '\"': + if (m_forWhat == ForAttributes) + os << """; + else + os << c; + break; + + default: + // Check for control characters and invalid utf-8 + + // Escape control characters in standard ascii + // see http://stackoverflow.com/questions/404107/why-are-control-characters-illegal-in-xml-1-0 + if (c < 0x09 || (c > 0x0D && c < 0x20) || c == 0x7F) { + hexEscapeChar(os, c); + break; + } + + // Plain ASCII: Write it to stream + if (c < 0x7F) { + os << c; + break; + } + + // UTF-8 territory + // Check if the encoding is valid and if it is not, hex escape bytes. + // Important: We do not check the exact decoded values for validity, only the encoding format + // First check that this bytes is a valid lead byte: + // This means that it is not encoded as 1111 1XXX + // Or as 10XX XXXX + if (c < 0xC0 || + c >= 0xF8) { + hexEscapeChar(os, c); + break; + } + + auto encBytes = trailingBytes(c); + // Are there enough bytes left to avoid accessing out-of-bounds memory? + if (idx + encBytes - 1 >= m_str.size()) { + hexEscapeChar(os, c); + break; + } + // The header is valid, check data + // The next encBytes bytes must together be a valid utf-8 + // This means: bitpattern 10XX XXXX and the extracted value is sane (ish) + bool valid = true; + uint32_t value = headerValue(c); + for (std::size_t n = 1; n < encBytes; ++n) { + uchar nc = m_str[idx + n]; + valid &= ((nc & 0xC0) == 0x80); + value = (value << 6) | (nc & 0x3F); + } + + if ( + // Wrong bit pattern of following bytes + (!valid) || + // Overlong encodings + (value < 0x80) || + (0x80 <= value && value < 0x800 && encBytes > 2) || + (0x800 < value && value < 0x10000 && encBytes > 3) || + // Encoded value out of range + (value >= 0x110000) + ) { + hexEscapeChar(os, c); + break; + } + + // If we got here, this is in fact a valid(ish) utf-8 sequence + for (std::size_t n = 0; n < encBytes; ++n) { + os << m_str[idx + n]; + } + idx += encBytes - 1; + break; + } + } + } + + std::ostream& operator << ( std::ostream& os, XmlEncode const& xmlEncode ) { + xmlEncode.encodeTo( os ); + return os; + } + + XmlWriter::ScopedElement::ScopedElement( XmlWriter* writer ) + : m_writer( writer ) + {} + + XmlWriter::ScopedElement::ScopedElement( ScopedElement&& other ) noexcept + : m_writer( other.m_writer ){ + other.m_writer = nullptr; + } + XmlWriter::ScopedElement& XmlWriter::ScopedElement::operator=( ScopedElement&& other ) noexcept { + if ( m_writer ) { + m_writer->endElement(); + } + m_writer = other.m_writer; + other.m_writer = nullptr; + return *this; + } + + XmlWriter::ScopedElement::~ScopedElement() { + if( m_writer ) + m_writer->endElement(); + } + + XmlWriter::ScopedElement& XmlWriter::ScopedElement::writeText( std::string const& text, bool indent ) { + m_writer->writeText( text, indent ); + return *this; + } + + XmlWriter::XmlWriter( std::ostream& os ) : m_os( os ) + { + writeDeclaration(); + } + + XmlWriter::~XmlWriter() { + while( !m_tags.empty() ) + endElement(); + } + + XmlWriter& XmlWriter::startElement( std::string const& name ) { + ensureTagClosed(); + newlineIfNecessary(); + m_os << m_indent << '<' << name; + m_tags.push_back( name ); + m_indent += " "; + m_tagIsOpen = true; + return *this; + } + + XmlWriter::ScopedElement XmlWriter::scopedElement( std::string const& name ) { + ScopedElement scoped( this ); + startElement( name ); + return scoped; + } + + XmlWriter& XmlWriter::endElement() { + newlineIfNecessary(); + m_indent = m_indent.substr( 0, m_indent.size()-2 ); + if( m_tagIsOpen ) { + m_os << "/>"; + m_tagIsOpen = false; + } + else { + m_os << m_indent << ""; + } + m_os << std::endl; + m_tags.pop_back(); + return *this; + } + + XmlWriter& XmlWriter::writeAttribute( std::string const& name, std::string const& attribute ) { + if( !name.empty() && !attribute.empty() ) + m_os << ' ' << name << "=\"" << XmlEncode( attribute, XmlEncode::ForAttributes ) << '"'; + return *this; + } + + XmlWriter& XmlWriter::writeAttribute( std::string const& name, bool attribute ) { + m_os << ' ' << name << "=\"" << ( attribute ? "true" : "false" ) << '"'; + return *this; + } + + XmlWriter& XmlWriter::writeText( std::string const& text, bool indent ) { + if( !text.empty() ){ + bool tagWasOpen = m_tagIsOpen; + ensureTagClosed(); + if( tagWasOpen && indent ) + m_os << m_indent; + m_os << XmlEncode( text ); + m_needsNewline = true; + } + return *this; + } + + XmlWriter& XmlWriter::writeComment( std::string const& text ) { + ensureTagClosed(); + m_os << m_indent << ""; + m_needsNewline = true; + return *this; + } + + void XmlWriter::writeStylesheetRef( std::string const& url ) { + m_os << "\n"; + } + + XmlWriter& XmlWriter::writeBlankLine() { + ensureTagClosed(); + m_os << '\n'; + return *this; + } + + void XmlWriter::ensureTagClosed() { + if( m_tagIsOpen ) { + m_os << ">" << std::endl; + m_tagIsOpen = false; + } + } + + void XmlWriter::writeDeclaration() { + m_os << "\n"; + } + + void XmlWriter::newlineIfNecessary() { + if( m_needsNewline ) { + m_os << std::endl; + m_needsNewline = false; + } + } +} +// end catch_xmlwriter.cpp +// start catch_reporter_bases.cpp + +#include +#include +#include +#include +#include + +namespace Catch { + void prepareExpandedExpression(AssertionResult& result) { + result.getExpandedExpression(); + } + + // Because formatting using c++ streams is stateful, drop down to C is required + // Alternatively we could use stringstream, but its performance is... not good. + std::string getFormattedDuration( double duration ) { + // Max exponent + 1 is required to represent the whole part + // + 1 for decimal point + // + 3 for the 3 decimal places + // + 1 for null terminator + const std::size_t maxDoubleSize = DBL_MAX_10_EXP + 1 + 1 + 3 + 1; + char buffer[maxDoubleSize]; + + // Save previous errno, to prevent sprintf from overwriting it + ErrnoGuard guard; +#ifdef _MSC_VER + sprintf_s(buffer, "%.3f", duration); +#else + sprintf(buffer, "%.3f", duration); +#endif + return std::string(buffer); + } + + TestEventListenerBase::TestEventListenerBase(ReporterConfig const & _config) + :StreamingReporterBase(_config) {} + + void TestEventListenerBase::assertionStarting(AssertionInfo const &) {} + + bool TestEventListenerBase::assertionEnded(AssertionStats const &) { + return false; + } + +} // end namespace Catch +// end catch_reporter_bases.cpp +// start catch_reporter_compact.cpp + +namespace { + +#ifdef CATCH_PLATFORM_MAC + const char* failedString() { return "FAILED"; } + const char* passedString() { return "PASSED"; } +#else + const char* failedString() { return "failed"; } + const char* passedString() { return "passed"; } +#endif + + // Colour::LightGrey + Catch::Colour::Code dimColour() { return Catch::Colour::FileName; } + + std::string bothOrAll( std::size_t count ) { + return count == 1 ? std::string() : + count == 2 ? "both " : "all " ; + } + +} // anon namespace + +namespace Catch { +namespace { +// Colour, message variants: +// - white: No tests ran. +// - red: Failed [both/all] N test cases, failed [both/all] M assertions. +// - white: Passed [both/all] N test cases (no assertions). +// - red: Failed N tests cases, failed M assertions. +// - green: Passed [both/all] N tests cases with M assertions. +void printTotals(std::ostream& out, const Totals& totals) { + if (totals.testCases.total() == 0) { + out << "No tests ran."; + } else if (totals.testCases.failed == totals.testCases.total()) { + Colour colour(Colour::ResultError); + const std::string qualify_assertions_failed = + totals.assertions.failed == totals.assertions.total() ? + bothOrAll(totals.assertions.failed) : std::string(); + out << + "Failed " << bothOrAll(totals.testCases.failed) + << pluralise(totals.testCases.failed, "test case") << ", " + "failed " << qualify_assertions_failed << + pluralise(totals.assertions.failed, "assertion") << '.'; + } else if (totals.assertions.total() == 0) { + out << + "Passed " << bothOrAll(totals.testCases.total()) + << pluralise(totals.testCases.total(), "test case") + << " (no assertions)."; + } else if (totals.assertions.failed) { + Colour colour(Colour::ResultError); + out << + "Failed " << pluralise(totals.testCases.failed, "test case") << ", " + "failed " << pluralise(totals.assertions.failed, "assertion") << '.'; + } else { + Colour colour(Colour::ResultSuccess); + out << + "Passed " << bothOrAll(totals.testCases.passed) + << pluralise(totals.testCases.passed, "test case") << + " with " << pluralise(totals.assertions.passed, "assertion") << '.'; + } +} + +// Implementation of CompactReporter formatting +class AssertionPrinter { +public: + AssertionPrinter& operator= (AssertionPrinter const&) = delete; + AssertionPrinter(AssertionPrinter const&) = delete; + AssertionPrinter(std::ostream& _stream, AssertionStats const& _stats, bool _printInfoMessages) + : stream(_stream) + , result(_stats.assertionResult) + , messages(_stats.infoMessages) + , itMessage(_stats.infoMessages.begin()) + , printInfoMessages(_printInfoMessages) {} + + void print() { + printSourceInfo(); + + itMessage = messages.begin(); + + switch (result.getResultType()) { + case ResultWas::Ok: + printResultType(Colour::ResultSuccess, passedString()); + printOriginalExpression(); + printReconstructedExpression(); + if (!result.hasExpression()) + printRemainingMessages(Colour::None); + else + printRemainingMessages(); + break; + case ResultWas::ExpressionFailed: + if (result.isOk()) + printResultType(Colour::ResultSuccess, failedString() + std::string(" - but was ok")); + else + printResultType(Colour::Error, failedString()); + printOriginalExpression(); + printReconstructedExpression(); + printRemainingMessages(); + break; + case ResultWas::ThrewException: + printResultType(Colour::Error, failedString()); + printIssue("unexpected exception with message:"); + printMessage(); + printExpressionWas(); + printRemainingMessages(); + break; + case ResultWas::FatalErrorCondition: + printResultType(Colour::Error, failedString()); + printIssue("fatal error condition with message:"); + printMessage(); + printExpressionWas(); + printRemainingMessages(); + break; + case ResultWas::DidntThrowException: + printResultType(Colour::Error, failedString()); + printIssue("expected exception, got none"); + printExpressionWas(); + printRemainingMessages(); + break; + case ResultWas::Info: + printResultType(Colour::None, "info"); + printMessage(); + printRemainingMessages(); + break; + case ResultWas::Warning: + printResultType(Colour::None, "warning"); + printMessage(); + printRemainingMessages(); + break; + case ResultWas::ExplicitFailure: + printResultType(Colour::Error, failedString()); + printIssue("explicitly"); + printRemainingMessages(Colour::None); + break; + // These cases are here to prevent compiler warnings + case ResultWas::Unknown: + case ResultWas::FailureBit: + case ResultWas::Exception: + printResultType(Colour::Error, "** internal error **"); + break; + } + } + +private: + void printSourceInfo() const { + Colour colourGuard(Colour::FileName); + stream << result.getSourceInfo() << ':'; + } + + void printResultType(Colour::Code colour, std::string const& passOrFail) const { + if (!passOrFail.empty()) { + { + Colour colourGuard(colour); + stream << ' ' << passOrFail; + } + stream << ':'; + } + } + + void printIssue(std::string const& issue) const { + stream << ' ' << issue; + } + + void printExpressionWas() { + if (result.hasExpression()) { + stream << ';'; + { + Colour colour(dimColour()); + stream << " expression was:"; + } + printOriginalExpression(); + } + } + + void printOriginalExpression() const { + if (result.hasExpression()) { + stream << ' ' << result.getExpression(); + } + } + + void printReconstructedExpression() const { + if (result.hasExpandedExpression()) { + { + Colour colour(dimColour()); + stream << " for: "; + } + stream << result.getExpandedExpression(); + } + } + + void printMessage() { + if (itMessage != messages.end()) { + stream << " '" << itMessage->message << '\''; + ++itMessage; + } + } + + void printRemainingMessages(Colour::Code colour = dimColour()) { + if (itMessage == messages.end()) + return; + + // using messages.end() directly yields (or auto) compilation error: + std::vector::const_iterator itEnd = messages.end(); + const std::size_t N = static_cast(std::distance(itMessage, itEnd)); + + { + Colour colourGuard(colour); + stream << " with " << pluralise(N, "message") << ':'; + } + + for (; itMessage != itEnd; ) { + // If this assertion is a warning ignore any INFO messages + if (printInfoMessages || itMessage->type != ResultWas::Info) { + stream << " '" << itMessage->message << '\''; + if (++itMessage != itEnd) { + Colour colourGuard(dimColour()); + stream << " and"; + } + } + } + } + +private: + std::ostream& stream; + AssertionResult const& result; + std::vector messages; + std::vector::const_iterator itMessage; + bool printInfoMessages; +}; + +} // anon namespace + + std::string CompactReporter::getDescription() { + return "Reports test results on a single line, suitable for IDEs"; + } + + ReporterPreferences CompactReporter::getPreferences() const { + return m_reporterPrefs; + } + + void CompactReporter::noMatchingTestCases( std::string const& spec ) { + stream << "No test cases matched '" << spec << '\'' << std::endl; + } + + void CompactReporter::assertionStarting( AssertionInfo const& ) {} + + bool CompactReporter::assertionEnded( AssertionStats const& _assertionStats ) { + AssertionResult const& result = _assertionStats.assertionResult; + + bool printInfoMessages = true; + + // Drop out if result was successful and we're not printing those + if( !m_config->includeSuccessfulResults() && result.isOk() ) { + if( result.getResultType() != ResultWas::Warning ) + return false; + printInfoMessages = false; + } + + AssertionPrinter printer( stream, _assertionStats, printInfoMessages ); + printer.print(); + + stream << std::endl; + return true; + } + + void CompactReporter::sectionEnded(SectionStats const& _sectionStats) { + if (m_config->showDurations() == ShowDurations::Always) { + stream << getFormattedDuration(_sectionStats.durationInSeconds) << " s: " << _sectionStats.sectionInfo.name << std::endl; + } + } + + void CompactReporter::testRunEnded( TestRunStats const& _testRunStats ) { + printTotals( stream, _testRunStats.totals ); + stream << '\n' << std::endl; + StreamingReporterBase::testRunEnded( _testRunStats ); + } + + CompactReporter::~CompactReporter() {} + + CATCH_REGISTER_REPORTER( "compact", CompactReporter ) + +} // end namespace Catch +// end catch_reporter_compact.cpp +// start catch_reporter_console.cpp + +#include +#include + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable:4061) // Not all labels are EXPLICITLY handled in switch + // Note that 4062 (not all labels are handled + // and default is missing) is enabled +#endif + +namespace Catch { + +namespace { + +// Formatter impl for ConsoleReporter +class ConsoleAssertionPrinter { +public: + ConsoleAssertionPrinter& operator= (ConsoleAssertionPrinter const&) = delete; + ConsoleAssertionPrinter(ConsoleAssertionPrinter const&) = delete; + ConsoleAssertionPrinter(std::ostream& _stream, AssertionStats const& _stats, bool _printInfoMessages) + : stream(_stream), + stats(_stats), + result(_stats.assertionResult), + colour(Colour::None), + message(result.getMessage()), + messages(_stats.infoMessages), + printInfoMessages(_printInfoMessages) { + switch (result.getResultType()) { + case ResultWas::Ok: + colour = Colour::Success; + passOrFail = "PASSED"; + //if( result.hasMessage() ) + if (_stats.infoMessages.size() == 1) + messageLabel = "with message"; + if (_stats.infoMessages.size() > 1) + messageLabel = "with messages"; + break; + case ResultWas::ExpressionFailed: + if (result.isOk()) { + colour = Colour::Success; + passOrFail = "FAILED - but was ok"; + } else { + colour = Colour::Error; + passOrFail = "FAILED"; + } + if (_stats.infoMessages.size() == 1) + messageLabel = "with message"; + if (_stats.infoMessages.size() > 1) + messageLabel = "with messages"; + break; + case ResultWas::ThrewException: + colour = Colour::Error; + passOrFail = "FAILED"; + messageLabel = "due to unexpected exception with "; + if (_stats.infoMessages.size() == 1) + messageLabel += "message"; + if (_stats.infoMessages.size() > 1) + messageLabel += "messages"; + break; + case ResultWas::FatalErrorCondition: + colour = Colour::Error; + passOrFail = "FAILED"; + messageLabel = "due to a fatal error condition"; + break; + case ResultWas::DidntThrowException: + colour = Colour::Error; + passOrFail = "FAILED"; + messageLabel = "because no exception was thrown where one was expected"; + break; + case ResultWas::Info: + messageLabel = "info"; + break; + case ResultWas::Warning: + messageLabel = "warning"; + break; + case ResultWas::ExplicitFailure: + passOrFail = "FAILED"; + colour = Colour::Error; + if (_stats.infoMessages.size() == 1) + messageLabel = "explicitly with message"; + if (_stats.infoMessages.size() > 1) + messageLabel = "explicitly with messages"; + break; + // These cases are here to prevent compiler warnings + case ResultWas::Unknown: + case ResultWas::FailureBit: + case ResultWas::Exception: + passOrFail = "** internal error **"; + colour = Colour::Error; + break; + } + } + + void print() const { + printSourceInfo(); + if (stats.totals.assertions.total() > 0) { + if (result.isOk()) + stream << '\n'; + printResultType(); + printOriginalExpression(); + printReconstructedExpression(); + } else { + stream << '\n'; + } + printMessage(); + } + +private: + void printResultType() const { + if (!passOrFail.empty()) { + Colour colourGuard(colour); + stream << passOrFail << ":\n"; + } + } + void printOriginalExpression() const { + if (result.hasExpression()) { + Colour colourGuard(Colour::OriginalExpression); + stream << " "; + stream << result.getExpressionInMacro(); + stream << '\n'; + } + } + void printReconstructedExpression() const { + if (result.hasExpandedExpression()) { + stream << "with expansion:\n"; + Colour colourGuard(Colour::ReconstructedExpression); + stream << Column(result.getExpandedExpression()).indent(2) << '\n'; + } + } + void printMessage() const { + if (!messageLabel.empty()) + stream << messageLabel << ':' << '\n'; + for (auto const& msg : messages) { + // If this assertion is a warning ignore any INFO messages + if (printInfoMessages || msg.type != ResultWas::Info) + stream << Column(msg.message).indent(2) << '\n'; + } + } + void printSourceInfo() const { + Colour colourGuard(Colour::FileName); + stream << result.getSourceInfo() << ": "; + } + + std::ostream& stream; + AssertionStats const& stats; + AssertionResult const& result; + Colour::Code colour; + std::string passOrFail; + std::string messageLabel; + std::string message; + std::vector messages; + bool printInfoMessages; +}; + +std::size_t makeRatio(std::size_t number, std::size_t total) { + std::size_t ratio = total > 0 ? CATCH_CONFIG_CONSOLE_WIDTH * number / total : 0; + return (ratio == 0 && number > 0) ? 1 : ratio; +} + +std::size_t& findMax(std::size_t& i, std::size_t& j, std::size_t& k) { + if (i > j && i > k) + return i; + else if (j > k) + return j; + else + return k; +} + +struct ColumnInfo { + enum Justification { Left, Right }; + std::string name; + int width; + Justification justification; +}; +struct ColumnBreak {}; +struct RowBreak {}; + +class Duration { + enum class Unit { + Auto, + Nanoseconds, + Microseconds, + Milliseconds, + Seconds, + Minutes + }; + static const uint64_t s_nanosecondsInAMicrosecond = 1000; + static const uint64_t s_nanosecondsInAMillisecond = 1000 * s_nanosecondsInAMicrosecond; + static const uint64_t s_nanosecondsInASecond = 1000 * s_nanosecondsInAMillisecond; + static const uint64_t s_nanosecondsInAMinute = 60 * s_nanosecondsInASecond; + + uint64_t m_inNanoseconds; + Unit m_units; + +public: + explicit Duration(uint64_t inNanoseconds, Unit units = Unit::Auto) + : m_inNanoseconds(inNanoseconds), + m_units(units) { + if (m_units == Unit::Auto) { + if (m_inNanoseconds < s_nanosecondsInAMicrosecond) + m_units = Unit::Nanoseconds; + else if (m_inNanoseconds < s_nanosecondsInAMillisecond) + m_units = Unit::Microseconds; + else if (m_inNanoseconds < s_nanosecondsInASecond) + m_units = Unit::Milliseconds; + else if (m_inNanoseconds < s_nanosecondsInAMinute) + m_units = Unit::Seconds; + else + m_units = Unit::Minutes; + } + + } + + auto value() const -> double { + switch (m_units) { + case Unit::Microseconds: + return m_inNanoseconds / static_cast(s_nanosecondsInAMicrosecond); + case Unit::Milliseconds: + return m_inNanoseconds / static_cast(s_nanosecondsInAMillisecond); + case Unit::Seconds: + return m_inNanoseconds / static_cast(s_nanosecondsInASecond); + case Unit::Minutes: + return m_inNanoseconds / static_cast(s_nanosecondsInAMinute); + default: + return static_cast(m_inNanoseconds); + } + } + auto unitsAsString() const -> std::string { + switch (m_units) { + case Unit::Nanoseconds: + return "ns"; + case Unit::Microseconds: + return "µs"; + case Unit::Milliseconds: + return "ms"; + case Unit::Seconds: + return "s"; + case Unit::Minutes: + return "m"; + default: + return "** internal error **"; + } + + } + friend auto operator << (std::ostream& os, Duration const& duration) -> std::ostream& { + return os << duration.value() << " " << duration.unitsAsString(); + } +}; +} // end anon namespace + +class TablePrinter { + std::ostream& m_os; + std::vector m_columnInfos; + std::ostringstream m_oss; + int m_currentColumn = -1; + bool m_isOpen = false; + +public: + TablePrinter( std::ostream& os, std::vector columnInfos ) + : m_os( os ), + m_columnInfos( std::move( columnInfos ) ) {} + + auto columnInfos() const -> std::vector const& { + return m_columnInfos; + } + + void open() { + if (!m_isOpen) { + m_isOpen = true; + *this << RowBreak(); + for (auto const& info : m_columnInfos) + *this << info.name << ColumnBreak(); + *this << RowBreak(); + m_os << Catch::getLineOfChars<'-'>() << "\n"; + } + } + void close() { + if (m_isOpen) { + *this << RowBreak(); + m_os << std::endl; + m_isOpen = false; + } + } + + template + friend TablePrinter& operator << (TablePrinter& tp, T const& value) { + tp.m_oss << value; + return tp; + } + + friend TablePrinter& operator << (TablePrinter& tp, ColumnBreak) { + auto colStr = tp.m_oss.str(); + // This takes account of utf8 encodings + auto strSize = Catch::StringRef(colStr).numberOfCharacters(); + tp.m_oss.str(""); + tp.open(); + if (tp.m_currentColumn == static_cast(tp.m_columnInfos.size() - 1)) { + tp.m_currentColumn = -1; + tp.m_os << "\n"; + } + tp.m_currentColumn++; + + auto colInfo = tp.m_columnInfos[tp.m_currentColumn]; + auto padding = (strSize + 2 < static_cast(colInfo.width)) + ? std::string(colInfo.width - (strSize + 2), ' ') + : std::string(); + if (colInfo.justification == ColumnInfo::Left) + tp.m_os << colStr << padding << " "; + else + tp.m_os << padding << colStr << " "; + return tp; + } + + friend TablePrinter& operator << (TablePrinter& tp, RowBreak) { + if (tp.m_currentColumn > 0) { + tp.m_os << "\n"; + tp.m_currentColumn = -1; + } + return tp; + } +}; + +ConsoleReporter::ConsoleReporter(ReporterConfig const& config) + : StreamingReporterBase(config), + m_tablePrinter(new TablePrinter(config.stream(), + { + { "benchmark name", CATCH_CONFIG_CONSOLE_WIDTH - 32, ColumnInfo::Left }, + { "iters", 8, ColumnInfo::Right }, + { "elapsed ns", 14, ColumnInfo::Right }, + { "average", 14, ColumnInfo::Right } + })) {} +ConsoleReporter::~ConsoleReporter() = default; + +std::string ConsoleReporter::getDescription() { + return "Reports test results as plain lines of text"; +} + +void ConsoleReporter::noMatchingTestCases(std::string const& spec) { + stream << "No test cases matched '" << spec << '\'' << std::endl; +} + +void ConsoleReporter::assertionStarting(AssertionInfo const&) {} + +bool ConsoleReporter::assertionEnded(AssertionStats const& _assertionStats) { + AssertionResult const& result = _assertionStats.assertionResult; + + bool includeResults = m_config->includeSuccessfulResults() || !result.isOk(); + + // Drop out if result was successful but we're not printing them. + if (!includeResults && result.getResultType() != ResultWas::Warning) + return false; + + lazyPrint(); + + ConsoleAssertionPrinter printer(stream, _assertionStats, includeResults); + printer.print(); + stream << std::endl; + return true; +} + +void ConsoleReporter::sectionStarting(SectionInfo const& _sectionInfo) { + m_headerPrinted = false; + StreamingReporterBase::sectionStarting(_sectionInfo); +} +void ConsoleReporter::sectionEnded(SectionStats const& _sectionStats) { + m_tablePrinter->close(); + if (_sectionStats.missingAssertions) { + lazyPrint(); + Colour colour(Colour::ResultError); + if (m_sectionStack.size() > 1) + stream << "\nNo assertions in section"; + else + stream << "\nNo assertions in test case"; + stream << " '" << _sectionStats.sectionInfo.name << "'\n" << std::endl; + } + if (m_config->showDurations() == ShowDurations::Always) { + stream << getFormattedDuration(_sectionStats.durationInSeconds) << " s: " << _sectionStats.sectionInfo.name << std::endl; + } + if (m_headerPrinted) { + m_headerPrinted = false; + } + StreamingReporterBase::sectionEnded(_sectionStats); +} + +void ConsoleReporter::benchmarkStarting(BenchmarkInfo const& info) { + lazyPrintWithoutClosingBenchmarkTable(); + + auto nameCol = Column( info.name ).width( static_cast( m_tablePrinter->columnInfos()[0].width - 2 ) ); + + bool firstLine = true; + for (auto line : nameCol) { + if (!firstLine) + (*m_tablePrinter) << ColumnBreak() << ColumnBreak() << ColumnBreak(); + else + firstLine = false; + + (*m_tablePrinter) << line << ColumnBreak(); + } +} +void ConsoleReporter::benchmarkEnded(BenchmarkStats const& stats) { + Duration average(stats.elapsedTimeInNanoseconds / stats.iterations); + (*m_tablePrinter) + << stats.iterations << ColumnBreak() + << stats.elapsedTimeInNanoseconds << ColumnBreak() + << average << ColumnBreak(); +} + +void ConsoleReporter::testCaseEnded(TestCaseStats const& _testCaseStats) { + m_tablePrinter->close(); + StreamingReporterBase::testCaseEnded(_testCaseStats); + m_headerPrinted = false; +} +void ConsoleReporter::testGroupEnded(TestGroupStats const& _testGroupStats) { + if (currentGroupInfo.used) { + printSummaryDivider(); + stream << "Summary for group '" << _testGroupStats.groupInfo.name << "':\n"; + printTotals(_testGroupStats.totals); + stream << '\n' << std::endl; + } + StreamingReporterBase::testGroupEnded(_testGroupStats); +} +void ConsoleReporter::testRunEnded(TestRunStats const& _testRunStats) { + printTotalsDivider(_testRunStats.totals); + printTotals(_testRunStats.totals); + stream << std::endl; + StreamingReporterBase::testRunEnded(_testRunStats); +} + +void ConsoleReporter::lazyPrint() { + + m_tablePrinter->close(); + lazyPrintWithoutClosingBenchmarkTable(); +} + +void ConsoleReporter::lazyPrintWithoutClosingBenchmarkTable() { + + if (!currentTestRunInfo.used) + lazyPrintRunInfo(); + if (!currentGroupInfo.used) + lazyPrintGroupInfo(); + + if (!m_headerPrinted) { + printTestCaseAndSectionHeader(); + m_headerPrinted = true; + } +} +void ConsoleReporter::lazyPrintRunInfo() { + stream << '\n' << getLineOfChars<'~'>() << '\n'; + Colour colour(Colour::SecondaryText); + stream << currentTestRunInfo->name + << " is a Catch v" << libraryVersion() << " host application.\n" + << "Run with -? for options\n\n"; + + if (m_config->rngSeed() != 0) + stream << "Randomness seeded to: " << m_config->rngSeed() << "\n\n"; + + currentTestRunInfo.used = true; +} +void ConsoleReporter::lazyPrintGroupInfo() { + if (!currentGroupInfo->name.empty() && currentGroupInfo->groupsCounts > 1) { + printClosedHeader("Group: " + currentGroupInfo->name); + currentGroupInfo.used = true; + } +} +void ConsoleReporter::printTestCaseAndSectionHeader() { + assert(!m_sectionStack.empty()); + printOpenHeader(currentTestCaseInfo->name); + + if (m_sectionStack.size() > 1) { + Colour colourGuard(Colour::Headers); + + auto + it = m_sectionStack.begin() + 1, // Skip first section (test case) + itEnd = m_sectionStack.end(); + for (; it != itEnd; ++it) + printHeaderString(it->name, 2); + } + + SourceLineInfo lineInfo = m_sectionStack.back().lineInfo; + + if (!lineInfo.empty()) { + stream << getLineOfChars<'-'>() << '\n'; + Colour colourGuard(Colour::FileName); + stream << lineInfo << '\n'; + } + stream << getLineOfChars<'.'>() << '\n' << std::endl; +} + +void ConsoleReporter::printClosedHeader(std::string const& _name) { + printOpenHeader(_name); + stream << getLineOfChars<'.'>() << '\n'; +} +void ConsoleReporter::printOpenHeader(std::string const& _name) { + stream << getLineOfChars<'-'>() << '\n'; + { + Colour colourGuard(Colour::Headers); + printHeaderString(_name); + } +} + +// if string has a : in first line will set indent to follow it on +// subsequent lines +void ConsoleReporter::printHeaderString(std::string const& _string, std::size_t indent) { + std::size_t i = _string.find(": "); + if (i != std::string::npos) + i += 2; + else + i = 0; + stream << Column(_string).indent(indent + i).initialIndent(indent) << '\n'; +} + +struct SummaryColumn { + + SummaryColumn( std::string _label, Colour::Code _colour ) + : label( std::move( _label ) ), + colour( _colour ) {} + SummaryColumn addRow( std::size_t count ) { + ReusableStringStream rss; + rss << count; + std::string row = rss.str(); + for (auto& oldRow : rows) { + while (oldRow.size() < row.size()) + oldRow = ' ' + oldRow; + while (oldRow.size() > row.size()) + row = ' ' + row; + } + rows.push_back(row); + return *this; + } + + std::string label; + Colour::Code colour; + std::vector rows; + +}; + +void ConsoleReporter::printTotals( Totals const& totals ) { + if (totals.testCases.total() == 0) { + stream << Colour(Colour::Warning) << "No tests ran\n"; + } else if (totals.assertions.total() > 0 && totals.testCases.allPassed()) { + stream << Colour(Colour::ResultSuccess) << "All tests passed"; + stream << " (" + << pluralise(totals.assertions.passed, "assertion") << " in " + << pluralise(totals.testCases.passed, "test case") << ')' + << '\n'; + } else { + + std::vector columns; + columns.push_back(SummaryColumn("", Colour::None) + .addRow(totals.testCases.total()) + .addRow(totals.assertions.total())); + columns.push_back(SummaryColumn("passed", Colour::Success) + .addRow(totals.testCases.passed) + .addRow(totals.assertions.passed)); + columns.push_back(SummaryColumn("failed", Colour::ResultError) + .addRow(totals.testCases.failed) + .addRow(totals.assertions.failed)); + columns.push_back(SummaryColumn("failed as expected", Colour::ResultExpectedFailure) + .addRow(totals.testCases.failedButOk) + .addRow(totals.assertions.failedButOk)); + + printSummaryRow("test cases", columns, 0); + printSummaryRow("assertions", columns, 1); + } +} +void ConsoleReporter::printSummaryRow(std::string const& label, std::vector const& cols, std::size_t row) { + for (auto col : cols) { + std::string value = col.rows[row]; + if (col.label.empty()) { + stream << label << ": "; + if (value != "0") + stream << value; + else + stream << Colour(Colour::Warning) << "- none -"; + } else if (value != "0") { + stream << Colour(Colour::LightGrey) << " | "; + stream << Colour(col.colour) + << value << ' ' << col.label; + } + } + stream << '\n'; +} + +void ConsoleReporter::printTotalsDivider(Totals const& totals) { + if (totals.testCases.total() > 0) { + std::size_t failedRatio = makeRatio(totals.testCases.failed, totals.testCases.total()); + std::size_t failedButOkRatio = makeRatio(totals.testCases.failedButOk, totals.testCases.total()); + std::size_t passedRatio = makeRatio(totals.testCases.passed, totals.testCases.total()); + while (failedRatio + failedButOkRatio + passedRatio < CATCH_CONFIG_CONSOLE_WIDTH - 1) + findMax(failedRatio, failedButOkRatio, passedRatio)++; + while (failedRatio + failedButOkRatio + passedRatio > CATCH_CONFIG_CONSOLE_WIDTH - 1) + findMax(failedRatio, failedButOkRatio, passedRatio)--; + + stream << Colour(Colour::Error) << std::string(failedRatio, '='); + stream << Colour(Colour::ResultExpectedFailure) << std::string(failedButOkRatio, '='); + if (totals.testCases.allPassed()) + stream << Colour(Colour::ResultSuccess) << std::string(passedRatio, '='); + else + stream << Colour(Colour::Success) << std::string(passedRatio, '='); + } else { + stream << Colour(Colour::Warning) << std::string(CATCH_CONFIG_CONSOLE_WIDTH - 1, '='); + } + stream << '\n'; +} +void ConsoleReporter::printSummaryDivider() { + stream << getLineOfChars<'-'>() << '\n'; +} + +CATCH_REGISTER_REPORTER("console", ConsoleReporter) + +} // end namespace Catch + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif +// end catch_reporter_console.cpp +// start catch_reporter_junit.cpp + +#include +#include +#include +#include + +namespace Catch { + + namespace { + std::string getCurrentTimestamp() { + // Beware, this is not reentrant because of backward compatibility issues + // Also, UTC only, again because of backward compatibility (%z is C++11) + time_t rawtime; + std::time(&rawtime); + auto const timeStampSize = sizeof("2017-01-16T17:06:45Z"); + +#ifdef _MSC_VER + std::tm timeInfo = {}; + gmtime_s(&timeInfo, &rawtime); +#else + std::tm* timeInfo; + timeInfo = std::gmtime(&rawtime); +#endif + + char timeStamp[timeStampSize]; + const char * const fmt = "%Y-%m-%dT%H:%M:%SZ"; + +#ifdef _MSC_VER + std::strftime(timeStamp, timeStampSize, fmt, &timeInfo); +#else + std::strftime(timeStamp, timeStampSize, fmt, timeInfo); +#endif + return std::string(timeStamp); + } + + std::string fileNameTag(const std::vector &tags) { + auto it = std::find_if(begin(tags), + end(tags), + [] (std::string const& tag) {return tag.front() == '#'; }); + if (it != tags.end()) + return it->substr(1); + return std::string(); + } + } // anonymous namespace + + JunitReporter::JunitReporter( ReporterConfig const& _config ) + : CumulativeReporterBase( _config ), + xml( _config.stream() ) + { + m_reporterPrefs.shouldRedirectStdOut = true; + m_reporterPrefs.shouldReportAllAssertions = true; + } + + JunitReporter::~JunitReporter() {} + + std::string JunitReporter::getDescription() { + return "Reports test results in an XML format that looks like Ant's junitreport target"; + } + + void JunitReporter::noMatchingTestCases( std::string const& /*spec*/ ) {} + + void JunitReporter::testRunStarting( TestRunInfo const& runInfo ) { + CumulativeReporterBase::testRunStarting( runInfo ); + xml.startElement( "testsuites" ); + } + + void JunitReporter::testGroupStarting( GroupInfo const& groupInfo ) { + suiteTimer.start(); + stdOutForSuite.clear(); + stdErrForSuite.clear(); + unexpectedExceptions = 0; + CumulativeReporterBase::testGroupStarting( groupInfo ); + } + + void JunitReporter::testCaseStarting( TestCaseInfo const& testCaseInfo ) { + m_okToFail = testCaseInfo.okToFail(); + } + + bool JunitReporter::assertionEnded( AssertionStats const& assertionStats ) { + if( assertionStats.assertionResult.getResultType() == ResultWas::ThrewException && !m_okToFail ) + unexpectedExceptions++; + return CumulativeReporterBase::assertionEnded( assertionStats ); + } + + void JunitReporter::testCaseEnded( TestCaseStats const& testCaseStats ) { + stdOutForSuite += testCaseStats.stdOut; + stdErrForSuite += testCaseStats.stdErr; + CumulativeReporterBase::testCaseEnded( testCaseStats ); + } + + void JunitReporter::testGroupEnded( TestGroupStats const& testGroupStats ) { + double suiteTime = suiteTimer.getElapsedSeconds(); + CumulativeReporterBase::testGroupEnded( testGroupStats ); + writeGroup( *m_testGroups.back(), suiteTime ); + } + + void JunitReporter::testRunEndedCumulative() { + xml.endElement(); + } + + void JunitReporter::writeGroup( TestGroupNode const& groupNode, double suiteTime ) { + XmlWriter::ScopedElement e = xml.scopedElement( "testsuite" ); + TestGroupStats const& stats = groupNode.value; + xml.writeAttribute( "name", stats.groupInfo.name ); + xml.writeAttribute( "errors", unexpectedExceptions ); + xml.writeAttribute( "failures", stats.totals.assertions.failed-unexpectedExceptions ); + xml.writeAttribute( "tests", stats.totals.assertions.total() ); + xml.writeAttribute( "hostname", "tbd" ); // !TBD + if( m_config->showDurations() == ShowDurations::Never ) + xml.writeAttribute( "time", "" ); + else + xml.writeAttribute( "time", suiteTime ); + xml.writeAttribute( "timestamp", getCurrentTimestamp() ); + + // Write test cases + for( auto const& child : groupNode.children ) + writeTestCase( *child ); + + xml.scopedElement( "system-out" ).writeText( trim( stdOutForSuite ), false ); + xml.scopedElement( "system-err" ).writeText( trim( stdErrForSuite ), false ); + } + + void JunitReporter::writeTestCase( TestCaseNode const& testCaseNode ) { + TestCaseStats const& stats = testCaseNode.value; + + // All test cases have exactly one section - which represents the + // test case itself. That section may have 0-n nested sections + assert( testCaseNode.children.size() == 1 ); + SectionNode const& rootSection = *testCaseNode.children.front(); + + std::string className = stats.testInfo.className; + + if( className.empty() ) { + className = fileNameTag(stats.testInfo.tags); + if ( className.empty() ) + className = "global"; + } + + if ( !m_config->name().empty() ) + className = m_config->name() + "." + className; + + writeSection( className, "", rootSection ); + } + + void JunitReporter::writeSection( std::string const& className, + std::string const& rootName, + SectionNode const& sectionNode ) { + std::string name = trim( sectionNode.stats.sectionInfo.name ); + if( !rootName.empty() ) + name = rootName + '/' + name; + + if( !sectionNode.assertions.empty() || + !sectionNode.stdOut.empty() || + !sectionNode.stdErr.empty() ) { + XmlWriter::ScopedElement e = xml.scopedElement( "testcase" ); + if( className.empty() ) { + xml.writeAttribute( "classname", name ); + xml.writeAttribute( "name", "root" ); + } + else { + xml.writeAttribute( "classname", className ); + xml.writeAttribute( "name", name ); + } + xml.writeAttribute( "time", ::Catch::Detail::stringify( sectionNode.stats.durationInSeconds ) ); + + writeAssertions( sectionNode ); + + if( !sectionNode.stdOut.empty() ) + xml.scopedElement( "system-out" ).writeText( trim( sectionNode.stdOut ), false ); + if( !sectionNode.stdErr.empty() ) + xml.scopedElement( "system-err" ).writeText( trim( sectionNode.stdErr ), false ); + } + for( auto const& childNode : sectionNode.childSections ) + if( className.empty() ) + writeSection( name, "", *childNode ); + else + writeSection( className, name, *childNode ); + } + + void JunitReporter::writeAssertions( SectionNode const& sectionNode ) { + for( auto const& assertion : sectionNode.assertions ) + writeAssertion( assertion ); + } + + void JunitReporter::writeAssertion( AssertionStats const& stats ) { + AssertionResult const& result = stats.assertionResult; + if( !result.isOk() ) { + std::string elementName; + switch( result.getResultType() ) { + case ResultWas::ThrewException: + case ResultWas::FatalErrorCondition: + elementName = "error"; + break; + case ResultWas::ExplicitFailure: + elementName = "failure"; + break; + case ResultWas::ExpressionFailed: + elementName = "failure"; + break; + case ResultWas::DidntThrowException: + elementName = "failure"; + break; + + // We should never see these here: + case ResultWas::Info: + case ResultWas::Warning: + case ResultWas::Ok: + case ResultWas::Unknown: + case ResultWas::FailureBit: + case ResultWas::Exception: + elementName = "internalError"; + break; + } + + XmlWriter::ScopedElement e = xml.scopedElement( elementName ); + + xml.writeAttribute( "message", result.getExpandedExpression() ); + xml.writeAttribute( "type", result.getTestMacroName() ); + + ReusableStringStream rss; + if( !result.getMessage().empty() ) + rss << result.getMessage() << '\n'; + for( auto const& msg : stats.infoMessages ) + if( msg.type == ResultWas::Info ) + rss << msg.message << '\n'; + + rss << "at " << result.getSourceInfo(); + xml.writeText( rss.str(), false ); + } + } + + CATCH_REGISTER_REPORTER( "junit", JunitReporter ) + +} // end namespace Catch +// end catch_reporter_junit.cpp +// start catch_reporter_listening.cpp + +#include + +namespace Catch { + + ListeningReporter::ListeningReporter() { + // We will assume that listeners will always want all assertions + m_preferences.shouldReportAllAssertions = true; + } + + void ListeningReporter::addListener( IStreamingReporterPtr&& listener ) { + m_listeners.push_back( std::move( listener ) ); + } + + void ListeningReporter::addReporter(IStreamingReporterPtr&& reporter) { + assert(!m_reporter && "Listening reporter can wrap only 1 real reporter"); + m_reporter = std::move( reporter ); + m_preferences.shouldRedirectStdOut = m_reporter->getPreferences().shouldRedirectStdOut; + } + + ReporterPreferences ListeningReporter::getPreferences() const { + return m_preferences; + } + + std::set ListeningReporter::getSupportedVerbosities() { + return std::set{ }; + } + + void ListeningReporter::noMatchingTestCases( std::string const& spec ) { + for ( auto const& listener : m_listeners ) { + listener->noMatchingTestCases( spec ); + } + m_reporter->noMatchingTestCases( spec ); + } + + void ListeningReporter::benchmarkStarting( BenchmarkInfo const& benchmarkInfo ) { + for ( auto const& listener : m_listeners ) { + listener->benchmarkStarting( benchmarkInfo ); + } + m_reporter->benchmarkStarting( benchmarkInfo ); + } + void ListeningReporter::benchmarkEnded( BenchmarkStats const& benchmarkStats ) { + for ( auto const& listener : m_listeners ) { + listener->benchmarkEnded( benchmarkStats ); + } + m_reporter->benchmarkEnded( benchmarkStats ); + } + + void ListeningReporter::testRunStarting( TestRunInfo const& testRunInfo ) { + for ( auto const& listener : m_listeners ) { + listener->testRunStarting( testRunInfo ); + } + m_reporter->testRunStarting( testRunInfo ); + } + + void ListeningReporter::testGroupStarting( GroupInfo const& groupInfo ) { + for ( auto const& listener : m_listeners ) { + listener->testGroupStarting( groupInfo ); + } + m_reporter->testGroupStarting( groupInfo ); + } + + void ListeningReporter::testCaseStarting( TestCaseInfo const& testInfo ) { + for ( auto const& listener : m_listeners ) { + listener->testCaseStarting( testInfo ); + } + m_reporter->testCaseStarting( testInfo ); + } + + void ListeningReporter::sectionStarting( SectionInfo const& sectionInfo ) { + for ( auto const& listener : m_listeners ) { + listener->sectionStarting( sectionInfo ); + } + m_reporter->sectionStarting( sectionInfo ); + } + + void ListeningReporter::assertionStarting( AssertionInfo const& assertionInfo ) { + for ( auto const& listener : m_listeners ) { + listener->assertionStarting( assertionInfo ); + } + m_reporter->assertionStarting( assertionInfo ); + } + + // The return value indicates if the messages buffer should be cleared: + bool ListeningReporter::assertionEnded( AssertionStats const& assertionStats ) { + for( auto const& listener : m_listeners ) { + static_cast( listener->assertionEnded( assertionStats ) ); + } + return m_reporter->assertionEnded( assertionStats ); + } + + void ListeningReporter::sectionEnded( SectionStats const& sectionStats ) { + for ( auto const& listener : m_listeners ) { + listener->sectionEnded( sectionStats ); + } + m_reporter->sectionEnded( sectionStats ); + } + + void ListeningReporter::testCaseEnded( TestCaseStats const& testCaseStats ) { + for ( auto const& listener : m_listeners ) { + listener->testCaseEnded( testCaseStats ); + } + m_reporter->testCaseEnded( testCaseStats ); + } + + void ListeningReporter::testGroupEnded( TestGroupStats const& testGroupStats ) { + for ( auto const& listener : m_listeners ) { + listener->testGroupEnded( testGroupStats ); + } + m_reporter->testGroupEnded( testGroupStats ); + } + + void ListeningReporter::testRunEnded( TestRunStats const& testRunStats ) { + for ( auto const& listener : m_listeners ) { + listener->testRunEnded( testRunStats ); + } + m_reporter->testRunEnded( testRunStats ); + } + + void ListeningReporter::skipTest( TestCaseInfo const& testInfo ) { + for ( auto const& listener : m_listeners ) { + listener->skipTest( testInfo ); + } + m_reporter->skipTest( testInfo ); + } + + bool ListeningReporter::isMulti() const { + return true; + } + +} // end namespace Catch +// end catch_reporter_listening.cpp +// start catch_reporter_xml.cpp + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable:4061) // Not all labels are EXPLICITLY handled in switch + // Note that 4062 (not all labels are handled + // and default is missing) is enabled +#endif + +namespace Catch { + XmlReporter::XmlReporter( ReporterConfig const& _config ) + : StreamingReporterBase( _config ), + m_xml(_config.stream()) + { + m_reporterPrefs.shouldRedirectStdOut = true; + m_reporterPrefs.shouldReportAllAssertions = true; + } + + XmlReporter::~XmlReporter() = default; + + std::string XmlReporter::getDescription() { + return "Reports test results as an XML document"; + } + + std::string XmlReporter::getStylesheetRef() const { + return std::string(); + } + + void XmlReporter::writeSourceInfo( SourceLineInfo const& sourceInfo ) { + m_xml + .writeAttribute( "filename", sourceInfo.file ) + .writeAttribute( "line", sourceInfo.line ); + } + + void XmlReporter::noMatchingTestCases( std::string const& s ) { + StreamingReporterBase::noMatchingTestCases( s ); + } + + void XmlReporter::testRunStarting( TestRunInfo const& testInfo ) { + StreamingReporterBase::testRunStarting( testInfo ); + std::string stylesheetRef = getStylesheetRef(); + if( !stylesheetRef.empty() ) + m_xml.writeStylesheetRef( stylesheetRef ); + m_xml.startElement( "Catch" ); + if( !m_config->name().empty() ) + m_xml.writeAttribute( "name", m_config->name() ); + } + + void XmlReporter::testGroupStarting( GroupInfo const& groupInfo ) { + StreamingReporterBase::testGroupStarting( groupInfo ); + m_xml.startElement( "Group" ) + .writeAttribute( "name", groupInfo.name ); + } + + void XmlReporter::testCaseStarting( TestCaseInfo const& testInfo ) { + StreamingReporterBase::testCaseStarting(testInfo); + m_xml.startElement( "TestCase" ) + .writeAttribute( "name", trim( testInfo.name ) ) + .writeAttribute( "description", testInfo.description ) + .writeAttribute( "tags", testInfo.tagsAsString() ); + + writeSourceInfo( testInfo.lineInfo ); + + if ( m_config->showDurations() == ShowDurations::Always ) + m_testCaseTimer.start(); + m_xml.ensureTagClosed(); + } + + void XmlReporter::sectionStarting( SectionInfo const& sectionInfo ) { + StreamingReporterBase::sectionStarting( sectionInfo ); + if( m_sectionDepth++ > 0 ) { + m_xml.startElement( "Section" ) + .writeAttribute( "name", trim( sectionInfo.name ) ); + writeSourceInfo( sectionInfo.lineInfo ); + m_xml.ensureTagClosed(); + } + } + + void XmlReporter::assertionStarting( AssertionInfo const& ) { } + + bool XmlReporter::assertionEnded( AssertionStats const& assertionStats ) { + + AssertionResult const& result = assertionStats.assertionResult; + + bool includeResults = m_config->includeSuccessfulResults() || !result.isOk(); + + if( includeResults || result.getResultType() == ResultWas::Warning ) { + // Print any info messages in tags. + for( auto const& msg : assertionStats.infoMessages ) { + if( msg.type == ResultWas::Info && includeResults ) { + m_xml.scopedElement( "Info" ) + .writeText( msg.message ); + } else if ( msg.type == ResultWas::Warning ) { + m_xml.scopedElement( "Warning" ) + .writeText( msg.message ); + } + } + } + + // Drop out if result was successful but we're not printing them. + if( !includeResults && result.getResultType() != ResultWas::Warning ) + return true; + + // Print the expression if there is one. + if( result.hasExpression() ) { + m_xml.startElement( "Expression" ) + .writeAttribute( "success", result.succeeded() ) + .writeAttribute( "type", result.getTestMacroName() ); + + writeSourceInfo( result.getSourceInfo() ); + + m_xml.scopedElement( "Original" ) + .writeText( result.getExpression() ); + m_xml.scopedElement( "Expanded" ) + .writeText( result.getExpandedExpression() ); + } + + // And... Print a result applicable to each result type. + switch( result.getResultType() ) { + case ResultWas::ThrewException: + m_xml.startElement( "Exception" ); + writeSourceInfo( result.getSourceInfo() ); + m_xml.writeText( result.getMessage() ); + m_xml.endElement(); + break; + case ResultWas::FatalErrorCondition: + m_xml.startElement( "FatalErrorCondition" ); + writeSourceInfo( result.getSourceInfo() ); + m_xml.writeText( result.getMessage() ); + m_xml.endElement(); + break; + case ResultWas::Info: + m_xml.scopedElement( "Info" ) + .writeText( result.getMessage() ); + break; + case ResultWas::Warning: + // Warning will already have been written + break; + case ResultWas::ExplicitFailure: + m_xml.startElement( "Failure" ); + writeSourceInfo( result.getSourceInfo() ); + m_xml.writeText( result.getMessage() ); + m_xml.endElement(); + break; + default: + break; + } + + if( result.hasExpression() ) + m_xml.endElement(); + + return true; + } + + void XmlReporter::sectionEnded( SectionStats const& sectionStats ) { + StreamingReporterBase::sectionEnded( sectionStats ); + if( --m_sectionDepth > 0 ) { + XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResults" ); + e.writeAttribute( "successes", sectionStats.assertions.passed ); + e.writeAttribute( "failures", sectionStats.assertions.failed ); + e.writeAttribute( "expectedFailures", sectionStats.assertions.failedButOk ); + + if ( m_config->showDurations() == ShowDurations::Always ) + e.writeAttribute( "durationInSeconds", sectionStats.durationInSeconds ); + + m_xml.endElement(); + } + } + + void XmlReporter::testCaseEnded( TestCaseStats const& testCaseStats ) { + StreamingReporterBase::testCaseEnded( testCaseStats ); + XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResult" ); + e.writeAttribute( "success", testCaseStats.totals.assertions.allOk() ); + + if ( m_config->showDurations() == ShowDurations::Always ) + e.writeAttribute( "durationInSeconds", m_testCaseTimer.getElapsedSeconds() ); + + if( !testCaseStats.stdOut.empty() ) + m_xml.scopedElement( "StdOut" ).writeText( trim( testCaseStats.stdOut ), false ); + if( !testCaseStats.stdErr.empty() ) + m_xml.scopedElement( "StdErr" ).writeText( trim( testCaseStats.stdErr ), false ); + + m_xml.endElement(); + } + + void XmlReporter::testGroupEnded( TestGroupStats const& testGroupStats ) { + StreamingReporterBase::testGroupEnded( testGroupStats ); + // TODO: Check testGroupStats.aborting and act accordingly. + m_xml.scopedElement( "OverallResults" ) + .writeAttribute( "successes", testGroupStats.totals.assertions.passed ) + .writeAttribute( "failures", testGroupStats.totals.assertions.failed ) + .writeAttribute( "expectedFailures", testGroupStats.totals.assertions.failedButOk ); + m_xml.endElement(); + } + + void XmlReporter::testRunEnded( TestRunStats const& testRunStats ) { + StreamingReporterBase::testRunEnded( testRunStats ); + m_xml.scopedElement( "OverallResults" ) + .writeAttribute( "successes", testRunStats.totals.assertions.passed ) + .writeAttribute( "failures", testRunStats.totals.assertions.failed ) + .writeAttribute( "expectedFailures", testRunStats.totals.assertions.failedButOk ); + m_xml.endElement(); + } + + CATCH_REGISTER_REPORTER( "xml", XmlReporter ) + +} // end namespace Catch + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif +// end catch_reporter_xml.cpp + +namespace Catch { + LeakDetector leakDetector; +} + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + +// end catch_impl.hpp +#endif + +#ifdef CATCH_CONFIG_MAIN +// start catch_default_main.hpp + +#ifndef __OBJC__ + +#if defined(CATCH_CONFIG_WCHAR) && defined(WIN32) && defined(_UNICODE) && !defined(DO_NOT_USE_WMAIN) +// Standard C/C++ Win32 Unicode wmain entry point +extern "C" int wmain (int argc, wchar_t * argv[], wchar_t * []) { +#else +// Standard C/C++ main entry point +int main (int argc, char * argv[]) { +#endif + + return Catch::Session().run( argc, argv ); +} + +#else // __OBJC__ + +// Objective-C entry point +int main (int argc, char * const argv[]) { +#if !CATCH_ARC_ENABLED + NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init]; +#endif + + Catch::registerTestMethods(); + int result = Catch::Session().run( argc, (char**)argv ); + +#if !CATCH_ARC_ENABLED + [pool drain]; +#endif + + return result; +} + +#endif // __OBJC__ + +// end catch_default_main.hpp +#endif + +#if !defined(CATCH_CONFIG_IMPL_ONLY) + +#ifdef CLARA_CONFIG_MAIN_NOT_DEFINED +# undef CLARA_CONFIG_MAIN +#endif + +#if !defined(CATCH_CONFIG_DISABLE) +////// +// If this config identifier is defined then all CATCH macros are prefixed with CATCH_ +#ifdef CATCH_CONFIG_PREFIX_ALL + +#define CATCH_REQUIRE( ... ) INTERNAL_CATCH_TEST( "CATCH_REQUIRE", Catch::ResultDisposition::Normal, __VA_ARGS__ ) +#define CATCH_REQUIRE_FALSE( ... ) INTERNAL_CATCH_TEST( "CATCH_REQUIRE_FALSE", Catch::ResultDisposition::Normal | Catch::ResultDisposition::FalseTest, __VA_ARGS__ ) + +#define CATCH_REQUIRE_THROWS( ... ) INTERNAL_CATCH_THROWS( "CATCH_REQUIRE_THROWS", Catch::ResultDisposition::Normal, "", __VA_ARGS__ ) +#define CATCH_REQUIRE_THROWS_AS( expr, exceptionType ) INTERNAL_CATCH_THROWS_AS( "CATCH_REQUIRE_THROWS_AS", exceptionType, Catch::ResultDisposition::Normal, expr ) +#define CATCH_REQUIRE_THROWS_WITH( expr, matcher ) INTERNAL_CATCH_THROWS_STR_MATCHES( "CATCH_REQUIRE_THROWS_WITH", Catch::ResultDisposition::Normal, matcher, expr ) +#if !defined(CATCH_CONFIG_DISABLE_MATCHERS) +#define CATCH_REQUIRE_THROWS_MATCHES( expr, exceptionType, matcher ) INTERNAL_CATCH_THROWS_MATCHES( "CATCH_REQUIRE_THROWS_MATCHES", exceptionType, Catch::ResultDisposition::Normal, matcher, expr ) +#endif// CATCH_CONFIG_DISABLE_MATCHERS +#define CATCH_REQUIRE_NOTHROW( ... ) INTERNAL_CATCH_NO_THROW( "CATCH_REQUIRE_NOTHROW", Catch::ResultDisposition::Normal, __VA_ARGS__ ) + +#define CATCH_CHECK( ... ) INTERNAL_CATCH_TEST( "CATCH_CHECK", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) +#define CATCH_CHECK_FALSE( ... ) INTERNAL_CATCH_TEST( "CATCH_CHECK_FALSE", Catch::ResultDisposition::ContinueOnFailure | Catch::ResultDisposition::FalseTest, __VA_ARGS__ ) +#define CATCH_CHECKED_IF( ... ) INTERNAL_CATCH_IF( "CATCH_CHECKED_IF", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) +#define CATCH_CHECKED_ELSE( ... ) INTERNAL_CATCH_ELSE( "CATCH_CHECKED_ELSE", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) +#define CATCH_CHECK_NOFAIL( ... ) INTERNAL_CATCH_TEST( "CATCH_CHECK_NOFAIL", Catch::ResultDisposition::ContinueOnFailure | Catch::ResultDisposition::SuppressFail, __VA_ARGS__ ) + +#define CATCH_CHECK_THROWS( ... ) INTERNAL_CATCH_THROWS( "CATCH_CHECK_THROWS", Catch::ResultDisposition::ContinueOnFailure, "", __VA_ARGS__ ) +#define CATCH_CHECK_THROWS_AS( expr, exceptionType ) INTERNAL_CATCH_THROWS_AS( "CATCH_CHECK_THROWS_AS", exceptionType, Catch::ResultDisposition::ContinueOnFailure, expr ) +#define CATCH_CHECK_THROWS_WITH( expr, matcher ) INTERNAL_CATCH_THROWS_STR_MATCHES( "CATCH_CHECK_THROWS_WITH", Catch::ResultDisposition::ContinueOnFailure, matcher, expr ) +#if !defined(CATCH_CONFIG_DISABLE_MATCHERS) +#define CATCH_CHECK_THROWS_MATCHES( expr, exceptionType, matcher ) INTERNAL_CATCH_THROWS_MATCHES( "CATCH_CHECK_THROWS_MATCHES", exceptionType, Catch::ResultDisposition::ContinueOnFailure, matcher, expr ) +#endif // CATCH_CONFIG_DISABLE_MATCHERS +#define CATCH_CHECK_NOTHROW( ... ) INTERNAL_CATCH_NO_THROW( "CATCH_CHECK_NOTHROW", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) + +#if !defined(CATCH_CONFIG_DISABLE_MATCHERS) +#define CATCH_CHECK_THAT( arg, matcher ) INTERNAL_CHECK_THAT( "CATCH_CHECK_THAT", matcher, Catch::ResultDisposition::ContinueOnFailure, arg ) + +#define CATCH_REQUIRE_THAT( arg, matcher ) INTERNAL_CHECK_THAT( "CATCH_REQUIRE_THAT", matcher, Catch::ResultDisposition::Normal, arg ) +#endif // CATCH_CONFIG_DISABLE_MATCHERS + +#define CATCH_INFO( msg ) INTERNAL_CATCH_INFO( "CATCH_INFO", msg ) +#define CATCH_WARN( msg ) INTERNAL_CATCH_MSG( "CATCH_WARN", Catch::ResultWas::Warning, Catch::ResultDisposition::ContinueOnFailure, msg ) +#define CATCH_CAPTURE( msg ) INTERNAL_CATCH_INFO( "CATCH_CAPTURE", #msg " := " << ::Catch::Detail::stringify(msg) ) + +#define CATCH_TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE( __VA_ARGS__ ) +#define CATCH_TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TEST_CASE_METHOD( className, __VA_ARGS__ ) +#define CATCH_METHOD_AS_TEST_CASE( method, ... ) INTERNAL_CATCH_METHOD_AS_TEST_CASE( method, __VA_ARGS__ ) +#define CATCH_REGISTER_TEST_CASE( Function, ... ) INTERNAL_CATCH_REGISTER_TESTCASE( Function, __VA_ARGS__ ) +#define CATCH_SECTION( ... ) INTERNAL_CATCH_SECTION( __VA_ARGS__ ) +#define CATCH_DYNAMIC_SECTION( ... ) INTERNAL_CATCH_DYNAMIC_SECTION( __VA_ARGS__ ) +#define CATCH_FAIL( ... ) INTERNAL_CATCH_MSG( "CATCH_FAIL", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, __VA_ARGS__ ) +#define CATCH_FAIL_CHECK( ... ) INTERNAL_CATCH_MSG( "CATCH_FAIL_CHECK", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) +#define CATCH_SUCCEED( ... ) INTERNAL_CATCH_MSG( "CATCH_SUCCEED", Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) + +#define CATCH_ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE() + +// "BDD-style" convenience wrappers +#define CATCH_SCENARIO( ... ) CATCH_TEST_CASE( "Scenario: " __VA_ARGS__ ) +#define CATCH_SCENARIO_METHOD( className, ... ) INTERNAL_CATCH_TEST_CASE_METHOD( className, "Scenario: " __VA_ARGS__ ) +#define CATCH_GIVEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " Given: " << desc ) +#define CATCH_WHEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " When: " << desc ) +#define CATCH_AND_WHEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( "And when: " << desc ) +#define CATCH_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " Then: " << desc ) +#define CATCH_AND_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " And: " << desc ) + +// If CATCH_CONFIG_PREFIX_ALL is not defined then the CATCH_ prefix is not required +#else + +#define REQUIRE( ... ) INTERNAL_CATCH_TEST( "REQUIRE", Catch::ResultDisposition::Normal, __VA_ARGS__ ) +#define REQUIRE_FALSE( ... ) INTERNAL_CATCH_TEST( "REQUIRE_FALSE", Catch::ResultDisposition::Normal | Catch::ResultDisposition::FalseTest, __VA_ARGS__ ) + +#define REQUIRE_THROWS( ... ) INTERNAL_CATCH_THROWS( "REQUIRE_THROWS", Catch::ResultDisposition::Normal, __VA_ARGS__ ) +#define REQUIRE_THROWS_AS( expr, exceptionType ) INTERNAL_CATCH_THROWS_AS( "REQUIRE_THROWS_AS", exceptionType, Catch::ResultDisposition::Normal, expr ) +#define REQUIRE_THROWS_WITH( expr, matcher ) INTERNAL_CATCH_THROWS_STR_MATCHES( "REQUIRE_THROWS_WITH", Catch::ResultDisposition::Normal, matcher, expr ) +#if !defined(CATCH_CONFIG_DISABLE_MATCHERS) +#define REQUIRE_THROWS_MATCHES( expr, exceptionType, matcher ) INTERNAL_CATCH_THROWS_MATCHES( "REQUIRE_THROWS_MATCHES", exceptionType, Catch::ResultDisposition::Normal, matcher, expr ) +#endif // CATCH_CONFIG_DISABLE_MATCHERS +#define REQUIRE_NOTHROW( ... ) INTERNAL_CATCH_NO_THROW( "REQUIRE_NOTHROW", Catch::ResultDisposition::Normal, __VA_ARGS__ ) + +#define CHECK( ... ) INTERNAL_CATCH_TEST( "CHECK", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) +#define CHECK_FALSE( ... ) INTERNAL_CATCH_TEST( "CHECK_FALSE", Catch::ResultDisposition::ContinueOnFailure | Catch::ResultDisposition::FalseTest, __VA_ARGS__ ) +#define CHECKED_IF( ... ) INTERNAL_CATCH_IF( "CHECKED_IF", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) +#define CHECKED_ELSE( ... ) INTERNAL_CATCH_ELSE( "CHECKED_ELSE", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) +#define CHECK_NOFAIL( ... ) INTERNAL_CATCH_TEST( "CHECK_NOFAIL", Catch::ResultDisposition::ContinueOnFailure | Catch::ResultDisposition::SuppressFail, __VA_ARGS__ ) + +#define CHECK_THROWS( ... ) INTERNAL_CATCH_THROWS( "CHECK_THROWS", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) +#define CHECK_THROWS_AS( expr, exceptionType ) INTERNAL_CATCH_THROWS_AS( "CHECK_THROWS_AS", exceptionType, Catch::ResultDisposition::ContinueOnFailure, expr ) +#define CHECK_THROWS_WITH( expr, matcher ) INTERNAL_CATCH_THROWS_STR_MATCHES( "CHECK_THROWS_WITH", Catch::ResultDisposition::ContinueOnFailure, matcher, expr ) +#if !defined(CATCH_CONFIG_DISABLE_MATCHERS) +#define CHECK_THROWS_MATCHES( expr, exceptionType, matcher ) INTERNAL_CATCH_THROWS_MATCHES( "CHECK_THROWS_MATCHES", exceptionType, Catch::ResultDisposition::ContinueOnFailure, matcher, expr ) +#endif // CATCH_CONFIG_DISABLE_MATCHERS +#define CHECK_NOTHROW( ... ) INTERNAL_CATCH_NO_THROW( "CHECK_NOTHROW", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) + +#if !defined(CATCH_CONFIG_DISABLE_MATCHERS) +#define CHECK_THAT( arg, matcher ) INTERNAL_CHECK_THAT( "CHECK_THAT", matcher, Catch::ResultDisposition::ContinueOnFailure, arg ) + +#define REQUIRE_THAT( arg, matcher ) INTERNAL_CHECK_THAT( "REQUIRE_THAT", matcher, Catch::ResultDisposition::Normal, arg ) +#endif // CATCH_CONFIG_DISABLE_MATCHERS + +#define INFO( msg ) INTERNAL_CATCH_INFO( "INFO", msg ) +#define WARN( msg ) INTERNAL_CATCH_MSG( "WARN", Catch::ResultWas::Warning, Catch::ResultDisposition::ContinueOnFailure, msg ) +#define CAPTURE( msg ) INTERNAL_CATCH_INFO( "CAPTURE", #msg " := " << ::Catch::Detail::stringify(msg) ) + +#define TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE( __VA_ARGS__ ) +#define TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TEST_CASE_METHOD( className, __VA_ARGS__ ) +#define METHOD_AS_TEST_CASE( method, ... ) INTERNAL_CATCH_METHOD_AS_TEST_CASE( method, __VA_ARGS__ ) +#define REGISTER_TEST_CASE( Function, ... ) INTERNAL_CATCH_REGISTER_TESTCASE( Function, __VA_ARGS__ ) +#define SECTION( ... ) INTERNAL_CATCH_SECTION( __VA_ARGS__ ) +#define DYNAMIC_SECTION( ... ) INTERNAL_CATCH_DYNAMIC_SECTION( __VA_ARGS__ ) +#define FAIL( ... ) INTERNAL_CATCH_MSG( "FAIL", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, __VA_ARGS__ ) +#define FAIL_CHECK( ... ) INTERNAL_CATCH_MSG( "FAIL_CHECK", Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) +#define SUCCEED( ... ) INTERNAL_CATCH_MSG( "SUCCEED", Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) +#define ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE() + +#endif + +#define CATCH_TRANSLATE_EXCEPTION( signature ) INTERNAL_CATCH_TRANSLATE_EXCEPTION( signature ) + +// "BDD-style" convenience wrappers +#define SCENARIO( ... ) TEST_CASE( "Scenario: " __VA_ARGS__ ) +#define SCENARIO_METHOD( className, ... ) INTERNAL_CATCH_TEST_CASE_METHOD( className, "Scenario: " __VA_ARGS__ ) + +#define GIVEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " Given: " << desc ) +#define WHEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " When: " << desc ) +#define AND_WHEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( "And when: " << desc ) +#define THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " Then: " << desc ) +#define AND_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " And: " << desc ) + +using Catch::Detail::Approx; + +#else // CATCH_CONFIG_DISABLE + +////// +// If this config identifier is defined then all CATCH macros are prefixed with CATCH_ +#ifdef CATCH_CONFIG_PREFIX_ALL + +#define CATCH_REQUIRE( ... ) (void)(0) +#define CATCH_REQUIRE_FALSE( ... ) (void)(0) + +#define CATCH_REQUIRE_THROWS( ... ) (void)(0) +#define CATCH_REQUIRE_THROWS_AS( expr, exceptionType ) (void)(0) +#define CATCH_REQUIRE_THROWS_WITH( expr, matcher ) (void)(0) +#if !defined(CATCH_CONFIG_DISABLE_MATCHERS) +#define CATCH_REQUIRE_THROWS_MATCHES( expr, exceptionType, matcher ) (void)(0) +#endif// CATCH_CONFIG_DISABLE_MATCHERS +#define CATCH_REQUIRE_NOTHROW( ... ) (void)(0) + +#define CATCH_CHECK( ... ) (void)(0) +#define CATCH_CHECK_FALSE( ... ) (void)(0) +#define CATCH_CHECKED_IF( ... ) if (__VA_ARGS__) +#define CATCH_CHECKED_ELSE( ... ) if (!(__VA_ARGS__)) +#define CATCH_CHECK_NOFAIL( ... ) (void)(0) + +#define CATCH_CHECK_THROWS( ... ) (void)(0) +#define CATCH_CHECK_THROWS_AS( expr, exceptionType ) (void)(0) +#define CATCH_CHECK_THROWS_WITH( expr, matcher ) (void)(0) +#if !defined(CATCH_CONFIG_DISABLE_MATCHERS) +#define CATCH_CHECK_THROWS_MATCHES( expr, exceptionType, matcher ) (void)(0) +#endif // CATCH_CONFIG_DISABLE_MATCHERS +#define CATCH_CHECK_NOTHROW( ... ) (void)(0) + +#if !defined(CATCH_CONFIG_DISABLE_MATCHERS) +#define CATCH_CHECK_THAT( arg, matcher ) (void)(0) + +#define CATCH_REQUIRE_THAT( arg, matcher ) (void)(0) +#endif // CATCH_CONFIG_DISABLE_MATCHERS + +#define CATCH_INFO( msg ) (void)(0) +#define CATCH_WARN( msg ) (void)(0) +#define CATCH_CAPTURE( msg ) (void)(0) + +#define CATCH_TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ )) +#define CATCH_TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ )) +#define CATCH_METHOD_AS_TEST_CASE( method, ... ) +#define CATCH_REGISTER_TEST_CASE( Function, ... ) (void)(0) +#define CATCH_SECTION( ... ) +#define CATCH_DYNAMIC_SECTION( ... ) +#define CATCH_FAIL( ... ) (void)(0) +#define CATCH_FAIL_CHECK( ... ) (void)(0) +#define CATCH_SUCCEED( ... ) (void)(0) + +#define CATCH_ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ )) + +// "BDD-style" convenience wrappers +#define CATCH_SCENARIO( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ )) +#define CATCH_SCENARIO_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_METHOD_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ), className ) +#define CATCH_GIVEN( desc ) +#define CATCH_WHEN( desc ) +#define CATCH_AND_WHEN( desc ) +#define CATCH_THEN( desc ) +#define CATCH_AND_THEN( desc ) + +// If CATCH_CONFIG_PREFIX_ALL is not defined then the CATCH_ prefix is not required +#else + +#define REQUIRE( ... ) (void)(0) +#define REQUIRE_FALSE( ... ) (void)(0) + +#define REQUIRE_THROWS( ... ) (void)(0) +#define REQUIRE_THROWS_AS( expr, exceptionType ) (void)(0) +#define REQUIRE_THROWS_WITH( expr, matcher ) (void)(0) +#if !defined(CATCH_CONFIG_DISABLE_MATCHERS) +#define REQUIRE_THROWS_MATCHES( expr, exceptionType, matcher ) (void)(0) +#endif // CATCH_CONFIG_DISABLE_MATCHERS +#define REQUIRE_NOTHROW( ... ) (void)(0) + +#define CHECK( ... ) (void)(0) +#define CHECK_FALSE( ... ) (void)(0) +#define CHECKED_IF( ... ) if (__VA_ARGS__) +#define CHECKED_ELSE( ... ) if (!(__VA_ARGS__)) +#define CHECK_NOFAIL( ... ) (void)(0) + +#define CHECK_THROWS( ... ) (void)(0) +#define CHECK_THROWS_AS( expr, exceptionType ) (void)(0) +#define CHECK_THROWS_WITH( expr, matcher ) (void)(0) +#if !defined(CATCH_CONFIG_DISABLE_MATCHERS) +#define CHECK_THROWS_MATCHES( expr, exceptionType, matcher ) (void)(0) +#endif // CATCH_CONFIG_DISABLE_MATCHERS +#define CHECK_NOTHROW( ... ) (void)(0) + +#if !defined(CATCH_CONFIG_DISABLE_MATCHERS) +#define CHECK_THAT( arg, matcher ) (void)(0) + +#define REQUIRE_THAT( arg, matcher ) (void)(0) +#endif // CATCH_CONFIG_DISABLE_MATCHERS + +#define INFO( msg ) (void)(0) +#define WARN( msg ) (void)(0) +#define CAPTURE( msg ) (void)(0) + +#define TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ )) +#define TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ )) +#define METHOD_AS_TEST_CASE( method, ... ) +#define REGISTER_TEST_CASE( Function, ... ) (void)(0) +#define SECTION( ... ) +#define DYNAMIC_SECTION( ... ) +#define FAIL( ... ) (void)(0) +#define FAIL_CHECK( ... ) (void)(0) +#define SUCCEED( ... ) (void)(0) +#define ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ )) + +#endif + +#define CATCH_TRANSLATE_EXCEPTION( signature ) INTERNAL_CATCH_TRANSLATE_EXCEPTION_NO_REG( INTERNAL_CATCH_UNIQUE_NAME( catch_internal_ExceptionTranslator ), signature ) + +// "BDD-style" convenience wrappers +#define SCENARIO( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ) ) +#define SCENARIO_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_METHOD_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ), className ) + +#define GIVEN( desc ) +#define WHEN( desc ) +#define AND_WHEN( desc ) +#define THEN( desc ) +#define AND_THEN( desc ) + +using Catch::Detail::Approx; + +#endif + +#endif // ! CATCH_CONFIG_IMPL_ONLY + +// start catch_reenable_warnings.h + + +#ifdef __clang__ +# ifdef __ICC // icpc defines the __clang__ macro +# pragma warning(pop) +# else +# pragma clang diagnostic pop +# endif +#elif defined __GNUC__ +# pragma GCC diagnostic pop +#endif + +// end catch_reenable_warnings.h +// end catch.hpp +#endif // TWOBLUECUBES_SINGLE_INCLUDE_CATCH_HPP_INCLUDED + diff --git a/3rd/cpp-httplib/httplib.h b/3rd/cpp-httplib/httplib.h new file mode 100644 index 000000000..b89650d34 --- /dev/null +++ b/3rd/cpp-httplib/httplib.h @@ -0,0 +1,4026 @@ +// +// httplib.h +// +// Copyright (c) 2019 Yuji Hirose. All rights reserved. +// MIT License +// + +#ifndef CPPHTTPLIB_HTTPLIB_H +#define CPPHTTPLIB_HTTPLIB_H + +/* + * Configuration + */ + +#ifndef CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND +#define CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND 5 +#endif + +#ifndef CPPHTTPLIB_KEEPALIVE_TIMEOUT_USECOND +#define CPPHTTPLIB_KEEPALIVE_TIMEOUT_USECOND 0 +#endif + +#ifndef CPPHTTPLIB_KEEPALIVE_MAX_COUNT +#define CPPHTTPLIB_KEEPALIVE_MAX_COUNT 5 +#endif + +#ifndef CPPHTTPLIB_READ_TIMEOUT_SECOND +#define CPPHTTPLIB_READ_TIMEOUT_SECOND 5 +#endif + +#ifndef CPPHTTPLIB_READ_TIMEOUT_USECOND +#define CPPHTTPLIB_READ_TIMEOUT_USECOND 0 +#endif + +#ifndef CPPHTTPLIB_REQUEST_URI_MAX_LENGTH +#define CPPHTTPLIB_REQUEST_URI_MAX_LENGTH 8192 +#endif + +#ifndef CPPHTTPLIB_REDIRECT_MAX_COUNT +#define CPPHTTPLIB_REDIRECT_MAX_COUNT 20 +#endif + +#ifndef CPPHTTPLIB_PAYLOAD_MAX_LENGTH +#define CPPHTTPLIB_PAYLOAD_MAX_LENGTH (std::numeric_limits::max)() +#endif + +#ifndef CPPHTTPLIB_RECV_BUFSIZ +#define CPPHTTPLIB_RECV_BUFSIZ size_t(4096u) +#endif + +#ifndef CPPHTTPLIB_THREAD_POOL_COUNT +#define CPPHTTPLIB_THREAD_POOL_COUNT 8 +#endif + +/* + * Headers + */ + +#ifdef _WIN32 +#ifndef _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_WARNINGS +#endif //_CRT_SECURE_NO_WARNINGS + +#ifndef _CRT_NONSTDC_NO_DEPRECATE +#define _CRT_NONSTDC_NO_DEPRECATE +#endif //_CRT_NONSTDC_NO_DEPRECATE + +#if defined(_MSC_VER) +#ifdef _WIN64 +typedef __int64 ssize_t; +#else +typedef int ssize_t; +#endif + +#if _MSC_VER < 1900 +#define snprintf _snprintf_s +#endif +#endif // _MSC_VER + +#ifndef S_ISREG +#define S_ISREG(m) (((m)&S_IFREG) == S_IFREG) +#endif // S_ISREG + +#ifndef S_ISDIR +#define S_ISDIR(m) (((m)&S_IFDIR) == S_IFDIR) +#endif // S_ISDIR + +#ifndef NOMINMAX +#define NOMINMAX +#endif // NOMINMAX + +#include +#include +#include + +#ifndef WSA_FLAG_NO_HANDLE_INHERIT +#define WSA_FLAG_NO_HANDLE_INHERIT 0x80 +#endif + +#ifdef _MSC_VER +#pragma comment(lib, "ws2_32.lib") +#endif + +#ifndef strcasecmp +#define strcasecmp _stricmp +#endif // strcasecmp + +typedef SOCKET socket_t; +#ifdef CPPHTTPLIB_USE_POLL +#define poll(fds, nfds, timeout) WSAPoll(fds, nfds, timeout) +#endif + +#else // not _WIN32 + +#include +#include +#include +#include +#ifdef CPPHTTPLIB_USE_POLL +#include +#endif +#include +#include +#include +#include +#include + +typedef int socket_t; +#define INVALID_SOCKET (-1) +#endif //_WIN32 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +#include +#include +#include + +// #if OPENSSL_VERSION_NUMBER < 0x1010100fL +// #error Sorry, OpenSSL versions prior to 1.1.1 are not supported +// #endif + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#include +inline const unsigned char *ASN1_STRING_get0_data(const ASN1_STRING *asn1) { + return M_ASN1_STRING_data(asn1); +} +#endif +#endif + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT +#include +#endif + +/* + * Declaration + */ +namespace httplib { + +namespace detail { + +struct ci { + bool operator()(const std::string &s1, const std::string &s2) const { + return std::lexicographical_compare( + s1.begin(), s1.end(), s2.begin(), s2.end(), + [](char c1, char c2) { return ::tolower(c1) < ::tolower(c2); }); + } +}; + +} // namespace detail + +enum class HttpVersion { v1_0 = 0, v1_1 }; + +typedef std::multimap Headers; + +typedef std::multimap Params; +typedef std::smatch Match; + +typedef std::function DataSink; + +typedef std::function Done; + +typedef std::function + ContentProvider; + +typedef std::function + ContentProviderWithCloser; + +typedef std::function + ContentReceiver; + +typedef std::function ContentReader; + +typedef std::function Progress; + +struct Response; +typedef std::function ResponseHandler; + +struct MultipartFile { + std::string filename; + std::string content_type; + size_t offset = 0; + size_t length = 0; +}; +typedef std::multimap MultipartFiles; + +struct MultipartFormData { + std::string name; + std::string content; + std::string filename; + std::string content_type; +}; +typedef std::vector MultipartFormDataItems; + +typedef std::pair Range; +typedef std::vector Ranges; + +struct Request { + std::string method; + std::string path; + Headers headers; + std::string body; + + // for server + std::string version; + std::string target; + Params params; + MultipartFiles files; + Ranges ranges; + Match matches; + + // for client + size_t redirect_count = CPPHTTPLIB_REDIRECT_MAX_COUNT; + ResponseHandler response_handler; + ContentReceiver content_receiver; + Progress progress; + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + const SSL *ssl; +#endif + + bool has_header(const char *key) const; + std::string get_header_value(const char *key, size_t id = 0) const; + size_t get_header_value_count(const char *key) const; + void set_header(const char *key, const char *val); + void set_header(const char *key, const std::string &val); + + bool has_param(const char *key) const; + std::string get_param_value(const char *key, size_t id = 0) const; + size_t get_param_value_count(const char *key) const; + + bool has_file(const char *key) const; + MultipartFile get_file_value(const char *key) const; + + // private members... + size_t content_length; + ContentProvider content_provider; +}; + +struct Response { + std::string version; + int status; + Headers headers; + std::string body; + + bool has_header(const char *key) const; + std::string get_header_value(const char *key, size_t id = 0) const; + size_t get_header_value_count(const char *key) const; + void set_header(const char *key, const char *val); + void set_header(const char *key, const std::string &val); + + void set_redirect(const char *uri); + void set_content(const char *s, size_t n, const char *content_type); + void set_content(const std::string &s, const char *content_type); + + void set_content_provider( + size_t length, + std::function provider, + std::function resource_releaser = [] {}); + + void set_chunked_content_provider( + std::function provider, + std::function resource_releaser = [] {}); + + Response() : status(-1), content_length(0) {} + + ~Response() { + if (content_provider_resource_releaser) { + content_provider_resource_releaser(); + } + } + + // private members... + size_t content_length; + ContentProviderWithCloser content_provider; + std::function content_provider_resource_releaser; +}; + +class Stream { +public: + virtual ~Stream() {} + virtual int read(char *ptr, size_t size) = 0; + virtual int write(const char *ptr, size_t size1) = 0; + virtual int write(const char *ptr) = 0; + virtual int write(const std::string &s) = 0; + virtual std::string get_remote_addr() const = 0; + + template + int write_format(const char *fmt, const Args &... args); +}; + +class SocketStream : public Stream { +public: + SocketStream(socket_t sock, time_t read_timeout_sec, + time_t read_timeout_usec); + virtual ~SocketStream(); + + virtual int read(char *ptr, size_t size); + virtual int write(const char *ptr, size_t size); + virtual int write(const char *ptr); + virtual int write(const std::string &s); + virtual std::string get_remote_addr() const; + +private: + socket_t sock_; + time_t read_timeout_sec_; + time_t read_timeout_usec_; +}; + +class BufferStream : public Stream { +public: + BufferStream() {} + virtual ~BufferStream() {} + + virtual int read(char *ptr, size_t size); + virtual int write(const char *ptr, size_t size); + virtual int write(const char *ptr); + virtual int write(const std::string &s); + virtual std::string get_remote_addr() const; + + const std::string &get_buffer() const; + +private: + std::string buffer; +}; + +class TaskQueue { +public: + TaskQueue() {} + virtual ~TaskQueue() {} + virtual void enqueue(std::function fn) = 0; + virtual void shutdown() = 0; +}; + +#if CPPHTTPLIB_THREAD_POOL_COUNT > 0 +class ThreadPool : public TaskQueue { +public: + ThreadPool(size_t n) : shutdown_(false) { + while (n) { + auto t = std::make_shared(worker(*this)); + threads_.push_back(t); + n--; + } + } + + ThreadPool(const ThreadPool &) = delete; + virtual ~ThreadPool() {} + + virtual void enqueue(std::function fn) override { + std::unique_lock lock(mutex_); + jobs_.push_back(fn); + cond_.notify_one(); + } + + virtual void shutdown() override { + // Stop all worker threads... + { + std::unique_lock lock(mutex_); + shutdown_ = true; + } + + cond_.notify_all(); + + // Join... + for (auto t : threads_) { + t->join(); + } + } + +private: + struct worker { + worker(ThreadPool &pool) : pool_(pool) {} + + void operator()() { + for (;;) { + std::function fn; + { + std::unique_lock lock(pool_.mutex_); + + pool_.cond_.wait( + lock, [&] { return !pool_.jobs_.empty() || pool_.shutdown_; }); + + if (pool_.shutdown_ && pool_.jobs_.empty()) { break; } + + fn = pool_.jobs_.front(); + pool_.jobs_.pop_front(); + } + + assert(true == static_cast(fn)); + fn(); + } + } + + ThreadPool &pool_; + }; + friend struct worker; + + std::vector> threads_; + std::list> jobs_; + + bool shutdown_; + + std::condition_variable cond_; + std::mutex mutex_; +}; +#else +class Threads : public TaskQueue { +public: + Threads() : running_threads_(0) {} + virtual ~Threads() {} + + virtual void enqueue(std::function fn) override { + std::thread([=]() { + { + std::lock_guard guard(running_threads_mutex_); + running_threads_++; + } + + fn(); + + { + std::lock_guard guard(running_threads_mutex_); + running_threads_--; + } + }).detach(); + } + + virtual void shutdown() override { + for (;;) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + std::lock_guard guard(running_threads_mutex_); + if (!running_threads_) { break; } + } + } + +private: + std::mutex running_threads_mutex_; + int running_threads_; +}; +#endif + +class Server { +public: + typedef std::function Handler; + typedef std::function + HandlerWithContentReader; + typedef std::function Logger; + + Server(); + + virtual ~Server(); + + virtual bool is_valid() const; + + Server &Get(const char *pattern, Handler handler); + Server &Post(const char *pattern, Handler handler); + Server &Post(const char *pattern, HandlerWithContentReader handler); + Server &Put(const char *pattern, Handler handler); + Server &Put(const char *pattern, HandlerWithContentReader handler); + Server &Patch(const char *pattern, Handler handler); + Server &Patch(const char *pattern, HandlerWithContentReader handler); + Server &Delete(const char *pattern, Handler handler); + Server &Options(const char *pattern, Handler handler); + + bool set_base_dir(const char *path); + void set_file_request_handler(Handler handler); + + void set_error_handler(Handler handler); + void set_logger(Logger logger); + + void set_keep_alive_max_count(size_t count); + void set_read_timeout(time_t sec, time_t usec); + void set_payload_max_length(size_t length); + + bool bind_to_port(const char *host, int port, int socket_flags = 0); + int bind_to_any_port(const char *host, int socket_flags = 0); + bool listen_after_bind(); + + bool listen(const char *host, int port, int socket_flags = 0); + + bool is_running() const; + void stop(); + + std::function new_task_queue; + +protected: + bool process_request(Stream &strm, bool last_connection, + bool &connection_close, + std::function setup_request); + + size_t keep_alive_max_count_; + time_t read_timeout_sec_; + time_t read_timeout_usec_; + size_t payload_max_length_; + +private: + typedef std::vector> Handlers; + typedef std::vector> + HandersForContentReader; + + socket_t create_server_socket(const char *host, int port, + int socket_flags) const; + int bind_internal(const char *host, int port, int socket_flags); + bool listen_internal(); + + bool routing(Request &req, Response &res, ContentReader content_reader); + bool handle_file_request(Request &req, Response &res); + bool dispatch_request(Request &req, Response &res, Handlers &handlers); + bool dispatch_request_for_content_reader(Request &req, Response &res, + ContentReader content_reader, + HandersForContentReader &handlers); + + bool parse_request_line(const char *s, Request &req); + bool write_response(Stream &strm, bool last_connection, const Request &req, + Response &res); + bool write_content_with_provider(Stream &strm, const Request &req, + Response &res, const std::string &boundary, + const std::string &content_type); + bool read_content(Stream &strm, bool last_connection, Request &req, + Response &res); + bool read_content_with_content_receiver(Stream &strm, bool last_connection, + Request &req, Response &res, + ContentReceiver reveiver); + + virtual bool process_and_close_socket(socket_t sock); + + std::atomic is_running_; + std::atomic svr_sock_; + std::string base_dir_; + Handler file_request_handler_; + Handlers get_handlers_; + Handlers post_handlers_; + HandersForContentReader post_handlers_for_content_reader; + Handlers put_handlers_; + HandersForContentReader put_handlers_for_content_reader; + Handlers patch_handlers_; + HandersForContentReader patch_handlers_for_content_reader; + Handlers delete_handlers_; + Handlers options_handlers_; + Handler error_handler_; + Logger logger_; +}; + +class Client { +public: + Client(const char *host, int port = 80, time_t timeout_sec = 300); + + virtual ~Client(); + + virtual bool is_valid() const; + + std::shared_ptr Get(const char *path); + + std::shared_ptr Get(const char *path, const Headers &headers); + + std::shared_ptr Get(const char *path, Progress progress); + + std::shared_ptr Get(const char *path, const Headers &headers, + Progress progress); + + std::shared_ptr Get(const char *path, + ContentReceiver content_receiver); + + std::shared_ptr Get(const char *path, const Headers &headers, + ContentReceiver content_receiver); + + std::shared_ptr + Get(const char *path, ContentReceiver content_receiver, Progress progress); + + std::shared_ptr Get(const char *path, const Headers &headers, + ContentReceiver content_receiver, + Progress progress); + + std::shared_ptr Get(const char *path, const Headers &headers, + ResponseHandler response_handler, + ContentReceiver content_receiver); + + std::shared_ptr Get(const char *path, const Headers &headers, + ResponseHandler response_handler, + ContentReceiver content_receiver, + Progress progress); + + std::shared_ptr Head(const char *path); + + std::shared_ptr Head(const char *path, const Headers &headers); + + std::shared_ptr Post(const char *path, const std::string &body, + const char *content_type, + bool compress = false); + + std::shared_ptr Post(const char *path, const Headers &headers, + const std::string &body, + const char *content_type, + bool compress = false); + + std::shared_ptr Post(const char *path, size_t content_length, + ContentProvider content_provider, + const char *content_type, + bool compress = false); + + std::shared_ptr Post(const char *path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const char *content_type, + bool compress = false); + + std::shared_ptr Post(const char *path, const Params ¶ms, + bool compress = false); + + std::shared_ptr Post(const char *path, const Headers &headers, + const Params ¶ms, bool compress = false); + + std::shared_ptr Post(const char *path, + const MultipartFormDataItems &items, + bool compress = false); + + std::shared_ptr Post(const char *path, const Headers &headers, + const MultipartFormDataItems &items, + bool compress = false); + + std::shared_ptr Put(const char *path, const std::string &body, + const char *content_type, + bool compress = false); + + std::shared_ptr Put(const char *path, const Headers &headers, + const std::string &body, + const char *content_type, + bool compress = false); + + std::shared_ptr Put(const char *path, size_t content_length, + ContentProvider content_provider, + const char *content_type, + bool compress = false); + + std::shared_ptr Put(const char *path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const char *content_type, + bool compress = false); + + std::shared_ptr Patch(const char *path, const std::string &body, + const char *content_type, + bool compress = false); + + std::shared_ptr Patch(const char *path, const Headers &headers, + const std::string &body, + const char *content_type, + bool compress = false); + + std::shared_ptr Patch(const char *path, size_t content_length, + ContentProvider content_provider, + const char *content_type, + bool compress = false); + + std::shared_ptr Patch(const char *path, const Headers &headers, + size_t content_length, + ContentProvider content_provider, + const char *content_type, + bool compress = false); + + std::shared_ptr Delete(const char *path); + + std::shared_ptr Delete(const char *path, const std::string &body, + const char *content_type); + + std::shared_ptr Delete(const char *path, const Headers &headers); + + std::shared_ptr Delete(const char *path, const Headers &headers, + const std::string &body, + const char *content_type); + + std::shared_ptr Options(const char *path); + + std::shared_ptr Options(const char *path, const Headers &headers); + + bool send(const Request &req, Response &res); + + bool send(const std::vector &requests, + std::vector &responses); + + void set_keep_alive_max_count(size_t count); + void set_read_timeout(time_t sec, time_t usec); + + void follow_location(bool on); + +protected: + bool process_request(Stream &strm, const Request &req, Response &res, + bool last_connection, bool &connection_close); + + const std::string host_; + const int port_; + time_t timeout_sec_; + const std::string host_and_port_; + size_t keep_alive_max_count_; + time_t read_timeout_sec_; + time_t read_timeout_usec_; + size_t follow_location_; + +private: + socket_t create_client_socket() const; + bool read_response_line(Stream &strm, Response &res); + void write_request(Stream &strm, const Request &req, bool last_connection); + bool redirect(const Request &req, Response &res); + + std::shared_ptr + send_with_content_provider(const char *method, const char *path, + const Headers &headers, const std::string &body, + size_t content_length, + ContentProvider content_provider, + const char *content_type, bool compress); + + virtual bool process_and_close_socket( + socket_t sock, size_t request_count, + std::function + callback); + + virtual bool is_ssl() const; +}; + +inline void Get(std::vector &requests, const char *path, + const Headers &headers) { + Request req; + req.method = "GET"; + req.path = path; + req.headers = headers; + requests.emplace_back(std::move(req)); +} + +inline void Get(std::vector &requests, const char *path) { + Get(requests, path, Headers()); +} + +inline void Post(std::vector &requests, const char *path, + const Headers &headers, const std::string &body, + const char *content_type) { + Request req; + req.method = "POST"; + req.path = path; + req.headers = headers; + req.headers.emplace("Content-Type", content_type); + req.body = body; + requests.emplace_back(std::move(req)); +} + +inline void Post(std::vector &requests, const char *path, + const std::string &body, const char *content_type) { + Post(requests, path, Headers(), body, content_type); +} + +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +class SSLSocketStream : public Stream { +public: + SSLSocketStream(socket_t sock, SSL *ssl, time_t read_timeout_sec, + time_t read_timeout_usec); + virtual ~SSLSocketStream(); + + virtual int read(char *ptr, size_t size); + virtual int write(const char *ptr, size_t size); + virtual int write(const char *ptr); + virtual int write(const std::string &s); + virtual std::string get_remote_addr() const; + +private: + socket_t sock_; + SSL *ssl_; + time_t read_timeout_sec_; + time_t read_timeout_usec_; +}; + +class SSLServer : public Server { +public: + SSLServer(const char *cert_path, const char *private_key_path, + const char *client_ca_cert_file_path = nullptr, + const char *client_ca_cert_dir_path = nullptr); + + virtual ~SSLServer(); + + virtual bool is_valid() const; + +private: + virtual bool process_and_close_socket(socket_t sock); + + SSL_CTX *ctx_; + std::mutex ctx_mutex_; +}; + +class SSLClient : public Client { +public: + SSLClient(const char *host, int port = 443, time_t timeout_sec = 300, + const char *client_cert_path = nullptr, + const char *client_key_path = nullptr); + + virtual ~SSLClient(); + + virtual bool is_valid() const; + + void set_ca_cert_path(const char *ca_ceert_file_path, + const char *ca_cert_dir_path = nullptr); + void enable_server_certificate_verification(bool enabled); + + long get_openssl_verify_result() const; + + SSL_CTX *ssl_context() const noexcept; + +private: + virtual bool process_and_close_socket( + socket_t sock, size_t request_count, + std::function + callback); + virtual bool is_ssl() const; + + bool verify_host(X509 *server_cert) const; + bool verify_host_with_subject_alt_name(X509 *server_cert) const; + bool verify_host_with_common_name(X509 *server_cert) const; + bool check_host_name(const char *pattern, size_t pattern_len) const; + + SSL_CTX *ctx_; + std::mutex ctx_mutex_; + std::vector host_components_; + std::string ca_cert_file_path_; + std::string ca_cert_dir_path_; + bool server_certificate_verification_ = false; + long verify_result_ = 0; +}; +#endif + +/* + * Implementation + */ + +namespace detail { + +inline bool is_hex(char c, int &v) { + if (0x20 <= c && isdigit(c)) { + v = c - '0'; + return true; + } else if ('A' <= c && c <= 'F') { + v = c - 'A' + 10; + return true; + } else if ('a' <= c && c <= 'f') { + v = c - 'a' + 10; + return true; + } + return false; +} + +inline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt, + int &val) { + if (i >= s.size()) { return false; } + + val = 0; + for (; cnt; i++, cnt--) { + if (!s[i]) { return false; } + int v = 0; + if (is_hex(s[i], v)) { + val = val * 16 + v; + } else { + return false; + } + } + return true; +} + +inline std::string from_i_to_hex(size_t n) { + const char *charset = "0123456789abcdef"; + std::string ret; + do { + ret = charset[n & 15] + ret; + n >>= 4; + } while (n > 0); + return ret; +} + +inline size_t to_utf8(int code, char *buff) { + if (code < 0x0080) { + buff[0] = (code & 0x7F); + return 1; + } else if (code < 0x0800) { + buff[0] = (0xC0 | ((code >> 6) & 0x1F)); + buff[1] = (0x80 | (code & 0x3F)); + return 2; + } else if (code < 0xD800) { + buff[0] = (0xE0 | ((code >> 12) & 0xF)); + buff[1] = (0x80 | ((code >> 6) & 0x3F)); + buff[2] = (0x80 | (code & 0x3F)); + return 3; + } else if (code < 0xE000) { // D800 - DFFF is invalid... + return 0; + } else if (code < 0x10000) { + buff[0] = (0xE0 | ((code >> 12) & 0xF)); + buff[1] = (0x80 | ((code >> 6) & 0x3F)); + buff[2] = (0x80 | (code & 0x3F)); + return 3; + } else if (code < 0x110000) { + buff[0] = (0xF0 | ((code >> 18) & 0x7)); + buff[1] = (0x80 | ((code >> 12) & 0x3F)); + buff[2] = (0x80 | ((code >> 6) & 0x3F)); + buff[3] = (0x80 | (code & 0x3F)); + return 4; + } + + // NOTREACHED + return 0; +} + +// NOTE: This code came up with the following stackoverflow post: +// https://stackoverflow.com/questions/180947/base64-decode-snippet-in-c +inline std::string base64_encode(const std::string &in) { + static const auto lookup = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + std::string out; + out.reserve(in.size()); + + int val = 0; + int valb = -6; + + for (uint8_t c : in) { + val = (val << 8) + c; + valb += 8; + while (valb >= 0) { + out.push_back(lookup[(val >> valb) & 0x3F]); + valb -= 6; + } + } + + if (valb > -6) { out.push_back(lookup[((val << 8) >> (valb + 8)) & 0x3F]); } + + while (out.size() % 4) { + out.push_back('='); + } + + return out; +} + +inline bool is_file(const std::string &path) { + struct stat st; + return stat(path.c_str(), &st) >= 0 && S_ISREG(st.st_mode); +} + +inline bool is_dir(const std::string &path) { + struct stat st; + return stat(path.c_str(), &st) >= 0 && S_ISDIR(st.st_mode); +} + +inline bool is_valid_path(const std::string &path) { + size_t level = 0; + size_t i = 0; + + // Skip slash + while (i < path.size() && path[i] == '/') { + i++; + } + + while (i < path.size()) { + // Read component + auto beg = i; + while (i < path.size() && path[i] != '/') { + i++; + } + + auto len = i - beg; + assert(len > 0); + + if (!path.compare(beg, len, ".")) { + ; + } else if (!path.compare(beg, len, "..")) { + if (level == 0) { return false; } + level--; + } else { + level++; + } + + // Skip slash + while (i < path.size() && path[i] == '/') { + i++; + } + } + + return true; +} + +inline void read_file(const std::string &path, std::string &out) { + std::ifstream fs(path, std::ios_base::binary); + fs.seekg(0, std::ios_base::end); + auto size = fs.tellg(); + fs.seekg(0); + out.resize(static_cast(size)); + fs.read(&out[0], size); +} + +inline std::string file_extension(const std::string &path) { + std::smatch m; + auto re = std::regex("\\.([a-zA-Z0-9]+)$"); + if (std::regex_search(path, m, re)) { return m[1].str(); } + return std::string(); +} + +template void split(const char *b, const char *e, char d, Fn fn) { + int i = 0; + int beg = 0; + + while (e ? (b + i != e) : (b[i] != '\0')) { + if (b[i] == d) { + fn(&b[beg], &b[i]); + beg = i + 1; + } + i++; + } + + if (i) { fn(&b[beg], &b[i]); } +} + +// NOTE: until the read size reaches `fixed_buffer_size`, use `fixed_buffer` +// to store data. The call can set memory on stack for performance. +class stream_line_reader { +public: + stream_line_reader(Stream &strm, char *fixed_buffer, size_t fixed_buffer_size) + : strm_(strm), fixed_buffer_(fixed_buffer), + fixed_buffer_size_(fixed_buffer_size) {} + + const char *ptr() const { + if (glowable_buffer_.empty()) { + return fixed_buffer_; + } else { + return glowable_buffer_.data(); + } + } + + size_t size() const { + if (glowable_buffer_.empty()) { + return fixed_buffer_used_size_; + } else { + return glowable_buffer_.size(); + } + } + + bool getline() { + fixed_buffer_used_size_ = 0; + glowable_buffer_.clear(); + + for (size_t i = 0;; i++) { + char byte; + auto n = strm_.read(&byte, 1); + + if (n < 0) { + return false; + } else if (n == 0) { + if (i == 0) { + return false; + } else { + break; + } + } + + append(byte); + + if (byte == '\n') { break; } + } + + return true; + } + +private: + void append(char c) { + if (fixed_buffer_used_size_ < fixed_buffer_size_ - 1) { + fixed_buffer_[fixed_buffer_used_size_++] = c; + fixed_buffer_[fixed_buffer_used_size_] = '\0'; + } else { + if (glowable_buffer_.empty()) { + assert(fixed_buffer_[fixed_buffer_used_size_] == '\0'); + glowable_buffer_.assign(fixed_buffer_, fixed_buffer_used_size_); + } + glowable_buffer_ += c; + } + } + + Stream &strm_; + char *fixed_buffer_; + const size_t fixed_buffer_size_; + size_t fixed_buffer_used_size_; + std::string glowable_buffer_; +}; + +inline int close_socket(socket_t sock) { +#ifdef _WIN32 + return closesocket(sock); +#else + return close(sock); +#endif +} + +inline int select_read(socket_t sock, time_t sec, time_t usec) { +#ifdef CPPHTTPLIB_USE_POLL + struct pollfd pfd_read; + pfd_read.fd = sock; + pfd_read.events = POLLIN; + + auto timeout = static_cast(sec * 1000 + usec / 1000); + + return poll(&pfd_read, 1, timeout); +#else + fd_set fds; + FD_ZERO(&fds); + FD_SET(sock, &fds); + + timeval tv; + tv.tv_sec = static_cast(sec); + tv.tv_usec = static_cast(usec); + + return select(static_cast(sock + 1), &fds, nullptr, nullptr, &tv); +#endif +} + +inline bool wait_until_socket_is_ready(socket_t sock, time_t sec, time_t usec) { +#ifdef CPPHTTPLIB_USE_POLL + struct pollfd pfd_read; + pfd_read.fd = sock; + pfd_read.events = POLLIN | POLLOUT; + + auto timeout = static_cast(sec * 1000 + usec / 1000); + + if (poll(&pfd_read, 1, timeout) > 0 && + pfd_read.revents & (POLLIN | POLLOUT)) { + int error = 0; + socklen_t len = sizeof(error); + return getsockopt(sock, SOL_SOCKET, SO_ERROR, + reinterpret_cast(&error), &len) >= 0 && + !error; + } + return false; +#else + fd_set fdsr; + FD_ZERO(&fdsr); + FD_SET(sock, &fdsr); + + auto fdsw = fdsr; + auto fdse = fdsr; + + timeval tv; + tv.tv_sec = static_cast(sec); + tv.tv_usec = static_cast(usec); + + if (select(static_cast(sock + 1), &fdsr, &fdsw, &fdse, &tv) > 0 && + (FD_ISSET(sock, &fdsr) || FD_ISSET(sock, &fdsw))) { + int error = 0; + socklen_t len = sizeof(error); + return getsockopt(sock, SOL_SOCKET, SO_ERROR, (char *)&error, &len) >= 0 && + !error; + } + return false; +#endif +} + +template +inline bool process_and_close_socket(bool is_client_request, socket_t sock, + size_t keep_alive_max_count, + time_t read_timeout_sec, + time_t read_timeout_usec, T callback) { + assert(keep_alive_max_count > 0); + + bool ret = false; + + if (keep_alive_max_count > 1) { + auto count = keep_alive_max_count; + while (count > 0 && + (is_client_request || + detail::select_read(sock, CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND, + CPPHTTPLIB_KEEPALIVE_TIMEOUT_USECOND) > 0)) { + SocketStream strm(sock, read_timeout_sec, read_timeout_usec); + auto last_connection = count == 1; + auto connection_close = false; + + ret = callback(strm, last_connection, connection_close); + if (!ret || connection_close) { break; } + + count--; + } + } else { + SocketStream strm(sock, read_timeout_sec, read_timeout_usec); + auto dummy_connection_close = false; + ret = callback(strm, true, dummy_connection_close); + } + + close_socket(sock); + return ret; +} + +inline int shutdown_socket(socket_t sock) { +#ifdef _WIN32 + return shutdown(sock, SD_BOTH); +#else + return shutdown(sock, SHUT_RDWR); +#endif +} + +template +socket_t create_socket(const char *host, int port, Fn fn, + int socket_flags = 0) { +#ifdef _WIN32 +#define SO_SYNCHRONOUS_NONALERT 0x20 +#define SO_OPENTYPE 0x7008 + + int opt = SO_SYNCHRONOUS_NONALERT; + setsockopt(INVALID_SOCKET, SOL_SOCKET, SO_OPENTYPE, (char *)&opt, + sizeof(opt)); +#endif + + // Get address info + struct addrinfo hints; + struct addrinfo *result; + + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = socket_flags; + hints.ai_protocol = 0; + + auto service = std::to_string(port); + + if (getaddrinfo(host, service.c_str(), &hints, &result)) { + return INVALID_SOCKET; + } + + for (auto rp = result; rp; rp = rp->ai_next) { + // Create a socket +#ifdef _WIN32 + auto sock = WSASocketW(rp->ai_family, rp->ai_socktype, rp->ai_protocol, + nullptr, 0, WSA_FLAG_NO_HANDLE_INHERIT); +#else + auto sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); +#endif + if (sock == INVALID_SOCKET) { continue; } + +#ifndef _WIN32 + if (fcntl(sock, F_SETFD, FD_CLOEXEC) == -1) { continue; } +#endif + + // Make 'reuse address' option available + int yes = 1; + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&yes), + sizeof(yes)); +#ifdef SO_REUSEPORT + setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, reinterpret_cast(&yes), + sizeof(yes)); +#endif + + // bind or connect + if (fn(sock, *rp)) { + freeaddrinfo(result); + return sock; + } + + close_socket(sock); + } + + freeaddrinfo(result); + return INVALID_SOCKET; +} + +inline void set_nonblocking(socket_t sock, bool nonblocking) { +#ifdef _WIN32 + auto flags = nonblocking ? 1UL : 0UL; + ioctlsocket(sock, FIONBIO, &flags); +#else + auto flags = fcntl(sock, F_GETFL, 0); + fcntl(sock, F_SETFL, + nonblocking ? (flags | O_NONBLOCK) : (flags & (~O_NONBLOCK))); +#endif +} + +inline bool is_connection_error() { +#ifdef _WIN32 + return WSAGetLastError() != WSAEWOULDBLOCK; +#else + return errno != EINPROGRESS; +#endif +} + +inline std::string get_remote_addr(socket_t sock) { + struct sockaddr_storage addr; + socklen_t len = sizeof(addr); + + if (!getpeername(sock, reinterpret_cast(&addr), &len)) { + char ipstr[NI_MAXHOST]; + + if (!getnameinfo(reinterpret_cast(&addr), len, ipstr, + sizeof(ipstr), nullptr, 0, NI_NUMERICHOST)) { + return ipstr; + } + } + + return std::string(); +} + +inline const char *find_content_type(const std::string &path) { + auto ext = file_extension(path); + if (ext == "txt") { + return "text/plain"; + } else if (ext == "html" || ext == "htm") { + return "text/html"; + } else if (ext == "css") { + return "text/css"; + } else if (ext == "jpeg" || ext == "jpg") { + return "image/jpg"; + } else if (ext == "png") { + return "image/png"; + } else if (ext == "gif") { + return "image/gif"; + } else if (ext == "svg") { + return "image/svg+xml"; + } else if (ext == "ico") { + return "image/x-icon"; + } else if (ext == "json") { + return "application/json"; + } else if (ext == "pdf") { + return "application/pdf"; + } else if (ext == "js") { + return "application/javascript"; + } else if (ext == "xml") { + return "application/xml"; + } else if (ext == "xhtml") { + return "application/xhtml+xml"; + } + return nullptr; +} + +inline const char *status_message(int status) { + switch (status) { + case 200: return "OK"; + case 206: return "Partial Content"; + case 301: return "Moved Permanently"; + case 302: return "Found"; + case 303: return "See Other"; + case 304: return "Not Modified"; + case 400: return "Bad Request"; + case 403: return "Forbidden"; + case 404: return "Not Found"; + case 413: return "Payload Too Large"; + case 414: return "Request-URI Too Long"; + case 415: return "Unsupported Media Type"; + case 416: return "Range Not Satisfiable"; + + default: + case 500: return "Internal Server Error"; + } +} + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT +inline bool can_compress(const std::string &content_type) { + return !content_type.find("text/") || content_type == "image/svg+xml" || + content_type == "application/javascript" || + content_type == "application/json" || + content_type == "application/xml" || + content_type == "application/xhtml+xml"; +} + +inline bool compress(std::string &content) { + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + + auto ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8, + Z_DEFAULT_STRATEGY); + if (ret != Z_OK) { return false; } + + strm.avail_in = content.size(); + strm.next_in = + const_cast(reinterpret_cast(content.data())); + + std::string compressed; + + const auto bufsiz = 16384; + char buff[bufsiz]; + do { + strm.avail_out = bufsiz; + strm.next_out = reinterpret_cast(buff); + ret = deflate(&strm, Z_FINISH); + assert(ret != Z_STREAM_ERROR); + compressed.append(buff, bufsiz - strm.avail_out); + } while (strm.avail_out == 0); + + assert(ret == Z_STREAM_END); + assert(strm.avail_in == 0); + + content.swap(compressed); + + deflateEnd(&strm); + return true; +} + +class decompressor { +public: + decompressor() { + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + + // 15 is the value of wbits, which should be at the maximum possible value + // to ensure that any gzip stream can be decoded. The offset of 16 specifies + // that the stream to decompress will be formatted with a gzip wrapper. + is_valid_ = inflateInit2(&strm, 16 + 15) == Z_OK; + } + + ~decompressor() { inflateEnd(&strm); } + + bool is_valid() const { return is_valid_; } + + template + bool decompress(const char *data, size_t data_length, T callback) { + int ret = Z_OK; + + strm.avail_in = data_length; + strm.next_in = const_cast(reinterpret_cast(data)); + + const auto bufsiz = 16384; + char buff[bufsiz]; + do { + strm.avail_out = bufsiz; + strm.next_out = reinterpret_cast(buff); + + ret = inflate(&strm, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR); + switch (ret) { + case Z_NEED_DICT: + case Z_DATA_ERROR: + case Z_MEM_ERROR: inflateEnd(&strm); return false; + } + + if (!callback(buff, bufsiz - strm.avail_out)) { return false; } + } while (strm.avail_out == 0); + + return ret == Z_STREAM_END; + } + +private: + bool is_valid_; + z_stream strm; +}; +#endif + +inline bool has_header(const Headers &headers, const char *key) { + return headers.find(key) != headers.end(); +} + +inline const char *get_header_value(const Headers &headers, const char *key, + size_t id = 0, const char *def = nullptr) { + auto it = headers.find(key); + std::advance(it, id); + if (it != headers.end()) { return it->second.c_str(); } + return def; +} + +inline uint64_t get_header_value_uint64(const Headers &headers, const char *key, + int def = 0) { + auto it = headers.find(key); + if (it != headers.end()) { + return std::strtoull(it->second.data(), nullptr, 10); + } + return def; +} + +inline bool read_headers(Stream &strm, Headers &headers) { + static std::regex re(R"((.+?):\s*(.+?)\s*\r\n)"); + + const auto bufsiz = 2048; + char buf[bufsiz]; + + stream_line_reader line_reader(strm, buf, bufsiz); + + for (;;) { + if (!line_reader.getline()) { return false; } + if (!strcmp(line_reader.ptr(), "\r\n")) { break; } + std::cmatch m; + if (std::regex_match(line_reader.ptr(), m, re)) { + auto key = std::string(m[1]); + auto val = std::string(m[2]); + headers.emplace(key, val); + } + } + + return true; +} + +inline bool read_content_with_length(Stream &strm, uint64_t len, + Progress progress, ContentReceiver out) { + char buf[CPPHTTPLIB_RECV_BUFSIZ]; + + uint64_t r = 0; + while (r < len) { + auto read_len = static_cast(len - r); + auto n = strm.read(buf, std::min(read_len, CPPHTTPLIB_RECV_BUFSIZ)); + if (n <= 0) { return false; } + + if (!out(buf, n)) { return false; } + + r += n; + + if (progress) { + if (!progress(r, len)) { return false; } + } + } + + return true; +} + +inline void skip_content_with_length(Stream &strm, uint64_t len) { + char buf[CPPHTTPLIB_RECV_BUFSIZ]; + uint64_t r = 0; + while (r < len) { + auto read_len = static_cast(len - r); + auto n = strm.read(buf, std::min(read_len, CPPHTTPLIB_RECV_BUFSIZ)); + if (n <= 0) { return; } + r += n; + } +} + +inline bool read_content_without_length(Stream &strm, ContentReceiver out) { + char buf[CPPHTTPLIB_RECV_BUFSIZ]; + for (;;) { + auto n = strm.read(buf, CPPHTTPLIB_RECV_BUFSIZ); + if (n < 0) { + return false; + } else if (n == 0) { + return true; + } + if (!out(buf, n)) { return false; } + } + + return true; +} + +inline bool read_content_chunked(Stream &strm, ContentReceiver out) { + const auto bufsiz = 16; + char buf[bufsiz]; + + stream_line_reader line_reader(strm, buf, bufsiz); + + if (!line_reader.getline()) { return false; } + + auto chunk_len = std::stoi(line_reader.ptr(), 0, 16); + + while (chunk_len > 0) { + if (!read_content_with_length(strm, chunk_len, nullptr, out)) { + return false; + } + + if (!line_reader.getline()) { return false; } + + if (strcmp(line_reader.ptr(), "\r\n")) { break; } + + if (!line_reader.getline()) { return false; } + + chunk_len = std::stoi(line_reader.ptr(), 0, 16); + } + + if (chunk_len == 0) { + // Reader terminator after chunks + if (!line_reader.getline() || strcmp(line_reader.ptr(), "\r\n")) + return false; + } + + return true; +} + +inline bool is_chunked_transfer_encoding(const Headers &headers) { + return !strcasecmp(get_header_value(headers, "Transfer-Encoding", 0, ""), + "chunked"); +} + +template +bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status, + Progress progress, ContentReceiver receiver) { + + ContentReceiver out = [&](const char *buf, size_t n) { + return receiver(buf, n); + }; + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + detail::decompressor decompressor; + + if (!decompressor.is_valid()) { + status = 500; + return false; + } + + if (x.get_header_value("Content-Encoding") == "gzip") { + out = [&](const char *buf, size_t n) { + return decompressor.decompress( + buf, n, [&](const char *buf, size_t n) { return receiver(buf, n); }); + }; + } +#else + if (x.get_header_value("Content-Encoding") == "gzip") { + status = 415; + return false; + } +#endif + + auto ret = true; + auto exceed_payload_max_length = false; + + if (is_chunked_transfer_encoding(x.headers)) { + ret = read_content_chunked(strm, out); + } else if (!has_header(x.headers, "Content-Length")) { + ret = read_content_without_length(strm, out); + } else { + auto len = get_header_value_uint64(x.headers, "Content-Length", 0); + if (len > payload_max_length) { + exceed_payload_max_length = true; + skip_content_with_length(strm, len); + ret = false; + } else if (len > 0) { + ret = read_content_with_length(strm, len, progress, out); + } + } + + if (!ret) { status = exceed_payload_max_length ? 413 : 400; } + + return ret; +} + +template +inline int write_headers(Stream &strm, const T &info, const Headers &headers) { + auto write_len = 0; + for (const auto &x : info.headers) { + auto len = + strm.write_format("%s: %s\r\n", x.first.c_str(), x.second.c_str()); + if (len < 0) { return len; } + write_len += len; + } + for (const auto &x : headers) { + auto len = + strm.write_format("%s: %s\r\n", x.first.c_str(), x.second.c_str()); + if (len < 0) { return len; } + write_len += len; + } + auto len = strm.write("\r\n"); + if (len < 0) { return len; } + write_len += len; + return write_len; +} + +inline ssize_t write_content(Stream &strm, + ContentProviderWithCloser content_provider, + size_t offset, size_t length) { + size_t begin_offset = offset; + size_t end_offset = offset + length; + while (offset < end_offset) { + ssize_t written_length = 0; + content_provider( + offset, end_offset - offset, + [&](const char *d, size_t l) { + offset += l; + written_length = strm.write(d, l); + }, + [&](void) { written_length = -1; }); + if (written_length < 0) { return written_length; } + } + return static_cast(offset - begin_offset); +} + +inline ssize_t +write_content_chunked(Stream &strm, + ContentProviderWithCloser content_provider) { + size_t offset = 0; + auto data_available = true; + ssize_t total_written_length = 0; + while (data_available) { + ssize_t written_length = 0; + content_provider( + offset, 0, + [&](const char *d, size_t l) { + data_available = l > 0; + offset += l; + + // Emit chunked response header and footer for each chunk + auto chunk = from_i_to_hex(l) + "\r\n" + std::string(d, l) + "\r\n"; + written_length = strm.write(chunk); + }, + [&](void) { + data_available = false; + written_length = strm.write("0\r\n\r\n"); + }); + + if (written_length < 0) { return written_length; } + total_written_length += written_length; + } + return total_written_length; +} + +template +inline bool redirect(T &cli, const Request &req, Response &res, + const std::string &path) { + Request new_req; + new_req.method = req.method; + new_req.path = path; + new_req.headers = req.headers; + new_req.body = req.body; + new_req.redirect_count = req.redirect_count - 1; + new_req.response_handler = req.response_handler; + new_req.content_receiver = req.content_receiver; + new_req.progress = req.progress; + + Response new_res; + auto ret = cli.send(new_req, new_res); + if (ret) { res = new_res; } + return ret; +} + +inline std::string encode_url(const std::string &s) { + std::string result; + + for (auto i = 0; s[i]; i++) { + switch (s[i]) { + case ' ': result += "%20"; break; + case '+': result += "%2B"; break; + case '\r': result += "%0D"; break; + case '\n': result += "%0A"; break; + case '\'': result += "%27"; break; + case ',': result += "%2C"; break; + case ':': result += "%3A"; break; + case ';': result += "%3B"; break; + default: + auto c = static_cast(s[i]); + if (c >= 0x80) { + result += '%'; + char hex[4]; + size_t len = snprintf(hex, sizeof(hex) - 1, "%02X", c); + assert(len == 2); + result.append(hex, len); + } else { + result += s[i]; + } + break; + } + } + + return result; +} + +inline std::string decode_url(const std::string &s) { + std::string result; + + for (size_t i = 0; i < s.size(); i++) { + if (s[i] == '%' && i + 1 < s.size()) { + if (s[i + 1] == 'u') { + int val = 0; + if (from_hex_to_i(s, i + 2, 4, val)) { + // 4 digits Unicode codes + char buff[4]; + size_t len = to_utf8(val, buff); + if (len > 0) { result.append(buff, len); } + i += 5; // 'u0000' + } else { + result += s[i]; + } + } else { + int val = 0; + if (from_hex_to_i(s, i + 1, 2, val)) { + // 2 digits hex codes + result += static_cast(val); + i += 2; // '00' + } else { + result += s[i]; + } + } + } else if (s[i] == '+') { + result += ' '; + } else { + result += s[i]; + } + } + + return result; +} + +inline void parse_query_text(const std::string &s, Params ¶ms) { + split(&s[0], &s[s.size()], '&', [&](const char *b, const char *e) { + std::string key; + std::string val; + split(b, e, '=', [&](const char *b, const char *e) { + if (key.empty()) { + key.assign(b, e); + } else { + val.assign(b, e); + } + }); + params.emplace(key, decode_url(val)); + }); +} + +inline bool parse_multipart_boundary(const std::string &content_type, + std::string &boundary) { + auto pos = content_type.find("boundary="); + if (pos == std::string::npos) { return false; } + + boundary = content_type.substr(pos + 9); + return true; +} + +inline bool parse_multipart_formdata(const std::string &boundary, + const std::string &body, + MultipartFiles &files) { + static std::string dash = "--"; + static std::string crlf = "\r\n"; + + static std::regex re_content_type("Content-Type: (.*?)", + std::regex_constants::icase); + + static std::regex re_content_disposition( + "Content-Disposition: form-data; name=\"(.*?)\"(?:; filename=\"(.*?)\")?", + std::regex_constants::icase); + + auto dash_boundary = dash + boundary; + + auto pos = body.find(dash_boundary); + if (pos != 0) { return false; } + + pos += dash_boundary.size(); + + auto next_pos = body.find(crlf, pos); + if (next_pos == std::string::npos) { return false; } + + pos = next_pos + crlf.size(); + + while (pos < body.size()) { + next_pos = body.find(crlf, pos); + if (next_pos == std::string::npos) { return false; } + + std::string name; + MultipartFile file; + + auto header = body.substr(pos, (next_pos - pos)); + + while (pos != next_pos) { + std::smatch m; + if (std::regex_match(header, m, re_content_type)) { + file.content_type = m[1]; + } else if (std::regex_match(header, m, re_content_disposition)) { + name = m[1]; + file.filename = m[2]; + } + + pos = next_pos + crlf.size(); + + next_pos = body.find(crlf, pos); + if (next_pos == std::string::npos) { return false; } + + header = body.substr(pos, (next_pos - pos)); + } + + pos = next_pos + crlf.size(); + + next_pos = body.find(crlf + dash_boundary, pos); + + if (next_pos == std::string::npos) { return false; } + + file.offset = pos; + file.length = next_pos - pos; + + pos = next_pos + crlf.size() + dash_boundary.size(); + + next_pos = body.find(crlf, pos); + if (next_pos == std::string::npos) { return false; } + + files.emplace(name, file); + + pos = next_pos + crlf.size(); + } + + return true; +} + +inline bool parse_range_header(const std::string &s, Ranges &ranges) { + try { + static auto re_first_range = + std::regex(R"(bytes=(\d*-\d*(?:,\s*\d*-\d*)*))"); + std::smatch m; + if (std::regex_match(s, m, re_first_range)) { + auto pos = m.position(1); + auto len = m.length(1); + detail::split( + &s[pos], &s[pos + len], ',', [&](const char *b, const char *e) { + static auto re_another_range = std::regex(R"(\s*(\d*)-(\d*))"); + std::cmatch m; + if (std::regex_match(b, e, m, re_another_range)) { + ssize_t first = -1; + if (!m.str(1).empty()) { + first = static_cast(std::stoll(m.str(1))); + } + + ssize_t last = -1; + if (!m.str(2).empty()) { + last = static_cast(std::stoll(m.str(2))); + } + + if (first != -1 && last != -1 && first > last) { + throw std::runtime_error("invalid range error"); + } + ranges.emplace_back(std::make_pair(first, last)); + } + }); + return true; + } + return false; + } catch (...) { return false; } +} + +inline std::string to_lower(const char *beg, const char *end) { + std::string out; + auto it = beg; + while (it != end) { + out += static_cast(::tolower(*it)); + it++; + } + return out; +} + +inline std::string make_multipart_data_boundary() { + static const char data[] = + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + + std::random_device seed_gen; + std::mt19937 engine(seed_gen()); + + std::string result = "--cpp-httplib-multipart-data-"; + + for (auto i = 0; i < 16; i++) { + result += data[engine() % (sizeof(data) - 1)]; + } + + return result; +} + +inline std::pair +get_range_offset_and_length(const Request &req, size_t content_length, + size_t index) { + auto r = req.ranges[index]; + + if (r.first == -1 && r.second == -1) { + return std::make_pair(0, content_length); + } + + if (r.first == -1) { + r.first = content_length - r.second; + r.second = content_length - 1; + } + + if (r.second == -1) { r.second = content_length - 1; } + + return std::make_pair(r.first, r.second - r.first + 1); +} + +inline std::string make_content_range_header_field(size_t offset, size_t length, + size_t content_length) { + std::string field = "bytes "; + field += std::to_string(offset); + field += "-"; + field += std::to_string(offset + length - 1); + field += "/"; + field += std::to_string(content_length); + return field; +} + +template +bool process_multipart_ranges_data(const Request &req, Response &res, + const std::string &boundary, + const std::string &content_type, + SToken stoken, CToken ctoken, + Content content) { + for (size_t i = 0; i < req.ranges.size(); i++) { + ctoken("--"); + stoken(boundary); + ctoken("\r\n"); + if (!content_type.empty()) { + ctoken("Content-Type: "); + stoken(content_type); + ctoken("\r\n"); + } + + auto offsets = detail::get_range_offset_and_length(req, res.body.size(), i); + auto offset = offsets.first; + auto length = offsets.second; + + ctoken("Content-Range: "); + stoken(make_content_range_header_field(offset, length, res.body.size())); + ctoken("\r\n"); + ctoken("\r\n"); + if (!content(offset, length)) { return false; } + ctoken("\r\n"); + } + + ctoken("--"); + stoken(boundary); + ctoken("--\r\n"); + + return true; +} + +inline std::string make_multipart_ranges_data(const Request &req, Response &res, + const std::string &boundary, + const std::string &content_type) { + std::string data; + + process_multipart_ranges_data( + req, res, boundary, content_type, + [&](const std::string &token) { data += token; }, + [&](const char *token) { data += token; }, + [&](size_t offset, size_t length) { + data += res.body.substr(offset, length); + return true; + }); + + return data; +} + +inline size_t +get_multipart_ranges_data_length(const Request &req, Response &res, + const std::string &boundary, + const std::string &content_type) { + size_t data_length = 0; + + process_multipart_ranges_data( + req, res, boundary, content_type, + [&](const std::string &token) { data_length += token.size(); }, + [&](const char *token) { data_length += strlen(token); }, + [&](size_t /*offset*/, size_t length) { + data_length += length; + return true; + }); + + return data_length; +} + +inline bool write_multipart_ranges_data(Stream &strm, const Request &req, + Response &res, + const std::string &boundary, + const std::string &content_type) { + return process_multipart_ranges_data( + req, res, boundary, content_type, + [&](const std::string &token) { strm.write(token); }, + [&](const char *token) { strm.write(token); }, + [&](size_t offset, size_t length) { + return detail::write_content(strm, res.content_provider, offset, + length) >= 0; + }); +} + +inline std::pair +get_range_offset_and_length(const Request &req, const Response &res, + size_t index) { + auto r = req.ranges[index]; + + if (r.second == -1) { r.second = res.content_length - 1; } + + return std::make_pair(r.first, r.second - r.first + 1); +} + +#ifdef _WIN32 +class WSInit { +public: + WSInit() { + WSADATA wsaData; + WSAStartup(0x0002, &wsaData); + } + + ~WSInit() { WSACleanup(); } +}; + +static WSInit wsinit_; +#endif + +} // namespace detail + +// Header utilities +inline std::pair make_range_header(Ranges ranges) { + std::string field = "bytes="; + auto i = 0; + for (auto r : ranges) { + if (i != 0) { field += ", "; } + if (r.first != -1) { field += std::to_string(r.first); } + field += '-'; + if (r.second != -1) { field += std::to_string(r.second); } + i++; + } + return std::make_pair("Range", field); +} + +inline std::pair +make_basic_authentication_header(const std::string &username, + const std::string &password) { + auto field = "Basic " + detail::base64_encode(username + ":" + password); + return std::make_pair("Authorization", field); +} + +// Request implementation +inline bool Request::has_header(const char *key) const { + return detail::has_header(headers, key); +} + +inline std::string Request::get_header_value(const char *key, size_t id) const { + return detail::get_header_value(headers, key, id, ""); +} + +inline size_t Request::get_header_value_count(const char *key) const { + auto r = headers.equal_range(key); + return std::distance(r.first, r.second); +} + +inline void Request::set_header(const char *key, const char *val) { + headers.emplace(key, val); +} + +inline void Request::set_header(const char *key, const std::string &val) { + headers.emplace(key, val); +} + +inline bool Request::has_param(const char *key) const { + return params.find(key) != params.end(); +} + +inline std::string Request::get_param_value(const char *key, size_t id) const { + auto it = params.find(key); + std::advance(it, id); + if (it != params.end()) { return it->second; } + return std::string(); +} + +inline size_t Request::get_param_value_count(const char *key) const { + auto r = params.equal_range(key); + return std::distance(r.first, r.second); +} + +inline bool Request::has_file(const char *key) const { + return files.find(key) != files.end(); +} + +inline MultipartFile Request::get_file_value(const char *key) const { + auto it = files.find(key); + if (it != files.end()) { return it->second; } + return MultipartFile(); +} + +// Response implementation +inline bool Response::has_header(const char *key) const { + return headers.find(key) != headers.end(); +} + +inline std::string Response::get_header_value(const char *key, + size_t id) const { + return detail::get_header_value(headers, key, id, ""); +} + +inline size_t Response::get_header_value_count(const char *key) const { + auto r = headers.equal_range(key); + return std::distance(r.first, r.second); +} + +inline void Response::set_header(const char *key, const char *val) { + headers.emplace(key, val); +} + +inline void Response::set_header(const char *key, const std::string &val) { + headers.emplace(key, val); +} + +inline void Response::set_redirect(const char *url) { + set_header("Location", url); + status = 302; +} + +inline void Response::set_content(const char *s, size_t n, + const char *content_type) { + body.assign(s, n); + set_header("Content-Type", content_type); +} + +inline void Response::set_content(const std::string &s, + const char *content_type) { + body = s; + set_header("Content-Type", content_type); +} + +inline void Response::set_content_provider( + size_t length, + std::function provider, + std::function resource_releaser) { + assert(length > 0); + content_length = length; + content_provider = [provider](size_t offset, size_t length, DataSink sink, + Done) { provider(offset, length, sink); }; + content_provider_resource_releaser = resource_releaser; +} + +inline void Response::set_chunked_content_provider( + std::function provider, + std::function resource_releaser) { + content_length = 0; + content_provider = [provider](size_t offset, size_t, DataSink sink, + Done done) { provider(offset, sink, done); }; + content_provider_resource_releaser = resource_releaser; +} + +// Rstream implementation +template +inline int Stream::write_format(const char *fmt, const Args &... args) { + const auto bufsiz = 2048; + char buf[bufsiz]; + +#if defined(_MSC_VER) && _MSC_VER < 1900 + auto n = _snprintf_s(buf, bufsiz, bufsiz - 1, fmt, args...); +#else + auto n = snprintf(buf, bufsiz - 1, fmt, args...); +#endif + if (n <= 0) { return n; } + + if (n >= bufsiz - 1) { + std::vector glowable_buf(bufsiz); + + while (n >= static_cast(glowable_buf.size() - 1)) { + glowable_buf.resize(glowable_buf.size() * 2); +#if defined(_MSC_VER) && _MSC_VER < 1900 + n = _snprintf_s(&glowable_buf[0], glowable_buf.size(), + glowable_buf.size() - 1, fmt, args...); +#else + n = snprintf(&glowable_buf[0], glowable_buf.size() - 1, fmt, args...); +#endif + } + return write(&glowable_buf[0], n); + } else { + return write(buf, n); + } +} + +// Socket stream implementation +inline SocketStream::SocketStream(socket_t sock, time_t read_timeout_sec, + time_t read_timeout_usec) + : sock_(sock), read_timeout_sec_(read_timeout_sec), + read_timeout_usec_(read_timeout_usec) {} + +inline SocketStream::~SocketStream() {} + +inline int SocketStream::read(char *ptr, size_t size) { + if (detail::select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0) { + return recv(sock_, ptr, static_cast(size), 0); + } + return -1; +} + +inline int SocketStream::write(const char *ptr, size_t size) { + return send(sock_, ptr, static_cast(size), 0); +} + +inline int SocketStream::write(const char *ptr) { + return write(ptr, strlen(ptr)); +} + +inline int SocketStream::write(const std::string &s) { + return write(s.data(), s.size()); +} + +inline std::string SocketStream::get_remote_addr() const { + return detail::get_remote_addr(sock_); +} + +// Buffer stream implementation +inline int BufferStream::read(char *ptr, size_t size) { +#if defined(_MSC_VER) && _MSC_VER < 1900 + return static_cast(buffer._Copy_s(ptr, size, size)); +#else + return static_cast(buffer.copy(ptr, size)); +#endif +} + +inline int BufferStream::write(const char *ptr, size_t size) { + buffer.append(ptr, size); + return static_cast(size); +} + +inline int BufferStream::write(const char *ptr) { + return write(ptr, strlen(ptr)); +} + +inline int BufferStream::write(const std::string &s) { + return write(s.data(), s.size()); +} + +inline std::string BufferStream::get_remote_addr() const { return ""; } + +inline const std::string &BufferStream::get_buffer() const { return buffer; } + +// HTTP server implementation +inline Server::Server() + : keep_alive_max_count_(CPPHTTPLIB_KEEPALIVE_MAX_COUNT), + read_timeout_sec_(CPPHTTPLIB_READ_TIMEOUT_SECOND), + read_timeout_usec_(CPPHTTPLIB_READ_TIMEOUT_USECOND), + payload_max_length_(CPPHTTPLIB_PAYLOAD_MAX_LENGTH), is_running_(false), + svr_sock_(INVALID_SOCKET) { +#ifndef _WIN32 + signal(SIGPIPE, SIG_IGN); +#endif + new_task_queue = [] { +#if CPPHTTPLIB_THREAD_POOL_COUNT > 0 + return new ThreadPool(CPPHTTPLIB_THREAD_POOL_COUNT); +#else + return new Threads(); +#endif + }; +} + +inline Server::~Server() {} + +inline Server &Server::Get(const char *pattern, Handler handler) { + get_handlers_.push_back(std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline Server &Server::Post(const char *pattern, Handler handler) { + post_handlers_.push_back(std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline Server &Server::Post(const char *pattern, + HandlerWithContentReader handler) { + post_handlers_for_content_reader.push_back( + std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline Server &Server::Put(const char *pattern, Handler handler) { + put_handlers_.push_back(std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline Server &Server::Put(const char *pattern, + HandlerWithContentReader handler) { + put_handlers_for_content_reader.push_back( + std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline Server &Server::Patch(const char *pattern, Handler handler) { + patch_handlers_.push_back(std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline Server &Server::Patch(const char *pattern, + HandlerWithContentReader handler) { + patch_handlers_for_content_reader.push_back( + std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline Server &Server::Delete(const char *pattern, Handler handler) { + delete_handlers_.push_back(std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline Server &Server::Options(const char *pattern, Handler handler) { + options_handlers_.push_back(std::make_pair(std::regex(pattern), handler)); + return *this; +} + +inline bool Server::set_base_dir(const char *path) { + if (detail::is_dir(path)) { + base_dir_ = path; + return true; + } + return false; +} + +inline void Server::set_file_request_handler(Handler handler) { + file_request_handler_ = handler; +} + +inline void Server::set_error_handler(Handler handler) { + error_handler_ = handler; +} + +inline void Server::set_logger(Logger logger) { logger_ = logger; } + +inline void Server::set_keep_alive_max_count(size_t count) { + keep_alive_max_count_ = count; +} + +inline void Server::set_read_timeout(time_t sec, time_t usec) { + read_timeout_sec_ = sec; + read_timeout_usec_ = usec; +} + +inline void Server::set_payload_max_length(size_t length) { + payload_max_length_ = length; +} + +inline bool Server::bind_to_port(const char *host, int port, int socket_flags) { + if (bind_internal(host, port, socket_flags) < 0) return false; + return true; +} +inline int Server::bind_to_any_port(const char *host, int socket_flags) { + return bind_internal(host, 0, socket_flags); +} + +inline bool Server::listen_after_bind() { return listen_internal(); } + +inline bool Server::listen(const char *host, int port, int socket_flags) { + return bind_to_port(host, port, socket_flags) && listen_internal(); +} + +inline bool Server::is_running() const { return is_running_; } + +inline void Server::stop() { + if (is_running_) { + assert(svr_sock_ != INVALID_SOCKET); + std::atomic sock(svr_sock_.exchange(INVALID_SOCKET)); + detail::shutdown_socket(sock); + detail::close_socket(sock); + } +} + +inline bool Server::parse_request_line(const char *s, Request &req) { + static std::regex re( + "(GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH|PRI) " + "(([^?]+)(?:\\?(.+?))?) (HTTP/1\\.[01])\r\n"); + + std::cmatch m; + if (std::regex_match(s, m, re)) { + req.version = std::string(m[5]); + req.method = std::string(m[1]); + req.target = std::string(m[2]); + req.path = detail::decode_url(m[3]); + + // Parse query text + auto len = std::distance(m[4].first, m[4].second); + if (len > 0) { detail::parse_query_text(m[4], req.params); } + + return true; + } + + return false; +} + +inline bool Server::write_response(Stream &strm, bool last_connection, + const Request &req, Response &res) { + assert(res.status != -1); + + if (400 <= res.status && error_handler_) { error_handler_(req, res); } + + // Response line + if (!strm.write_format("HTTP/1.1 %d %s\r\n", res.status, + detail::status_message(res.status))) { + return false; + } + + // Headers + if (last_connection || req.get_header_value("Connection") == "close") { + res.set_header("Connection", "close"); + } + + if (!last_connection && req.get_header_value("Connection") == "Keep-Alive") { + res.set_header("Connection", "Keep-Alive"); + } + + if (!res.has_header("Content-Type")) { + res.set_header("Content-Type", "text/plain"); + } + + if (!res.has_header("Accept-Ranges")) { + res.set_header("Accept-Ranges", "bytes"); + } + + std::string content_type; + std::string boundary; + + if (req.ranges.size() > 1) { + boundary = detail::make_multipart_data_boundary(); + + auto it = res.headers.find("Content-Type"); + if (it != res.headers.end()) { + content_type = it->second; + res.headers.erase(it); + } + + res.headers.emplace("Content-Type", + "multipart/byteranges; boundary=" + boundary); + } + + if (res.body.empty()) { + if (res.content_length > 0) { + size_t length = 0; + if (req.ranges.empty()) { + length = res.content_length; + } else if (req.ranges.size() == 1) { + auto offsets = + detail::get_range_offset_and_length(req, res.content_length, 0); + auto offset = offsets.first; + length = offsets.second; + auto content_range = detail::make_content_range_header_field( + offset, length, res.content_length); + res.set_header("Content-Range", content_range); + } else { + length = detail::get_multipart_ranges_data_length(req, res, boundary, + content_type); + } + res.set_header("Content-Length", std::to_string(length)); + } else { + if (res.content_provider) { + res.set_header("Transfer-Encoding", "chunked"); + } else { + res.set_header("Content-Length", "0"); + } + } + } else { + if (req.ranges.empty()) { + ; + } else if (req.ranges.size() == 1) { + auto offsets = + detail::get_range_offset_and_length(req, res.body.size(), 0); + auto offset = offsets.first; + auto length = offsets.second; + auto content_range = detail::make_content_range_header_field( + offset, length, res.body.size()); + res.set_header("Content-Range", content_range); + res.body = res.body.substr(offset, length); + } else { + res.body = + detail::make_multipart_ranges_data(req, res, boundary, content_type); + } + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + // TODO: 'Accpet-Encoding' has gzip, not gzip;q=0 + const auto &encodings = req.get_header_value("Accept-Encoding"); + if (encodings.find("gzip") != std::string::npos && + detail::can_compress(res.get_header_value("Content-Type"))) { + if (detail::compress(res.body)) { + res.set_header("Content-Encoding", "gzip"); + } + } +#endif + + auto length = std::to_string(res.body.size()); + res.set_header("Content-Length", length); + } + + if (!detail::write_headers(strm, res, Headers())) { return false; } + + // Body + if (req.method != "HEAD") { + if (!res.body.empty()) { + if (!strm.write(res.body)) { return false; } + } else if (res.content_provider) { + if (!write_content_with_provider(strm, req, res, boundary, + content_type)) { + return false; + } + } + } + + // Log + if (logger_) { logger_(req, res); } + + return true; +} + +inline bool +Server::write_content_with_provider(Stream &strm, const Request &req, + Response &res, const std::string &boundary, + const std::string &content_type) { + if (res.content_length) { + if (req.ranges.empty()) { + if (detail::write_content(strm, res.content_provider, 0, + res.content_length) < 0) { + return false; + } + } else if (req.ranges.size() == 1) { + auto offsets = + detail::get_range_offset_and_length(req, res.content_length, 0); + auto offset = offsets.first; + auto length = offsets.second; + if (detail::write_content(strm, res.content_provider, offset, length) < + 0) { + return false; + } + } else { + if (!detail::write_multipart_ranges_data(strm, req, res, boundary, + content_type)) { + return false; + } + } + } else { + if (detail::write_content_chunked(strm, res.content_provider) < 0) { + return false; + } + } + return true; +} + +inline bool Server::read_content(Stream &strm, bool last_connection, + Request &req, Response &res) { + if (!detail::read_content(strm, req, payload_max_length_, res.status, + Progress(), [&](const char *buf, size_t n) { + if (req.body.size() + n > req.body.max_size()) { + return false; + } + req.body.append(buf, n); + return true; + })) { + return write_response(strm, last_connection, req, res); + } + + const auto &content_type = req.get_header_value("Content-Type"); + + if (!content_type.find("application/x-www-form-urlencoded")) { + detail::parse_query_text(req.body, req.params); + } else if (!content_type.find("multipart/form-data")) { + std::string boundary; + if (!detail::parse_multipart_boundary(content_type, boundary) || + !detail::parse_multipart_formdata(boundary, req.body, req.files)) { + res.status = 400; + return write_response(strm, last_connection, req, res); + } + } + + return true; +} + +inline bool +Server::read_content_with_content_receiver(Stream &strm, bool last_connection, + Request &req, Response &res, + ContentReceiver receiver) { + if (!detail::read_content( + strm, req, payload_max_length_, res.status, Progress(), + [&](const char *buf, size_t n) { return receiver(buf, n); })) { + return write_response(strm, last_connection, req, res); + } + + return true; +} + +inline bool Server::handle_file_request(Request &req, Response &res) { + if (!base_dir_.empty() && detail::is_valid_path(req.path)) { + std::string path = base_dir_ + req.path; + + if (!path.empty() && path.back() == '/') { path += "index.html"; } + + if (detail::is_file(path)) { + detail::read_file(path, res.body); + auto type = detail::find_content_type(path); + if (type) { res.set_header("Content-Type", type); } + res.status = 200; + if (file_request_handler_) { file_request_handler_(req, res); } + return true; + } + } + + return false; +} + +inline socket_t Server::create_server_socket(const char *host, int port, + int socket_flags) const { + return detail::create_socket( + host, port, + [](socket_t sock, struct addrinfo &ai) -> bool { + if (::bind(sock, ai.ai_addr, static_cast(ai.ai_addrlen))) { + return false; + } + if (::listen(sock, 5)) { // Listen through 5 channels + return false; + } + return true; + }, + socket_flags); +} + +inline int Server::bind_internal(const char *host, int port, int socket_flags) { + if (!is_valid()) { return -1; } + + svr_sock_ = create_server_socket(host, port, socket_flags); + if (svr_sock_ == INVALID_SOCKET) { return -1; } + + if (port == 0) { + struct sockaddr_storage address; + socklen_t len = sizeof(address); + if (getsockname(svr_sock_, reinterpret_cast(&address), + &len) == -1) { + return -1; + } + if (address.ss_family == AF_INET) { + return ntohs(reinterpret_cast(&address)->sin_port); + } else if (address.ss_family == AF_INET6) { + return ntohs( + reinterpret_cast(&address)->sin6_port); + } else { + return -1; + } + } else { + return port; + } +} + +inline bool Server::listen_internal() { + auto ret = true; + is_running_ = true; + + { + std::unique_ptr task_queue(new_task_queue()); + + for (;;) { + if (svr_sock_ == INVALID_SOCKET) { + // The server socket was closed by 'stop' method. + break; + } + + auto val = detail::select_read(svr_sock_, 0, 100000); + + if (val == 0) { // Timeout + continue; + } + + socket_t sock = accept(svr_sock_, nullptr, nullptr); + + if (sock == INVALID_SOCKET) { + if (errno == EMFILE) { + // The per-process limit of open file descriptors has been reached. + // Try to accept new connections after a short sleep. + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + if (svr_sock_ != INVALID_SOCKET) { + detail::close_socket(svr_sock_); + ret = false; + } else { + ; // The server socket was closed by user. + } + break; + } + + task_queue->enqueue([=]() { process_and_close_socket(sock); }); + } + + task_queue->shutdown(); + } + + is_running_ = false; + return ret; +} + +inline bool Server::routing(Request &req, Response &res, + ContentReader content_reader) { + // File handler + if (req.method == "GET" && handle_file_request(req, res)) { return true; } + + // Content reader handler + if (req.method == "POST") { + if (dispatch_request_for_content_reader(req, res, content_reader, + post_handlers_for_content_reader)) { + return true; + } + } else if (req.method == "PUT") { + if (dispatch_request_for_content_reader(req, res, content_reader, + put_handlers_for_content_reader)) { + return true; + } + } else if (req.method == "PATCH") { + if (dispatch_request_for_content_reader( + req, res, content_reader, patch_handlers_for_content_reader)) { + return true; + } + } + + // Read content into `req.body` + if (!content_reader(nullptr)) { return false; } + + // Regular handler + if (req.method == "GET" || req.method == "HEAD") { + return dispatch_request(req, res, get_handlers_); + } else if (req.method == "POST") { + return dispatch_request(req, res, post_handlers_); + } else if (req.method == "PUT") { + return dispatch_request(req, res, put_handlers_); + } else if (req.method == "DELETE") { + return dispatch_request(req, res, delete_handlers_); + } else if (req.method == "OPTIONS") { + return dispatch_request(req, res, options_handlers_); + } else if (req.method == "PATCH") { + return dispatch_request(req, res, patch_handlers_); + } + + res.status = 400; + return false; +} + +inline bool Server::dispatch_request(Request &req, Response &res, + Handlers &handlers) { + for (const auto &x : handlers) { + const auto &pattern = x.first; + const auto &handler = x.second; + + if (std::regex_match(req.path, req.matches, pattern)) { + handler(req, res); + return true; + } + } + return false; +} + +inline bool +Server::dispatch_request_for_content_reader(Request &req, Response &res, + ContentReader content_reader, + HandersForContentReader &handlers) { + for (const auto &x : handlers) { + const auto &pattern = x.first; + const auto &handler = x.second; + + if (std::regex_match(req.path, req.matches, pattern)) { + handler(req, res, content_reader); + return true; + } + } + return false; +} + +inline bool +Server::process_request(Stream &strm, bool last_connection, + bool &connection_close, + std::function setup_request) { + const auto bufsiz = 2048; + char buf[bufsiz]; + + detail::stream_line_reader line_reader(strm, buf, bufsiz); + + // Connection has been closed on client + if (!line_reader.getline()) { return false; } + + Request req; + Response res; + + res.version = "HTTP/1.1"; + + // Check if the request URI doesn't exceed the limit + if (line_reader.size() > CPPHTTPLIB_REQUEST_URI_MAX_LENGTH) { + Headers dummy; + detail::read_headers(strm, dummy); + res.status = 414; + return write_response(strm, last_connection, req, res); + } + + // Request line and headers + if (!parse_request_line(line_reader.ptr(), req) || + !detail::read_headers(strm, req.headers)) { + res.status = 400; + return write_response(strm, last_connection, req, res); + } + + if (req.get_header_value("Connection") == "close") { + connection_close = true; + } + + if (req.version == "HTTP/1.0" && + req.get_header_value("Connection") != "Keep-Alive") { + connection_close = true; + } + + req.set_header("REMOTE_ADDR", strm.get_remote_addr()); + + if (req.has_header("Range")) { + const auto &range_header_value = req.get_header_value("Range"); + if (!detail::parse_range_header(range_header_value, req.ranges)) { + // TODO: error + } + } + + if (setup_request) { setup_request(req); } + + // Body + ContentReader content_reader = [&](ContentReceiver receiver) { + if (req.method == "POST" || req.method == "PUT" || req.method == "PATCH") { + if (receiver) { + return read_content_with_content_receiver(strm, last_connection, req, + res, receiver); + } else { + return read_content(strm, last_connection, req, res); + } + } else if (req.method == "PRI") { + return read_content(strm, last_connection, req, res); + } + return true; + }; + + // Rounting + if (routing(req, res, content_reader)) { + if (res.status == -1) { res.status = req.ranges.empty() ? 200 : 206; } + } else { + if (res.status == -1) { res.status = 404; } + } + + return write_response(strm, last_connection, req, res); +} + +inline bool Server::is_valid() const { return true; } + +inline bool Server::process_and_close_socket(socket_t sock) { + return detail::process_and_close_socket( + false, sock, keep_alive_max_count_, read_timeout_sec_, read_timeout_usec_, + [this](Stream &strm, bool last_connection, bool &connection_close) { + return process_request(strm, last_connection, connection_close, + nullptr); + }); +} + +// HTTP client implementation +inline Client::Client(const char *host, int port, time_t timeout_sec) + : host_(host), port_(port), timeout_sec_(timeout_sec), + host_and_port_(host_ + ":" + std::to_string(port_)), + keep_alive_max_count_(CPPHTTPLIB_KEEPALIVE_MAX_COUNT), + read_timeout_sec_(CPPHTTPLIB_READ_TIMEOUT_SECOND), + read_timeout_usec_(CPPHTTPLIB_READ_TIMEOUT_USECOND), + follow_location_(false) {} + +inline Client::~Client() {} + +inline bool Client::is_valid() const { return true; } + +inline socket_t Client::create_client_socket() const { + return detail::create_socket( + host_.c_str(), port_, [=](socket_t sock, struct addrinfo &ai) -> bool { + detail::set_nonblocking(sock, true); + + auto ret = connect(sock, ai.ai_addr, static_cast(ai.ai_addrlen)); + if (ret < 0) { + if (detail::is_connection_error() || + !detail::wait_until_socket_is_ready(sock, timeout_sec_, 0)) { + detail::close_socket(sock); + return false; + } + } + + detail::set_nonblocking(sock, false); + return true; + }); +} + +inline bool Client::read_response_line(Stream &strm, Response &res) { + const auto bufsiz = 2048; + char buf[bufsiz]; + + detail::stream_line_reader line_reader(strm, buf, bufsiz); + + if (!line_reader.getline()) { return false; } + + const static std::regex re("(HTTP/1\\.[01]) (\\d+?) .*\r\n"); + + std::cmatch m; + if (std::regex_match(line_reader.ptr(), m, re)) { + res.version = std::string(m[1]); + res.status = std::stoi(std::string(m[2])); + } + + return true; +} + +inline bool Client::send(const Request &req, Response &res) { + if (req.path.empty()) { return false; } + + auto sock = create_client_socket(); + if (sock == INVALID_SOCKET) { return false; } + + auto ret = process_and_close_socket( + sock, 1, [&](Stream &strm, bool last_connection, bool &connection_close) { + return process_request(strm, req, res, last_connection, + connection_close); + }); + + if (ret && follow_location_ && (300 < res.status && res.status < 400)) { + ret = redirect(req, res); + } + + return ret; +} + +inline bool Client::send(const std::vector &requests, + std::vector &responses) { + size_t i = 0; + while (i < requests.size()) { + auto sock = create_client_socket(); + if (sock == INVALID_SOCKET) { return false; } + + if (!process_and_close_socket( + sock, requests.size() - i, + [&](Stream &strm, bool last_connection, + bool &connection_close) -> bool { + auto &req = requests[i]; + auto res = Response(); + i++; + + if (req.path.empty()) { return false; } + auto ret = process_request(strm, req, res, last_connection, + connection_close); + + if (ret && follow_location_ && + (300 < res.status && res.status < 400)) { + ret = redirect(req, res); + } + + if (ret) { responses.emplace_back(std::move(res)); } + + return ret; + })) { + return false; + } + } + + return true; +} + +inline bool Client::redirect(const Request &req, Response &res) { + if (req.redirect_count == 0) { return false; } + + auto location = res.get_header_value("location"); + if (location.empty()) { return false; } + + std::regex re( + R"(^(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*(?:\?[^#]*)?)(?:#.*)?)"); + + auto scheme = is_ssl() ? "https" : "http"; + + std::smatch m; + if (regex_match(location, m, re)) { + auto next_scheme = m[1].str(); + auto next_host = m[2].str(); + auto next_path = m[3].str(); + if (next_host.empty()) { next_host = host_; } + if (next_path.empty()) { next_path = "/"; } + + if (next_scheme == scheme && next_host == host_) { + return detail::redirect(*this, req, res, next_path); + } else { + if (next_scheme == "https") { +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + SSLClient cli(next_host.c_str()); + cli.follow_location(true); + return detail::redirect(cli, req, res, next_path); +#else + return false; +#endif + } else { + Client cli(next_host.c_str()); + cli.follow_location(true); + return detail::redirect(cli, req, res, next_path); + } + } + } + return false; +} + +inline void Client::write_request(Stream &strm, const Request &req, + bool last_connection) { + BufferStream bstrm; + + // Request line + auto path = detail::encode_url(req.path); + + bstrm.write_format("%s %s HTTP/1.1\r\n", req.method.c_str(), path.c_str()); + + // Additonal headers + Headers headers; + if (last_connection) { headers.emplace("Connection", "close"); } + + if (!req.has_header("Host")) { + if (is_ssl()) { + if (port_ == 443) { + headers.emplace("Host", host_); + } else { + headers.emplace("Host", host_and_port_); + } + } else { + if (port_ == 80) { + headers.emplace("Host", host_); + } else { + headers.emplace("Host", host_and_port_); + } + } + } + + if (!req.has_header("Accept")) { headers.emplace("Accept", "*/*"); } + + if (!req.has_header("User-Agent")) { + headers.emplace("User-Agent", "cpp-httplib/0.2"); + } + + if (req.body.empty()) { + if (req.content_provider) { + auto length = std::to_string(req.content_length); + headers.emplace("Content-Length", length); + } else { + headers.emplace("Content-Length", "0"); + } + } else { + if (!req.has_header("Content-Type")) { + headers.emplace("Content-Type", "text/plain"); + } + + if (!req.has_header("Content-Length")) { + auto length = std::to_string(req.body.size()); + headers.emplace("Content-Length", length); + } + } + + detail::write_headers(bstrm, req, headers); + + // Flush buffer + auto &data = bstrm.get_buffer(); + strm.write(data.data(), data.size()); + + // Body + if (req.body.empty()) { + if (req.content_provider) { + size_t offset = 0; + size_t end_offset = req.content_length; + while (offset < end_offset) { + req.content_provider(offset, end_offset - offset, + [&](const char *d, size_t l) { + auto written_length = strm.write(d, l); + offset += written_length; + }); + } + } + } else { + strm.write(req.body); + } +} + +inline std::shared_ptr Client::send_with_content_provider( + const char *method, const char *path, const Headers &headers, + const std::string &body, size_t content_length, + ContentProvider content_provider, const char *content_type, bool compress) { + + Request req; + req.method = method; + req.headers = headers; + req.path = path; + + req.headers.emplace("Content-Type", content_type); + +#ifdef CPPHTTPLIB_ZLIB_SUPPORT + if (compress) { + if (content_provider) { + size_t offset = 0; + while (offset < content_length) { + content_provider(offset, content_length - offset, + [&](const char *data, size_t data_len) { + req.body.append(data, data_len); + offset += data_len; + }); + } + } else { + req.body = body; + } + + if (!detail::compress(req.body)) { return nullptr; } + req.headers.emplace("Content-Encoding", "gzip"); + } else +#endif + { + if (content_provider) { + req.content_length = content_length; + req.content_provider = content_provider; + } else { + req.body = body; + } + } + + auto res = std::make_shared(); + + return send(req, *res) ? res : nullptr; +} + +inline bool Client::process_request(Stream &strm, const Request &req, + Response &res, bool last_connection, + bool &connection_close) { + // Send request + write_request(strm, req, last_connection); + + // Receive response and headers + if (!read_response_line(strm, res) || + !detail::read_headers(strm, res.headers)) { + return false; + } + + if (res.get_header_value("Connection") == "close" || + res.version == "HTTP/1.0") { + connection_close = true; + } + + if (req.response_handler) { + if (!req.response_handler(res)) { return false; } + } + + // Body + if (req.method != "HEAD") { + ContentReceiver out = [&](const char *buf, size_t n) { + if (res.body.size() + n > res.body.max_size()) { return false; } + res.body.append(buf, n); + return true; + }; + + if (req.content_receiver) { + out = [&](const char *buf, size_t n) { + return req.content_receiver(buf, n); + }; + } + + int dummy_status; + if (!detail::read_content(strm, res, std::numeric_limits::max(), + dummy_status, req.progress, out)) { + return false; + } + } + + return true; +} + +inline bool Client::process_and_close_socket( + socket_t sock, size_t request_count, + std::function + callback) { + request_count = std::min(request_count, keep_alive_max_count_); + return detail::process_and_close_socket(true, sock, request_count, + read_timeout_sec_, read_timeout_usec_, + callback); +} + +inline bool Client::is_ssl() const { return false; } + +inline std::shared_ptr Client::Get(const char *path) { + Progress dummy; + return Get(path, Headers(), dummy); +} + +inline std::shared_ptr Client::Get(const char *path, + Progress progress) { + return Get(path, Headers(), progress); +} + +inline std::shared_ptr Client::Get(const char *path, + const Headers &headers) { + Progress dummy; + return Get(path, headers, dummy); +} + +inline std::shared_ptr +Client::Get(const char *path, const Headers &headers, Progress progress) { + Request req; + req.method = "GET"; + req.path = path; + req.headers = headers; + req.progress = progress; + + auto res = std::make_shared(); + return send(req, *res) ? res : nullptr; +} + +inline std::shared_ptr Client::Get(const char *path, + ContentReceiver content_receiver) { + Progress dummy; + return Get(path, Headers(), nullptr, content_receiver, dummy); +} + +inline std::shared_ptr Client::Get(const char *path, + ContentReceiver content_receiver, + Progress progress) { + return Get(path, Headers(), nullptr, content_receiver, progress); +} + +inline std::shared_ptr Client::Get(const char *path, + const Headers &headers, + ContentReceiver content_receiver) { + Progress dummy; + return Get(path, headers, nullptr, content_receiver, dummy); +} + +inline std::shared_ptr Client::Get(const char *path, + const Headers &headers, + ContentReceiver content_receiver, + Progress progress) { + return Get(path, headers, nullptr, content_receiver, progress); +} + +inline std::shared_ptr Client::Get(const char *path, + const Headers &headers, + ResponseHandler response_handler, + ContentReceiver content_receiver) { + Progress dummy; + return Get(path, headers, response_handler, content_receiver, dummy); +} + +inline std::shared_ptr Client::Get(const char *path, + const Headers &headers, + ResponseHandler response_handler, + ContentReceiver content_receiver, + Progress progress) { + Request req; + req.method = "GET"; + req.path = path; + req.headers = headers; + req.response_handler = response_handler; + req.content_receiver = content_receiver; + req.progress = progress; + + auto res = std::make_shared(); + return send(req, *res) ? res : nullptr; +} + +inline std::shared_ptr Client::Head(const char *path) { + return Head(path, Headers()); +} + +inline std::shared_ptr Client::Head(const char *path, + const Headers &headers) { + Request req; + req.method = "HEAD"; + req.headers = headers; + req.path = path; + + auto res = std::make_shared(); + + return send(req, *res) ? res : nullptr; +} + +inline std::shared_ptr Client::Post(const char *path, + const std::string &body, + const char *content_type, + bool compress) { + return Post(path, Headers(), body, content_type, compress); +} + +inline std::shared_ptr +Client::Post(const char *path, const Headers &headers, const std::string &body, + const char *content_type, bool compress) { + return send_with_content_provider("POST", path, headers, body, 0, nullptr, + content_type, compress); +} + +inline std::shared_ptr +Client::Post(const char *path, const Params ¶ms, bool compress) { + return Post(path, Headers(), params, compress); +} + +inline std::shared_ptr Client::Post(const char *path, + size_t content_length, + ContentProvider content_provider, + const char *content_type, + bool compress) { + return Post(path, Headers(), content_length, content_provider, content_type, + compress); +} + +inline std::shared_ptr +Client::Post(const char *path, const Headers &headers, size_t content_length, + ContentProvider content_provider, const char *content_type, + bool compress) { + return send_with_content_provider("POST", path, headers, std::string(), + content_length, content_provider, + content_type, compress); +} + +inline std::shared_ptr Client::Post(const char *path, + const Headers &headers, + const Params ¶ms, + bool compress) { + std::string query; + for (auto it = params.begin(); it != params.end(); ++it) { + if (it != params.begin()) { query += "&"; } + query += it->first; + query += "="; + query += detail::encode_url(it->second); + } + + return Post(path, headers, query, "application/x-www-form-urlencoded", + compress); +} + +inline std::shared_ptr +Client::Post(const char *path, const MultipartFormDataItems &items, + bool compress) { + return Post(path, Headers(), items, compress); +} + +inline std::shared_ptr +Client::Post(const char *path, const Headers &headers, + const MultipartFormDataItems &items, bool compress) { + auto boundary = detail::make_multipart_data_boundary(); + + std::string body; + + for (const auto &item : items) { + body += "--" + boundary + "\r\n"; + body += "Content-Disposition: form-data; name=\"" + item.name + "\""; + if (!item.filename.empty()) { + body += "; filename=\"" + item.filename + "\""; + } + body += "\r\n"; + if (!item.content_type.empty()) { + body += "Content-Type: " + item.content_type + "\r\n"; + } + body += "\r\n"; + body += item.content + "\r\n"; + } + + body += "--" + boundary + "--\r\n"; + + std::string content_type = "multipart/form-data; boundary=" + boundary; + return Post(path, headers, body, content_type.c_str(), compress); +} + +inline std::shared_ptr Client::Put(const char *path, + const std::string &body, + const char *content_type, + bool compress) { + return Put(path, Headers(), body, content_type, compress); +} + +inline std::shared_ptr +Client::Put(const char *path, const Headers &headers, const std::string &body, + const char *content_type, bool compress) { + return send_with_content_provider("PUT", path, headers, body, 0, nullptr, + content_type, compress); +} + +inline std::shared_ptr Client::Put(const char *path, + size_t content_length, + ContentProvider content_provider, + const char *content_type, + bool compress) { + return Put(path, Headers(), content_length, content_provider, content_type, + compress); +} + +inline std::shared_ptr +Client::Put(const char *path, const Headers &headers, size_t content_length, + ContentProvider content_provider, const char *content_type, + bool compress) { + return send_with_content_provider("PUT", path, headers, std::string(), + content_length, content_provider, + content_type, compress); +} + +inline std::shared_ptr Client::Patch(const char *path, + const std::string &body, + const char *content_type, + bool compress) { + return Patch(path, Headers(), body, content_type, compress); +} + +inline std::shared_ptr +Client::Patch(const char *path, const Headers &headers, const std::string &body, + const char *content_type, bool compress) { + return send_with_content_provider("PATCH", path, headers, body, 0, nullptr, + content_type, compress); +} + +inline std::shared_ptr Client::Patch(const char *path, + size_t content_length, + ContentProvider content_provider, + const char *content_type, + bool compress) { + return Patch(path, Headers(), content_length, content_provider, content_type, + compress); +} + +inline std::shared_ptr +Client::Patch(const char *path, const Headers &headers, size_t content_length, + ContentProvider content_provider, const char *content_type, + bool compress) { + return send_with_content_provider("PATCH", path, headers, std::string(), + content_length, content_provider, + content_type, compress); +} + +inline std::shared_ptr Client::Delete(const char *path) { + return Delete(path, Headers(), std::string(), nullptr); +} + +inline std::shared_ptr Client::Delete(const char *path, + const std::string &body, + const char *content_type) { + return Delete(path, Headers(), body, content_type); +} + +inline std::shared_ptr Client::Delete(const char *path, + const Headers &headers) { + return Delete(path, headers, std::string(), nullptr); +} + +inline std::shared_ptr Client::Delete(const char *path, + const Headers &headers, + const std::string &body, + const char *content_type) { + Request req; + req.method = "DELETE"; + req.headers = headers; + req.path = path; + + if (content_type) { req.headers.emplace("Content-Type", content_type); } + req.body = body; + + auto res = std::make_shared(); + + return send(req, *res) ? res : nullptr; +} + +inline std::shared_ptr Client::Options(const char *path) { + return Options(path, Headers()); +} + +inline std::shared_ptr Client::Options(const char *path, + const Headers &headers) { + Request req; + req.method = "OPTIONS"; + req.path = path; + req.headers = headers; + + auto res = std::make_shared(); + + return send(req, *res) ? res : nullptr; +} + +inline void Client::set_keep_alive_max_count(size_t count) { + keep_alive_max_count_ = count; +} + +inline void Client::set_read_timeout(time_t sec, time_t usec) { + read_timeout_sec_ = sec; + read_timeout_usec_ = usec; +} + +inline void Client::follow_location(bool on) { follow_location_ = on; } + +/* + * SSL Implementation + */ +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT +namespace detail { + +template +inline bool process_and_close_socket_ssl( + bool is_client_request, socket_t sock, size_t keep_alive_max_count, + time_t read_timeout_sec, time_t read_timeout_usec, SSL_CTX *ctx, + std::mutex &ctx_mutex, U SSL_connect_or_accept, V setup, T callback) { + assert(keep_alive_max_count > 0); + + SSL *ssl = nullptr; + { + std::lock_guard guard(ctx_mutex); + ssl = SSL_new(ctx); + } + + if (!ssl) { + close_socket(sock); + return false; + } + + auto bio = BIO_new_socket(static_cast(sock), BIO_NOCLOSE); + SSL_set_bio(ssl, bio, bio); + + if (!setup(ssl)) { + SSL_shutdown(ssl); + { + std::lock_guard guard(ctx_mutex); + SSL_free(ssl); + } + + close_socket(sock); + return false; + } + + bool ret = false; + + if (SSL_connect_or_accept(ssl) == 1) { + if (keep_alive_max_count > 1) { + auto count = keep_alive_max_count; + while (count > 0 && + (is_client_request || + detail::select_read(sock, CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND, + CPPHTTPLIB_KEEPALIVE_TIMEOUT_USECOND) > 0)) { + SSLSocketStream strm(sock, ssl, read_timeout_sec, read_timeout_usec); + auto last_connection = count == 1; + auto connection_close = false; + + ret = callback(ssl, strm, last_connection, connection_close); + if (!ret || connection_close) { break; } + + count--; + } + } else { + SSLSocketStream strm(sock, ssl, read_timeout_sec, read_timeout_usec); + auto dummy_connection_close = false; + ret = callback(ssl, strm, true, dummy_connection_close); + } + } + + SSL_shutdown(ssl); + { + std::lock_guard guard(ctx_mutex); + SSL_free(ssl); + } + + close_socket(sock); + + return ret; +} + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +static std::shared_ptr> openSSL_locks_; + +class SSLThreadLocks { +public: + SSLThreadLocks() { + openSSL_locks_ = + std::make_shared>(CRYPTO_num_locks()); + CRYPTO_set_locking_callback(locking_callback); + } + + ~SSLThreadLocks() { CRYPTO_set_locking_callback(nullptr); } + +private: + static void locking_callback(int mode, int type, const char * /*file*/, + int /*line*/) { + auto &locks = *openSSL_locks_; + if (mode & CRYPTO_LOCK) { + locks[type].lock(); + } else { + locks[type].unlock(); + } + } +}; + +#endif + +class SSLInit { +public: + SSLInit() { +#if OPENSSL_VERSION_NUMBER < 0x1010001fL + SSL_load_error_strings(); + SSL_library_init(); +#else + OPENSSL_init_ssl( + OPENSSL_INIT_LOAD_SSL_STRINGS | OPENSSL_INIT_LOAD_CRYPTO_STRINGS, NULL); +#endif + } + + ~SSLInit() { +#if OPENSSL_VERSION_NUMBER < 0x1010001fL + ERR_free_strings(); +#endif + } + +private: +#if OPENSSL_VERSION_NUMBER < 0x10100000L + SSLThreadLocks thread_init_; +#endif +}; + +static SSLInit sslinit_; + +} // namespace detail + +// SSL socket stream implementation +inline SSLSocketStream::SSLSocketStream(socket_t sock, SSL *ssl, + time_t read_timeout_sec, + time_t read_timeout_usec) + : sock_(sock), ssl_(ssl), read_timeout_sec_(read_timeout_sec), + read_timeout_usec_(read_timeout_usec) {} + +inline SSLSocketStream::~SSLSocketStream() {} + +inline int SSLSocketStream::read(char *ptr, size_t size) { + if (SSL_pending(ssl_) > 0 || + detail::select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0) { + return SSL_read(ssl_, ptr, static_cast(size)); + } + return -1; +} + +inline int SSLSocketStream::write(const char *ptr, size_t size) { + return SSL_write(ssl_, ptr, static_cast(size)); +} + +inline int SSLSocketStream::write(const char *ptr) { + return write(ptr, strlen(ptr)); +} + +inline int SSLSocketStream::write(const std::string &s) { + return write(s.data(), s.size()); +} + +inline std::string SSLSocketStream::get_remote_addr() const { + return detail::get_remote_addr(sock_); +} + +// SSL HTTP server implementation +inline SSLServer::SSLServer(const char *cert_path, const char *private_key_path, + const char *client_ca_cert_file_path, + const char *client_ca_cert_dir_path) { + ctx_ = SSL_CTX_new(SSLv23_server_method()); + + if (ctx_) { + SSL_CTX_set_options(ctx_, + SSL_OP_ALL | SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | + SSL_OP_NO_COMPRESSION | + SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION); + + // auto ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1); + // SSL_CTX_set_tmp_ecdh(ctx_, ecdh); + // EC_KEY_free(ecdh); + + if (SSL_CTX_use_certificate_chain_file(ctx_, cert_path) != 1 || + SSL_CTX_use_PrivateKey_file(ctx_, private_key_path, SSL_FILETYPE_PEM) != + 1) { + SSL_CTX_free(ctx_); + ctx_ = nullptr; + } else if (client_ca_cert_file_path || client_ca_cert_dir_path) { + // if (client_ca_cert_file_path) { + // auto list = SSL_load_client_CA_file(client_ca_cert_file_path); + // SSL_CTX_set_client_CA_list(ctx_, list); + // } + + SSL_CTX_load_verify_locations(ctx_, client_ca_cert_file_path, + client_ca_cert_dir_path); + + SSL_CTX_set_verify( + ctx_, + SSL_VERIFY_PEER | + SSL_VERIFY_FAIL_IF_NO_PEER_CERT, // SSL_VERIFY_CLIENT_ONCE, + nullptr); + } + } +} + +inline SSLServer::~SSLServer() { + if (ctx_) { SSL_CTX_free(ctx_); } +} + +inline bool SSLServer::is_valid() const { return ctx_; } + +inline bool SSLServer::process_and_close_socket(socket_t sock) { + return detail::process_and_close_socket_ssl( + false, sock, keep_alive_max_count_, read_timeout_sec_, read_timeout_usec_, + ctx_, ctx_mutex_, SSL_accept, [](SSL * /*ssl*/) { return true; }, + [this](SSL *ssl, Stream &strm, bool last_connection, + bool &connection_close) { + return process_request(strm, last_connection, connection_close, + [&](Request &req) { req.ssl = ssl; }); + }); +} + +// SSL HTTP client implementation +inline SSLClient::SSLClient(const char *host, int port, time_t timeout_sec, + const char *client_cert_path, + const char *client_key_path) + : Client(host, port, timeout_sec) { + ctx_ = SSL_CTX_new(SSLv23_client_method()); + + detail::split(&host_[0], &host_[host_.size()], '.', + [&](const char *b, const char *e) { + host_components_.emplace_back(std::string(b, e)); + }); + if (client_cert_path && client_key_path) { + if (SSL_CTX_use_certificate_file(ctx_, client_cert_path, + SSL_FILETYPE_PEM) != 1 || + SSL_CTX_use_PrivateKey_file(ctx_, client_key_path, SSL_FILETYPE_PEM) != + 1) { + SSL_CTX_free(ctx_); + ctx_ = nullptr; + } + } +} + +inline SSLClient::~SSLClient() { + if (ctx_) { SSL_CTX_free(ctx_); } +} + +inline bool SSLClient::is_valid() const { return ctx_; } + +inline void SSLClient::set_ca_cert_path(const char *ca_cert_file_path, + const char *ca_cert_dir_path) { + if (ca_cert_file_path) { ca_cert_file_path_ = ca_cert_file_path; } + if (ca_cert_dir_path) { ca_cert_dir_path_ = ca_cert_dir_path; } +} + +inline void SSLClient::enable_server_certificate_verification(bool enabled) { + server_certificate_verification_ = enabled; +} + +inline long SSLClient::get_openssl_verify_result() const { + return verify_result_; +} + +inline SSL_CTX *SSLClient::ssl_context() const noexcept { return ctx_; } + +inline bool SSLClient::process_and_close_socket( + socket_t sock, size_t request_count, + std::function + callback) { + + request_count = std::min(request_count, keep_alive_max_count_); + + return is_valid() && + detail::process_and_close_socket_ssl( + true, sock, request_count, read_timeout_sec_, read_timeout_usec_, + ctx_, ctx_mutex_, + [&](SSL *ssl) { + if (ca_cert_file_path_.empty()) { + SSL_CTX_set_verify(ctx_, SSL_VERIFY_NONE, nullptr); + } else { + if (!SSL_CTX_load_verify_locations( + ctx_, ca_cert_file_path_.c_str(), nullptr)) { + return false; + } + SSL_CTX_set_verify(ctx_, SSL_VERIFY_PEER, nullptr); + } + + if (SSL_connect(ssl) != 1) { return false; } + + if (server_certificate_verification_) { + verify_result_ = SSL_get_verify_result(ssl); + + if (verify_result_ != X509_V_OK) { return false; } + + auto server_cert = SSL_get_peer_certificate(ssl); + + if (server_cert == nullptr) { return false; } + + if (!verify_host(server_cert)) { + X509_free(server_cert); + return false; + } + X509_free(server_cert); + } + + return true; + }, + [&](SSL *ssl) { + SSL_set_tlsext_host_name(ssl, host_.c_str()); + return true; + }, + [&](SSL * /*ssl*/, Stream &strm, bool last_connection, + bool &connection_close) { + return callback(strm, last_connection, connection_close); + }); +} + +inline bool SSLClient::is_ssl() const { return true; } + +inline bool SSLClient::verify_host(X509 *server_cert) const { + /* Quote from RFC2818 section 3.1 "Server Identity" + + If a subjectAltName extension of type dNSName is present, that MUST + be used as the identity. Otherwise, the (most specific) Common Name + field in the Subject field of the certificate MUST be used. Although + the use of the Common Name is existing practice, it is deprecated and + Certification Authorities are encouraged to use the dNSName instead. + + Matching is performed using the matching rules specified by + [RFC2459]. If more than one identity of a given type is present in + the certificate (e.g., more than one dNSName name, a match in any one + of the set is considered acceptable.) Names may contain the wildcard + character * which is considered to match any single domain name + component or component fragment. E.g., *.a.com matches foo.a.com but + not bar.foo.a.com. f*.com matches foo.com but not bar.com. + + In some cases, the URI is specified as an IP address rather than a + hostname. In this case, the iPAddress subjectAltName must be present + in the certificate and must exactly match the IP in the URI. + + */ + return verify_host_with_subject_alt_name(server_cert) || + verify_host_with_common_name(server_cert); +} + +inline bool +SSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const { + auto ret = false; + + auto type = GEN_DNS; + + struct in6_addr addr6; + struct in_addr addr; + size_t addr_len = 0; + +#ifndef __MINGW32__ + if (inet_pton(AF_INET6, host_.c_str(), &addr6)) { + type = GEN_IPADD; + addr_len = sizeof(struct in6_addr); + } else if (inet_pton(AF_INET, host_.c_str(), &addr)) { + type = GEN_IPADD; + addr_len = sizeof(struct in_addr); + } +#endif + + auto alt_names = static_cast( + X509_get_ext_d2i(server_cert, NID_subject_alt_name, nullptr, nullptr)); + + if (alt_names) { + auto dsn_matched = false; + auto ip_mached = false; + + auto count = sk_GENERAL_NAME_num(alt_names); + + for (auto i = 0; i < count && !dsn_matched; i++) { + auto val = sk_GENERAL_NAME_value(alt_names, i); + if (val->type == type) { + auto name = (const char *)ASN1_STRING_get0_data(val->d.ia5); + auto name_len = (size_t)ASN1_STRING_length(val->d.ia5); + + if (strlen(name) == name_len) { + switch (type) { + case GEN_DNS: dsn_matched = check_host_name(name, name_len); break; + + case GEN_IPADD: + if (!memcmp(&addr6, name, addr_len) || + !memcmp(&addr, name, addr_len)) { + ip_mached = true; + } + break; + } + } + } + } + + if (dsn_matched || ip_mached) { ret = true; } + } + + GENERAL_NAMES_free((STACK_OF(GENERAL_NAME) *)alt_names); + + return ret; +} + +inline bool SSLClient::verify_host_with_common_name(X509 *server_cert) const { + const auto subject_name = X509_get_subject_name(server_cert); + + if (subject_name != nullptr) { + char name[BUFSIZ]; + auto name_len = X509_NAME_get_text_by_NID(subject_name, NID_commonName, + name, sizeof(name)); + + if (name_len != -1) { return check_host_name(name, name_len); } + } + + return false; +} + +inline bool SSLClient::check_host_name(const char *pattern, + size_t pattern_len) const { + if (host_.size() == pattern_len && host_ == pattern) { return true; } + + // Wildcard match + // https://bugs.launchpad.net/ubuntu/+source/firefox-3.0/+bug/376484 + std::vector pattern_components; + detail::split(&pattern[0], &pattern[pattern_len], '.', + [&](const char *b, const char *e) { + pattern_components.emplace_back(std::string(b, e)); + }); + + if (host_components_.size() != pattern_components.size()) { return false; } + + auto itr = pattern_components.begin(); + for (const auto &h : host_components_) { + auto &p = *itr; + if (p != h && p != "*") { + auto partial_match = (p.size() > 0 && p[p.size() - 1] == '*' && + !p.compare(0, p.size() - 1, h)); + if (!partial_match) { return false; } + } + ++itr; + } + + return true; +} +#endif + +} // namespace httplib + +#endif // CPPHTTPLIB_HTTPLIB_H diff --git a/3rd/datasketches/datasketches/MurmurHash3.h b/3rd/datasketches/datasketches/MurmurHash3.h new file mode 100644 index 000000000..45a64c679 --- /dev/null +++ b/3rd/datasketches/datasketches/MurmurHash3.h @@ -0,0 +1,178 @@ +// Minimally modified from Austin Applebee's code: +// * Removed MurmurHash3_x86_32 and MurmurHash3_x86_128 +// * Changed input seed in MurmurHash3_x64_128 to uint64_t +// * Define and use HashState reference to return result +// * Made entire hash function defined inline +//----------------------------------------------------------------------------- +// MurmurHash3 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. + +// Note - The x86 and x64 versions do _not_ produce the same results, as the +// algorithms are optimized for their respective platforms. You can still +// compile and run any of them on any platform, but your performance with the +// non-native version will be less than optimal. + +#ifndef _MURMURHASH3_H_ +#define _MURMURHASH3_H_ + +//----------------------------------------------------------------------------- +// Platform-specific functions and macros + +// Microsoft Visual Studio + +#if defined(_MSC_VER) + +typedef unsigned char uint8_t; +typedef unsigned int uint32_t; +typedef unsigned __int64 uint64_t; + +#define FORCE_INLINE __forceinline + +#include + +#define ROTL32(x,y) _rotl(x,y) +#define ROTL64(x,y) _rotl64(x,y) + +#define BIG_CONSTANT(x) (x) + +// Other compilers + +#else // defined(_MSC_VER) + +#include + +#define FORCE_INLINE inline __attribute__((always_inline)) + +inline uint32_t rotl32 ( uint32_t x, int8_t r ) +{ + return (x << r) | (x >> (32 - r)); +} + +inline uint64_t rotl64 ( uint64_t x, int8_t r ) +{ + return (x << r) | (x >> (64 - r)); +} + +#define ROTL32(x,y) rotl32(x,y) +#define ROTL64(x,y) rotl64(x,y) + +#define BIG_CONSTANT(x) (x##LLU) + +#endif // !defined(_MSC_VER) + +//----------------------------------------------------------------------------- + +//----------------------------------------------------------------------------- +// Return type - Using C++ reference for return type which should allow better +// compiler optimization than a void* pointer +typedef struct { + uint64_t h1; + uint64_t h2; +} HashState; + + +//----------------------------------------------------------------------------- +// Block read - if your platform needs to do endian-swapping or can only +// handle aligned reads, do the conversion here + +FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, int i ) +{ + return p[i]; +} + +//----------------------------------------------------------------------------- +// Finalization mix - force all bits of a hash block to avalanche + +FORCE_INLINE uint64_t fmix64 ( uint64_t k ) +{ + k ^= k >> 33; + k *= BIG_CONSTANT(0xff51afd7ed558ccd); + k ^= k >> 33; + k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53); + k ^= k >> 33; + + return k; +} + +FORCE_INLINE void MurmurHash3_x64_128(const void* key, int lenBytes, uint64_t seed, HashState& out) { + static const uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5); + static const uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f); + + const uint8_t* data = (const uint8_t*)key; + + out.h1 = seed; + out.h2 = seed; + + // Number of full 128-bit blocks of 16 bytes. + // Possible exclusion fo a remainder of up to 15 bytes. + const int nblocks = lenBytes >> 4; // bytes / 16 + + // Process the 128-bit blocks (the body) into teh hash + const uint64_t* blocks = (const uint64_t*)(data); + for (int i = 0; i < nblocks; ++i) { // 16 bytes per block + //uint64_t k1 = getblock64(blocks, 0); + //uint64_t k2 = getblock64(blocks, 1); + uint64_t k1 = getblock64(blocks,i*2+0); + uint64_t k2 = getblock64(blocks,i*2+1); + + k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; out.h1 ^= k1; + out.h1 = ROTL64(out.h1,27); + out.h1 += out.h2; + out.h1 = out.h1*5+0x52dce729; + + k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; out.h2 ^= k2; + out.h2 = ROTL64(out.h2,31); + out.h2 += out.h1; + out.h2 = out.h2*5+0x38495ab5; + + blocks += 2; + } + + // tail + //const uint8_t * tail = (const uint8_t*)blocks; + const uint8_t * tail = (const uint8_t*)(data + (nblocks << 4)); + + uint64_t k1 = 0; + uint64_t k2 = 0; + + switch(lenBytes & 15) + { + case 15: k2 ^= ((uint64_t)tail[14]) << 48; //@suppress("No break at end of case") + case 14: k2 ^= ((uint64_t)tail[13]) << 40; //@suppress("No break at end of case") + case 13: k2 ^= ((uint64_t)tail[12]) << 32; //@suppress("No break at end of case") + case 12: k2 ^= ((uint64_t)tail[11]) << 24; //@suppress("No break at end of case") + case 11: k2 ^= ((uint64_t)tail[10]) << 16; //@suppress("No break at end of case") + case 10: k2 ^= ((uint64_t)tail[ 9]) << 8; //@suppress("No break at end of case") + case 9: k2 ^= ((uint64_t)tail[ 8]) << 0; + k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; out.h2 ^= k2; + //@suppress("No break at end of case") + case 8: k1 ^= ((uint64_t)tail[ 7]) << 56; //@suppress("No break at end of case") + case 7: k1 ^= ((uint64_t)tail[ 6]) << 48; //@suppress("No break at end of case") + case 6: k1 ^= ((uint64_t)tail[ 5]) << 40; //@suppress("No break at end of case") + case 5: k1 ^= ((uint64_t)tail[ 4]) << 32; //@suppress("No break at end of case") + case 4: k1 ^= ((uint64_t)tail[ 3]) << 24; //@suppress("No break at end of case") + case 3: k1 ^= ((uint64_t)tail[ 2]) << 16; //@suppress("No break at end of case") + case 2: k1 ^= ((uint64_t)tail[ 1]) << 8; //@suppress("No break at end of case") + case 1: k1 ^= ((uint64_t)tail[ 0]) << 0; + k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; out.h1 ^= k1; + }; + + //---------- + // finalization + + out.h1 ^= lenBytes; + out.h2 ^= lenBytes; + + out.h1 += out.h2; + out.h2 += out.h1; + + out.h1 = fmix64(out.h1); + out.h2 = fmix64(out.h2); + + out.h1 += out.h2; + out.h2 += out.h1; +} + +//----------------------------------------------------------------------------- + +#endif // _MURMURHASH3_H_ \ No newline at end of file diff --git a/3rd/datasketches/datasketches/cpc/compression_data.hpp b/3rd/datasketches/datasketches/cpc/compression_data.hpp new file mode 100644 index 000000000..0889b479e --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/compression_data.hpp @@ -0,0 +1,6022 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// author Kevin Lang, Oath Research + +#ifndef CPC_COMPRESSION_DATA_HPP_ +#define CPC_COMPRESSION_DATA_HPP_ + +namespace datasketches { + +/* + The 23 length-limited Huffman codes in this file were created + by the ocaml program "generateHuffmanCodes.ml", which was + compiled and run as follows: + +~/ocaml-4.03.0/bin/ocamlopt -o generateHuffmanCodes columnProbabilities.ml generateHuffmanCodes.ml + +./generateHuffmanCodes > raw-encoding-tables.c + +Some manual cutting and pasting was then done to transfer the contents +of that file into this one. + +Only the encoding tables are defined by this file. The decoding tables (which are exact inverses) +are created at library startup time. +*/ + +static const uint16_t encoding_tables_for_high_entropy_byte [22][256] = { + // Sixteen Encoding Tables for the Steady State. + + // (table 0 of 22) (steady 0 of 16) (phase = 0.031250000 = 1.0 / 32.0) + // entropy: 4.4619200780464778333 + // avg_length: 4.5415773046232610355; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x9017, // ( 9, 23) 0 + 0x5009, // ( 5, 9) 1 + 0x7033, // ( 7, 51) 2 + 0x3002, // ( 3, 2) 3 + 0x9117, // ( 9, 279) 4 + 0x5019, // ( 5, 25) 5 + 0x7073, // ( 7, 115) 6 + 0x2000, // ( 2, 0) 7 + 0xa177, // (10, 375) 8 + 0x601d, // ( 6, 29) 9 + 0x803b, // ( 8, 59) 10 + 0x4001, // ( 4, 1) 11 + 0xa377, // (10, 887) 12 + 0x5005, // ( 5, 5) 13 + 0x80bb, // ( 8, 187) 14 + 0x3006, // ( 3, 6) 15 + 0xb0cf, // (11, 207) 16 + 0x700b, // ( 7, 11) 17 + 0xa0f7, // (10, 247) 18 + 0x5015, // ( 5, 21) 19 + 0xb4cf, // (11, 1231) 20 + 0x704b, // ( 7, 75) 21 + 0x9097, // ( 9, 151) 22 + 0x500d, // ( 5, 13) 23 + 0xc4af, // (12, 1199) 24 + 0x807b, // ( 8, 123) 25 + 0xa2f7, // (10, 759) 26 + 0x603d, // ( 6, 61) 27 + 0xccaf, // (12, 3247) 28 + 0x80fb, // ( 8, 251) 29 + 0xa1f7, // (10, 503) 30 + 0x6003, // ( 6, 3) 31 + 0xc2af, // (12, 687) 32 + 0x8007, // ( 8, 7) 33 + 0xb2cf, // (11, 719) 34 + 0x6023, // ( 6, 35) 35 + 0xcaaf, // (12, 2735) 36 + 0x8087, // ( 8, 135) 37 + 0xa3f7, // (10, 1015) 38 + 0x6013, // ( 6, 19) 39 + 0xc6af, // (12, 1711) 40 + 0x9197, // ( 9, 407) 41 + 0xceaf, // (12, 3759) 42 + 0x702b, // ( 7, 43) 43 + 0xc1af, // (12, 431) 44 + 0x9057, // ( 9, 87) 45 + 0xb6cf, // (11, 1743) 46 + 0x706b, // ( 7, 107) 47 + 0xc9af, // (12, 2479) 48 + 0xa00f, // (10, 15) 49 + 0xc5af, // (12, 1455) 50 + 0x8047, // ( 8, 71) 51 + 0xcdaf, // (12, 3503) 52 + 0xa20f, // (10, 527) 53 + 0xc3af, // (12, 943) 54 + 0x80c7, // ( 8, 199) 55 + 0xcbaf, // (12, 2991) 56 + 0xb1cf, // (11, 463) 57 + 0xc7af, // (12, 1967) 58 + 0x9157, // ( 9, 343) 59 + 0xcfaf, // (12, 4015) 60 + 0xb5cf, // (11, 1487) 61 + 0xc06f, // (12, 111) 62 + 0x90d7, // ( 9, 215) 63 + 0xc86f, // (12, 2159) 64 + 0x91d7, // ( 9, 471) 65 + 0xc46f, // (12, 1135) 66 + 0x701b, // ( 7, 27) 67 + 0xcc6f, // (12, 3183) 68 + 0x9037, // ( 9, 55) 69 + 0xb3cf, // (11, 975) 70 + 0x705b, // ( 7, 91) 71 + 0xc26f, // (12, 623) 72 + 0xa10f, // (10, 271) 73 + 0xca6f, // (12, 2671) 74 + 0x8027, // ( 8, 39) 75 + 0xc66f, // (12, 1647) 76 + 0xa30f, // (10, 783) 77 + 0xce6f, // (12, 3695) 78 + 0x80a7, // ( 8, 167) 79 + 0xc16f, // (12, 367) 80 + 0xb7cf, // (11, 1999) 81 + 0xc96f, // (12, 2415) 82 + 0x9137, // ( 9, 311) 83 + 0xc56f, // (12, 1391) 84 + 0xb02f, // (11, 47) 85 + 0xcd6f, // (12, 3439) 86 + 0x90b7, // ( 9, 183) 87 + 0xc36f, // (12, 879) 88 + 0xcb6f, // (12, 2927) 89 + 0xc76f, // (12, 1903) 90 + 0xa08f, // (10, 143) 91 + 0xcf6f, // (12, 3951) 92 + 0xc0ef, // (12, 239) 93 + 0xc8ef, // (12, 2287) 94 + 0xa28f, // (10, 655) 95 + 0xc4ef, // (12, 1263) 96 + 0xccef, // (12, 3311) 97 + 0xc2ef, // (12, 751) 98 + 0xa18f, // (10, 399) 99 + 0xcaef, // (12, 2799) 100 + 0xc6ef, // (12, 1775) 101 + 0xceef, // (12, 3823) 102 + 0xa38f, // (10, 911) 103 + 0xc1ef, // (12, 495) 104 + 0xc9ef, // (12, 2543) 105 + 0xc5ef, // (12, 1519) 106 + 0xb42f, // (11, 1071) 107 + 0xcdef, // (12, 3567) 108 + 0xc3ef, // (12, 1007) 109 + 0xcbef, // (12, 3055) 110 + 0xb22f, // (11, 559) 111 + 0xc7ef, // (12, 2031) 112 + 0xcfef, // (12, 4079) 113 + 0xc01f, // (12, 31) 114 + 0xc81f, // (12, 2079) 115 + 0xc41f, // (12, 1055) 116 + 0xcc1f, // (12, 3103) 117 + 0xc21f, // (12, 543) 118 + 0xca1f, // (12, 2591) 119 + 0xc61f, // (12, 1567) 120 + 0xce1f, // (12, 3615) 121 + 0xc11f, // (12, 287) 122 + 0xc91f, // (12, 2335) 123 + 0xc51f, // (12, 1311) 124 + 0xcd1f, // (12, 3359) 125 + 0xc31f, // (12, 799) 126 + 0xcb1f, // (12, 2847) 127 + 0xc71f, // (12, 1823) 128 + 0xa04f, // (10, 79) 129 + 0xcf1f, // (12, 3871) 130 + 0x8067, // ( 8, 103) 131 + 0xc09f, // (12, 159) 132 + 0xa24f, // (10, 591) 133 + 0xc89f, // (12, 2207) 134 + 0x80e7, // ( 8, 231) 135 + 0xc49f, // (12, 1183) 136 + 0xb62f, // (11, 1583) 137 + 0xcc9f, // (12, 3231) 138 + 0x91b7, // ( 9, 439) 139 + 0xc29f, // (12, 671) 140 + 0xb12f, // (11, 303) 141 + 0xca9f, // (12, 2719) 142 + 0x9077, // ( 9, 119) 143 + 0xc69f, // (12, 1695) 144 + 0xce9f, // (12, 3743) 145 + 0xc19f, // (12, 415) 146 + 0xa14f, // (10, 335) 147 + 0xc99f, // (12, 2463) 148 + 0xc59f, // (12, 1439) 149 + 0xcd9f, // (12, 3487) 150 + 0xa34f, // (10, 847) 151 + 0xc39f, // (12, 927) 152 + 0xcb9f, // (12, 2975) 153 + 0xc79f, // (12, 1951) 154 + 0xb52f, // (11, 1327) 155 + 0xcf9f, // (12, 3999) 156 + 0xc05f, // (12, 95) 157 + 0xc85f, // (12, 2143) 158 + 0xb32f, // (11, 815) 159 + 0xc45f, // (12, 1119) 160 + 0xcc5f, // (12, 3167) 161 + 0xc25f, // (12, 607) 162 + 0xb72f, // (11, 1839) 163 + 0xca5f, // (12, 2655) 164 + 0xc65f, // (12, 1631) 165 + 0xce5f, // (12, 3679) 166 + 0xb0af, // (11, 175) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 1 of 22) (steady 1 of 16) (phase = 0.093750000 = 3.0 / 32.0) + // entropy: 4.4574755684414029133 + // avg_length: 4.5336306265208552446; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0xa177, // (10, 375) 0 + 0x5009, // ( 5, 9) 1 + 0x803b, // ( 8, 59) 2 + 0x3002, // ( 3, 2) 3 + 0x9017, // ( 9, 23) 4 + 0x5019, // ( 5, 25) 5 + 0x700b, // ( 7, 11) 6 + 0x2000, // ( 2, 0) 7 + 0xb34f, // (11, 847) 8 + 0x601d, // ( 6, 29) 9 + 0x9117, // ( 9, 279) 10 + 0x4001, // ( 4, 1) 11 + 0xa377, // (10, 887) 12 + 0x603d, // ( 6, 61) 13 + 0x80bb, // ( 8, 187) 14 + 0x3006, // ( 3, 6) 15 + 0xc4af, // (12, 1199) 16 + 0x704b, // ( 7, 75) 17 + 0xa0f7, // (10, 247) 18 + 0x5005, // ( 5, 5) 19 + 0xb74f, // (11, 1871) 20 + 0x702b, // ( 7, 43) 21 + 0x9097, // ( 9, 151) 22 + 0x5015, // ( 5, 21) 23 + 0xccaf, // (12, 3247) 24 + 0x807b, // ( 8, 123) 25 + 0xb0cf, // (11, 207) 26 + 0x6003, // ( 6, 3) 27 + 0xc2af, // (12, 687) 28 + 0x80fb, // ( 8, 251) 29 + 0xa2f7, // (10, 759) 30 + 0x500d, // ( 5, 13) 31 + 0xcaaf, // (12, 2735) 32 + 0x8007, // ( 8, 7) 33 + 0xb4cf, // (11, 1231) 34 + 0x6023, // ( 6, 35) 35 + 0xc6af, // (12, 1711) 36 + 0x8087, // ( 8, 135) 37 + 0xa1f7, // (10, 503) 38 + 0x6013, // ( 6, 19) 39 + 0xceaf, // (12, 3759) 40 + 0x9197, // ( 9, 407) 41 + 0xc1af, // (12, 431) 42 + 0x706b, // ( 7, 107) 43 + 0xc9af, // (12, 2479) 44 + 0x9057, // ( 9, 87) 45 + 0xb2cf, // (11, 719) 46 + 0x6033, // ( 6, 51) 47 + 0xc5af, // (12, 1455) 48 + 0xa3f7, // (10, 1015) 49 + 0xcdaf, // (12, 3503) 50 + 0x8047, // ( 8, 71) 51 + 0xc3af, // (12, 943) 52 + 0xa00f, // (10, 15) 53 + 0xcbaf, // (12, 2991) 54 + 0x80c7, // ( 8, 199) 55 + 0xc7af, // (12, 1967) 56 + 0xb6cf, // (11, 1743) 57 + 0xcfaf, // (12, 4015) 58 + 0x9157, // ( 9, 343) 59 + 0xc06f, // (12, 111) 60 + 0xb1cf, // (11, 463) 61 + 0xc86f, // (12, 2159) 62 + 0x90d7, // ( 9, 215) 63 + 0xc46f, // (12, 1135) 64 + 0x91d7, // ( 9, 471) 65 + 0xcc6f, // (12, 3183) 66 + 0x701b, // ( 7, 27) 67 + 0xc26f, // (12, 623) 68 + 0x9037, // ( 9, 55) 69 + 0xb5cf, // (11, 1487) 70 + 0x705b, // ( 7, 91) 71 + 0xca6f, // (12, 2671) 72 + 0xa20f, // (10, 527) 73 + 0xc66f, // (12, 1647) 74 + 0x8027, // ( 8, 39) 75 + 0xce6f, // (12, 3695) 76 + 0xa10f, // (10, 271) 77 + 0xc16f, // (12, 367) 78 + 0x80a7, // ( 8, 167) 79 + 0xc96f, // (12, 2415) 80 + 0xb3cf, // (11, 975) 81 + 0xc56f, // (12, 1391) 82 + 0x9137, // ( 9, 311) 83 + 0xcd6f, // (12, 3439) 84 + 0xb7cf, // (11, 1999) 85 + 0xc36f, // (12, 879) 86 + 0x90b7, // ( 9, 183) 87 + 0xcb6f, // (12, 2927) 88 + 0xc76f, // (12, 1903) 89 + 0xcf6f, // (12, 3951) 90 + 0xa30f, // (10, 783) 91 + 0xc0ef, // (12, 239) 92 + 0xc8ef, // (12, 2287) 93 + 0xc4ef, // (12, 1263) 94 + 0xa08f, // (10, 143) 95 + 0xccef, // (12, 3311) 96 + 0xc2ef, // (12, 751) 97 + 0xcaef, // (12, 2799) 98 + 0xa28f, // (10, 655) 99 + 0xc6ef, // (12, 1775) 100 + 0xceef, // (12, 3823) 101 + 0xc1ef, // (12, 495) 102 + 0xa18f, // (10, 399) 103 + 0xc9ef, // (12, 2543) 104 + 0xc5ef, // (12, 1519) 105 + 0xcdef, // (12, 3567) 106 + 0xb02f, // (11, 47) 107 + 0xc3ef, // (12, 1007) 108 + 0xcbef, // (12, 3055) 109 + 0xc7ef, // (12, 2031) 110 + 0xb42f, // (11, 1071) 111 + 0xcfef, // (12, 4079) 112 + 0xc01f, // (12, 31) 113 + 0xc81f, // (12, 2079) 114 + 0xc41f, // (12, 1055) 115 + 0xcc1f, // (12, 3103) 116 + 0xc21f, // (12, 543) 117 + 0xca1f, // (12, 2591) 118 + 0xc61f, // (12, 1567) 119 + 0xce1f, // (12, 3615) 120 + 0xc11f, // (12, 287) 121 + 0xc91f, // (12, 2335) 122 + 0xc51f, // (12, 1311) 123 + 0xcd1f, // (12, 3359) 124 + 0xc31f, // (12, 799) 125 + 0xcb1f, // (12, 2847) 126 + 0xc71f, // (12, 1823) 127 + 0xcf1f, // (12, 3871) 128 + 0xa38f, // (10, 911) 129 + 0xc09f, // (12, 159) 130 + 0x8067, // ( 8, 103) 131 + 0xc89f, // (12, 2207) 132 + 0xa04f, // (10, 79) 133 + 0xc49f, // (12, 1183) 134 + 0x80e7, // ( 8, 231) 135 + 0xcc9f, // (12, 3231) 136 + 0xb22f, // (11, 559) 137 + 0xc29f, // (12, 671) 138 + 0x91b7, // ( 9, 439) 139 + 0xca9f, // (12, 2719) 140 + 0xb62f, // (11, 1583) 141 + 0xc69f, // (12, 1695) 142 + 0x9077, // ( 9, 119) 143 + 0xce9f, // (12, 3743) 144 + 0xc19f, // (12, 415) 145 + 0xc99f, // (12, 2463) 146 + 0xa24f, // (10, 591) 147 + 0xc59f, // (12, 1439) 148 + 0xcd9f, // (12, 3487) 149 + 0xc39f, // (12, 927) 150 + 0xa14f, // (10, 335) 151 + 0xcb9f, // (12, 2975) 152 + 0xc79f, // (12, 1951) 153 + 0xcf9f, // (12, 3999) 154 + 0xb12f, // (11, 303) 155 + 0xc05f, // (12, 95) 156 + 0xc85f, // (12, 2143) 157 + 0xc45f, // (12, 1119) 158 + 0xb52f, // (11, 1327) 159 + 0xcc5f, // (12, 3167) 160 + 0xc25f, // (12, 607) 161 + 0xca5f, // (12, 2655) 162 + 0xb32f, // (11, 815) 163 + 0xc65f, // (12, 1631) 164 + 0xce5f, // (12, 3679) 165 + 0xc15f, // (12, 351) 166 + 0xb72f, // (11, 1839) 167 + 0xc95f, // (12, 2399) 168 + 0xc55f, // (12, 1375) 169 + 0xcd5f, // (12, 3423) 170 + 0xc35f, // (12, 863) 171 + 0xcb5f, // (12, 2911) 172 + 0xc75f, // (12, 1887) 173 + 0xcf5f, // (12, 3935) 174 + 0xb0af, // (11, 175) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 2 of 22) (steady 2 of 16) (phase = 0.156250000 = 5.0 / 32.0) + // entropy: 4.4520619712441886762 + // avg_length: 4.5253989110544479146; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0xa177, // (10, 375) 0 + 0x5009, // ( 5, 9) 1 + 0x803b, // ( 8, 59) 2 + 0x3002, // ( 3, 2) 3 + 0xa377, // (10, 887) 4 + 0x5019, // ( 5, 25) 5 + 0x80bb, // ( 8, 187) 6 + 0x2000, // ( 2, 0) 7 + 0xb34f, // (11, 847) 8 + 0x601d, // ( 6, 29) 9 + 0x9057, // ( 9, 87) 10 + 0x4001, // ( 4, 1) 11 + 0xb74f, // (11, 1871) 12 + 0x603d, // ( 6, 61) 13 + 0x807b, // ( 8, 123) 14 + 0x3006, // ( 3, 6) 15 + 0xc72f, // (12, 1839) 16 + 0x700b, // ( 7, 11) 17 + 0xa0f7, // (10, 247) 18 + 0x5005, // ( 5, 5) 19 + 0xcf2f, // (12, 3887) 20 + 0x704b, // ( 7, 75) 21 + 0xa2f7, // (10, 759) 22 + 0x5015, // ( 5, 21) 23 + 0xc0af, // (12, 175) 24 + 0x80fb, // ( 8, 251) 25 + 0xb0cf, // (11, 207) 26 + 0x6003, // ( 6, 3) 27 + 0xc8af, // (12, 2223) 28 + 0x8007, // ( 8, 7) 29 + 0xa1f7, // (10, 503) 30 + 0x500d, // ( 5, 13) 31 + 0xc4af, // (12, 1199) 32 + 0x8087, // ( 8, 135) 33 + 0xb4cf, // (11, 1231) 34 + 0x6023, // ( 6, 35) 35 + 0xccaf, // (12, 3247) 36 + 0x8047, // ( 8, 71) 37 + 0xb2cf, // (11, 719) 38 + 0x6013, // ( 6, 19) 39 + 0xc2af, // (12, 687) 40 + 0x9157, // ( 9, 343) 41 + 0xcaaf, // (12, 2735) 42 + 0x702b, // ( 7, 43) 43 + 0xc6af, // (12, 1711) 44 + 0x90d7, // ( 9, 215) 45 + 0xceaf, // (12, 3759) 46 + 0x6033, // ( 6, 51) 47 + 0xc1af, // (12, 431) 48 + 0xa3f7, // (10, 1015) 49 + 0xc9af, // (12, 2479) 50 + 0x80c7, // ( 8, 199) 51 + 0xc5af, // (12, 1455) 52 + 0xa00f, // (10, 15) 53 + 0xcdaf, // (12, 3503) 54 + 0x8027, // ( 8, 39) 55 + 0xc3af, // (12, 943) 56 + 0xb6cf, // (11, 1743) 57 + 0xcbaf, // (12, 2991) 58 + 0x91d7, // ( 9, 471) 59 + 0xc7af, // (12, 1967) 60 + 0xb1cf, // (11, 463) 61 + 0xcfaf, // (12, 4015) 62 + 0x80a7, // ( 8, 167) 63 + 0xc06f, // (12, 111) 64 + 0x9037, // ( 9, 55) 65 + 0xc86f, // (12, 2159) 66 + 0x706b, // ( 7, 107) 67 + 0xc46f, // (12, 1135) 68 + 0x9137, // ( 9, 311) 69 + 0xcc6f, // (12, 3183) 70 + 0x701b, // ( 7, 27) 71 + 0xc26f, // (12, 623) 72 + 0xa20f, // (10, 527) 73 + 0xca6f, // (12, 2671) 74 + 0x8067, // ( 8, 103) 75 + 0xc66f, // (12, 1647) 76 + 0xa10f, // (10, 271) 77 + 0xce6f, // (12, 3695) 78 + 0x705b, // ( 7, 91) 79 + 0xc16f, // (12, 367) 80 + 0xb5cf, // (11, 1487) 81 + 0xc96f, // (12, 2415) 82 + 0x90b7, // ( 9, 183) 83 + 0xc56f, // (12, 1391) 84 + 0xb3cf, // (11, 975) 85 + 0xcd6f, // (12, 3439) 86 + 0x91b7, // ( 9, 439) 87 + 0xc36f, // (12, 879) 88 + 0xcb6f, // (12, 2927) 89 + 0xc76f, // (12, 1903) 90 + 0xa30f, // (10, 783) 91 + 0xcf6f, // (12, 3951) 92 + 0xc0ef, // (12, 239) 93 + 0xc8ef, // (12, 2287) 94 + 0xa08f, // (10, 143) 95 + 0xc4ef, // (12, 1263) 96 + 0xccef, // (12, 3311) 97 + 0xc2ef, // (12, 751) 98 + 0xa28f, // (10, 655) 99 + 0xcaef, // (12, 2799) 100 + 0xc6ef, // (12, 1775) 101 + 0xceef, // (12, 3823) 102 + 0xa18f, // (10, 399) 103 + 0xc1ef, // (12, 495) 104 + 0xc9ef, // (12, 2543) 105 + 0xc5ef, // (12, 1519) 106 + 0xb7cf, // (11, 1999) 107 + 0xcdef, // (12, 3567) 108 + 0xc3ef, // (12, 1007) 109 + 0xcbef, // (12, 3055) 110 + 0xb02f, // (11, 47) 111 + 0xc7ef, // (12, 2031) 112 + 0xcfef, // (12, 4079) 113 + 0xc01f, // (12, 31) 114 + 0xc81f, // (12, 2079) 115 + 0xc41f, // (12, 1055) 116 + 0xcc1f, // (12, 3103) 117 + 0xc21f, // (12, 543) 118 + 0xca1f, // (12, 2591) 119 + 0xc61f, // (12, 1567) 120 + 0xce1f, // (12, 3615) 121 + 0xc11f, // (12, 287) 122 + 0xc91f, // (12, 2335) 123 + 0xc51f, // (12, 1311) 124 + 0xcd1f, // (12, 3359) 125 + 0xc31f, // (12, 799) 126 + 0xcb1f, // (12, 2847) 127 + 0xc71f, // (12, 1823) 128 + 0xa38f, // (10, 911) 129 + 0xcf1f, // (12, 3871) 130 + 0x80e7, // ( 8, 231) 131 + 0xc09f, // (12, 159) 132 + 0xa04f, // (10, 79) 133 + 0xc89f, // (12, 2207) 134 + 0x8017, // ( 8, 23) 135 + 0xc49f, // (12, 1183) 136 + 0xb42f, // (11, 1071) 137 + 0xcc9f, // (12, 3231) 138 + 0x9077, // ( 9, 119) 139 + 0xc29f, // (12, 671) 140 + 0xb22f, // (11, 559) 141 + 0xca9f, // (12, 2719) 142 + 0x8097, // ( 8, 151) 143 + 0xc69f, // (12, 1695) 144 + 0xce9f, // (12, 3743) 145 + 0xc19f, // (12, 415) 146 + 0xa24f, // (10, 591) 147 + 0xc99f, // (12, 2463) 148 + 0xc59f, // (12, 1439) 149 + 0xcd9f, // (12, 3487) 150 + 0xa14f, // (10, 335) 151 + 0xc39f, // (12, 927) 152 + 0xcb9f, // (12, 2975) 153 + 0xc79f, // (12, 1951) 154 + 0xb62f, // (11, 1583) 155 + 0xcf9f, // (12, 3999) 156 + 0xc05f, // (12, 95) 157 + 0xc85f, // (12, 2143) 158 + 0xb12f, // (11, 303) 159 + 0xc45f, // (12, 1119) 160 + 0xcc5f, // (12, 3167) 161 + 0xc25f, // (12, 607) 162 + 0xb52f, // (11, 1327) 163 + 0xca5f, // (12, 2655) 164 + 0xc65f, // (12, 1631) 165 + 0xce5f, // (12, 3679) 166 + 0xb32f, // (11, 815) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 3 of 22) (steady 3 of 16) (phase = 0.218750000 = 7.0 / 32.0) + // entropy: 4.4457680500675866853 + // avg_length: 4.5181192844586535173; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0xb24f, // (11, 591) 0 + 0x601d, // ( 6, 29) 1 + 0x9097, // ( 9, 151) 2 + 0x3002, // ( 3, 2) 3 + 0xa1f7, // (10, 503) 4 + 0x5005, // ( 5, 5) 5 + 0x807b, // ( 8, 123) 6 + 0x2000, // ( 2, 0) 7 + 0xc52f, // (12, 1327) 8 + 0x603d, // ( 6, 61) 9 + 0x9197, // ( 9, 407) 10 + 0x4001, // ( 4, 1) 11 + 0xb64f, // (11, 1615) 12 + 0x6003, // ( 6, 3) 13 + 0x9057, // ( 9, 87) 14 + 0x3006, // ( 3, 6) 15 + 0xcd2f, // (12, 3375) 16 + 0x80fb, // ( 8, 251) 17 + 0xb14f, // (11, 335) 18 + 0x5015, // ( 5, 21) 19 + 0xc32f, // (12, 815) 20 + 0x702b, // ( 7, 43) 21 + 0xa3f7, // (10, 1015) 22 + 0x4009, // ( 4, 9) 23 + 0xcb2f, // (12, 2863) 24 + 0x8007, // ( 8, 7) 25 + 0xb54f, // (11, 1359) 26 + 0x6023, // ( 6, 35) 27 + 0xc72f, // (12, 1839) 28 + 0x8087, // ( 8, 135) 29 + 0xb34f, // (11, 847) 30 + 0x500d, // ( 5, 13) 31 + 0xcf2f, // (12, 3887) 32 + 0x9157, // ( 9, 343) 33 + 0xc0af, // (12, 175) 34 + 0x6013, // ( 6, 19) 35 + 0xc8af, // (12, 2223) 36 + 0x8047, // ( 8, 71) 37 + 0xb74f, // (11, 1871) 38 + 0x6033, // ( 6, 51) 39 + 0xc4af, // (12, 1199) 40 + 0x90d7, // ( 9, 215) 41 + 0xccaf, // (12, 3247) 42 + 0x706b, // ( 7, 107) 43 + 0xc2af, // (12, 687) 44 + 0x91d7, // ( 9, 471) 45 + 0xcaaf, // (12, 2735) 46 + 0x600b, // ( 6, 11) 47 + 0xc6af, // (12, 1711) 48 + 0xb0cf, // (11, 207) 49 + 0xceaf, // (12, 3759) 50 + 0x80c7, // ( 8, 199) 51 + 0xc1af, // (12, 431) 52 + 0xa00f, // (10, 15) 53 + 0xc9af, // (12, 2479) 54 + 0x8027, // ( 8, 39) 55 + 0xc5af, // (12, 1455) 56 + 0xb4cf, // (11, 1231) 57 + 0xcdaf, // (12, 3503) 58 + 0x9037, // ( 9, 55) 59 + 0xc3af, // (12, 943) 60 + 0xb2cf, // (11, 719) 61 + 0xcbaf, // (12, 2991) 62 + 0x80a7, // ( 8, 167) 63 + 0xc7af, // (12, 1967) 64 + 0xa20f, // (10, 527) 65 + 0xcfaf, // (12, 4015) 66 + 0x701b, // ( 7, 27) 67 + 0xc06f, // (12, 111) 68 + 0x9137, // ( 9, 311) 69 + 0xc86f, // (12, 2159) 70 + 0x705b, // ( 7, 91) 71 + 0xc46f, // (12, 1135) 72 + 0xb6cf, // (11, 1743) 73 + 0xcc6f, // (12, 3183) 74 + 0x8067, // ( 8, 103) 75 + 0xc26f, // (12, 623) 76 + 0xa10f, // (10, 271) 77 + 0xca6f, // (12, 2671) 78 + 0x703b, // ( 7, 59) 79 + 0xc66f, // (12, 1647) 80 + 0xce6f, // (12, 3695) 81 + 0xc16f, // (12, 367) 82 + 0x90b7, // ( 9, 183) 83 + 0xc96f, // (12, 2415) 84 + 0xb1cf, // (11, 463) 85 + 0xc56f, // (12, 1391) 86 + 0x91b7, // ( 9, 439) 87 + 0xcd6f, // (12, 3439) 88 + 0xc36f, // (12, 879) 89 + 0xcb6f, // (12, 2927) 90 + 0xa30f, // (10, 783) 91 + 0xc76f, // (12, 1903) 92 + 0xcf6f, // (12, 3951) 93 + 0xc0ef, // (12, 239) 94 + 0x9077, // ( 9, 119) 95 + 0xc8ef, // (12, 2287) 96 + 0xc4ef, // (12, 1263) 97 + 0xccef, // (12, 3311) 98 + 0xa08f, // (10, 143) 99 + 0xc2ef, // (12, 751) 100 + 0xcaef, // (12, 2799) 101 + 0xc6ef, // (12, 1775) 102 + 0xa28f, // (10, 655) 103 + 0xceef, // (12, 3823) 104 + 0xc1ef, // (12, 495) 105 + 0xc9ef, // (12, 2543) 106 + 0xb5cf, // (11, 1487) 107 + 0xc5ef, // (12, 1519) 108 + 0xcdef, // (12, 3567) 109 + 0xc3ef, // (12, 1007) 110 + 0xb3cf, // (11, 975) 111 + 0xcbef, // (12, 3055) 112 + 0xc7ef, // (12, 2031) 113 + 0xcfef, // (12, 4079) 114 + 0xc01f, // (12, 31) 115 + 0xc81f, // (12, 2079) 116 + 0xc41f, // (12, 1055) 117 + 0xcc1f, // (12, 3103) 118 + 0xc21f, // (12, 543) 119 + 0xca1f, // (12, 2591) 120 + 0xc61f, // (12, 1567) 121 + 0xce1f, // (12, 3615) 122 + 0xc11f, // (12, 287) 123 + 0xc91f, // (12, 2335) 124 + 0xc51f, // (12, 1311) 125 + 0xcd1f, // (12, 3359) 126 + 0xc31f, // (12, 799) 127 + 0xcb1f, // (12, 2847) 128 + 0xb7cf, // (11, 1999) 129 + 0xc71f, // (12, 1823) 130 + 0x80e7, // ( 8, 231) 131 + 0xcf1f, // (12, 3871) 132 + 0xa18f, // (10, 399) 133 + 0xc09f, // (12, 159) 134 + 0x8017, // ( 8, 23) 135 + 0xc89f, // (12, 2207) 136 + 0xc49f, // (12, 1183) 137 + 0xcc9f, // (12, 3231) 138 + 0x9177, // ( 9, 375) 139 + 0xc29f, // (12, 671) 140 + 0xb02f, // (11, 47) 141 + 0xca9f, // (12, 2719) 142 + 0x90f7, // ( 9, 247) 143 + 0xc69f, // (12, 1695) 144 + 0xce9f, // (12, 3743) 145 + 0xc19f, // (12, 415) 146 + 0xa38f, // (10, 911) 147 + 0xc99f, // (12, 2463) 148 + 0xc59f, // (12, 1439) 149 + 0xcd9f, // (12, 3487) 150 + 0xa04f, // (10, 79) 151 + 0xc39f, // (12, 927) 152 + 0xcb9f, // (12, 2975) 153 + 0xc79f, // (12, 1951) 154 + 0xb42f, // (11, 1071) 155 + 0xcf9f, // (12, 3999) 156 + 0xc05f, // (12, 95) 157 + 0xc85f, // (12, 2143) 158 + 0xb22f, // (11, 559) 159 + 0xc45f, // (12, 1119) 160 + 0xcc5f, // (12, 3167) 161 + 0xc25f, // (12, 607) 162 + 0xb62f, // (11, 1583) 163 + 0xca5f, // (12, 2655) 164 + 0xc65f, // (12, 1631) 165 + 0xce5f, // (12, 3679) 166 + 0xb12f, // (11, 303) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 4 of 22) (steady 4 of 16) (phase = 0.281250000 = 9.0 / 32.0) + // entropy: 4.4386754570568340839 + // avg_length: 4.5071584786605640716; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0xb24f, // (11, 591) 0 + 0x601d, // ( 6, 29) 1 + 0x9057, // ( 9, 87) 2 + 0x3002, // ( 3, 2) 3 + 0xb64f, // (11, 1615) 4 + 0x5005, // ( 5, 5) 5 + 0x807b, // ( 8, 123) 6 + 0x2000, // ( 2, 0) 7 + 0xc32f, // (12, 815) 8 + 0x700b, // ( 7, 11) 9 + 0xa0f7, // (10, 247) 10 + 0x4001, // ( 4, 1) 11 + 0xb14f, // (11, 335) 12 + 0x603d, // ( 6, 61) 13 + 0x9157, // ( 9, 343) 14 + 0x3006, // ( 3, 6) 15 + 0xcb2f, // (12, 2863) 16 + 0x80fb, // ( 8, 251) 17 + 0xb54f, // (11, 1359) 18 + 0x5015, // ( 5, 21) 19 + 0xc72f, // (12, 1839) 20 + 0x704b, // ( 7, 75) 21 + 0xa2f7, // (10, 759) 22 + 0x4009, // ( 4, 9) 23 + 0xcf2f, // (12, 3887) 24 + 0x8007, // ( 8, 7) 25 + 0xb34f, // (11, 847) 26 + 0x6003, // ( 6, 3) 27 + 0xc0af, // (12, 175) 28 + 0x8087, // ( 8, 135) 29 + 0xb74f, // (11, 1871) 30 + 0x500d, // ( 5, 13) 31 + 0xc8af, // (12, 2223) 32 + 0x90d7, // ( 9, 215) 33 + 0xc4af, // (12, 1199) 34 + 0x6023, // ( 6, 35) 35 + 0xccaf, // (12, 3247) 36 + 0x8047, // ( 8, 71) 37 + 0xb0cf, // (11, 207) 38 + 0x6013, // ( 6, 19) 39 + 0xc2af, // (12, 687) 40 + 0xa1f7, // (10, 503) 41 + 0xcaaf, // (12, 2735) 42 + 0x702b, // ( 7, 43) 43 + 0xc6af, // (12, 1711) 44 + 0x91d7, // ( 9, 471) 45 + 0xceaf, // (12, 3759) 46 + 0x6033, // ( 6, 51) 47 + 0xc1af, // (12, 431) 48 + 0xb4cf, // (11, 1231) 49 + 0xc9af, // (12, 2479) 50 + 0x80c7, // ( 8, 199) 51 + 0xc5af, // (12, 1455) 52 + 0xa3f7, // (10, 1015) 53 + 0xcdaf, // (12, 3503) 54 + 0x706b, // ( 7, 107) 55 + 0xc3af, // (12, 943) 56 + 0xb2cf, // (11, 719) 57 + 0xcbaf, // (12, 2991) 58 + 0x9037, // ( 9, 55) 59 + 0xc7af, // (12, 1967) 60 + 0xb6cf, // (11, 1743) 61 + 0xcfaf, // (12, 4015) 62 + 0x8027, // ( 8, 39) 63 + 0xc06f, // (12, 111) 64 + 0xa00f, // (10, 15) 65 + 0xc86f, // (12, 2159) 66 + 0x701b, // ( 7, 27) 67 + 0xc46f, // (12, 1135) 68 + 0x9137, // ( 9, 311) 69 + 0xcc6f, // (12, 3183) 70 + 0x705b, // ( 7, 91) 71 + 0xc26f, // (12, 623) 72 + 0xb1cf, // (11, 463) 73 + 0xca6f, // (12, 2671) 74 + 0x80a7, // ( 8, 167) 75 + 0xc66f, // (12, 1647) 76 + 0xa20f, // (10, 527) 77 + 0xce6f, // (12, 3695) 78 + 0x703b, // ( 7, 59) 79 + 0xc16f, // (12, 367) 80 + 0xc96f, // (12, 2415) 81 + 0xc56f, // (12, 1391) 82 + 0x90b7, // ( 9, 183) 83 + 0xcd6f, // (12, 3439) 84 + 0xb5cf, // (11, 1487) 85 + 0xc36f, // (12, 879) 86 + 0x8067, // ( 8, 103) 87 + 0xcb6f, // (12, 2927) 88 + 0xc76f, // (12, 1903) 89 + 0xcf6f, // (12, 3951) 90 + 0xa10f, // (10, 271) 91 + 0xc0ef, // (12, 239) 92 + 0xc8ef, // (12, 2287) 93 + 0xc4ef, // (12, 1263) 94 + 0x91b7, // ( 9, 439) 95 + 0xccef, // (12, 3311) 96 + 0xc2ef, // (12, 751) 97 + 0xcaef, // (12, 2799) 98 + 0xa30f, // (10, 783) 99 + 0xc6ef, // (12, 1775) 100 + 0xceef, // (12, 3823) 101 + 0xc1ef, // (12, 495) 102 + 0xa08f, // (10, 143) 103 + 0xc9ef, // (12, 2543) 104 + 0xc5ef, // (12, 1519) 105 + 0xcdef, // (12, 3567) 106 + 0xb3cf, // (11, 975) 107 + 0xc3ef, // (12, 1007) 108 + 0xcbef, // (12, 3055) 109 + 0xc7ef, // (12, 2031) 110 + 0xa28f, // (10, 655) 111 + 0xcfef, // (12, 4079) 112 + 0xc01f, // (12, 31) 113 + 0xc81f, // (12, 2079) 114 + 0xc41f, // (12, 1055) 115 + 0xcc1f, // (12, 3103) 116 + 0xc21f, // (12, 543) 117 + 0xca1f, // (12, 2591) 118 + 0xb7cf, // (11, 1999) 119 + 0xc61f, // (12, 1567) 120 + 0xce1f, // (12, 3615) 121 + 0xc11f, // (12, 287) 122 + 0xc91f, // (12, 2335) 123 + 0xc51f, // (12, 1311) 124 + 0xcd1f, // (12, 3359) 125 + 0xc31f, // (12, 799) 126 + 0xcb1f, // (12, 2847) 127 + 0xc71f, // (12, 1823) 128 + 0xb02f, // (11, 47) 129 + 0xcf1f, // (12, 3871) 130 + 0x80e7, // ( 8, 231) 131 + 0xc09f, // (12, 159) 132 + 0xa18f, // (10, 399) 133 + 0xc89f, // (12, 2207) 134 + 0x8017, // ( 8, 23) 135 + 0xc49f, // (12, 1183) 136 + 0xcc9f, // (12, 3231) 137 + 0xc29f, // (12, 671) 138 + 0x9077, // ( 9, 119) 139 + 0xca9f, // (12, 2719) 140 + 0xb42f, // (11, 1071) 141 + 0xc69f, // (12, 1695) 142 + 0x8097, // ( 8, 151) 143 + 0xce9f, // (12, 3743) 144 + 0xc19f, // (12, 415) 145 + 0xc99f, // (12, 2463) 146 + 0xa38f, // (10, 911) 147 + 0xc59f, // (12, 1439) 148 + 0xcd9f, // (12, 3487) 149 + 0xc39f, // (12, 927) 150 + 0x9177, // ( 9, 375) 151 + 0xcb9f, // (12, 2975) 152 + 0xc79f, // (12, 1951) 153 + 0xcf9f, // (12, 3999) 154 + 0xb22f, // (11, 559) 155 + 0xc05f, // (12, 95) 156 + 0xc85f, // (12, 2143) 157 + 0xc45f, // (12, 1119) 158 + 0xa04f, // (10, 79) 159 + 0xcc5f, // (12, 3167) 160 + 0xc25f, // (12, 607) 161 + 0xca5f, // (12, 2655) 162 + 0xb62f, // (11, 1583) 163 + 0xc65f, // (12, 1631) 164 + 0xce5f, // (12, 3679) 165 + 0xc15f, // (12, 351) 166 + 0xb12f, // (11, 303) 167 + 0xc95f, // (12, 2399) 168 + 0xc55f, // (12, 1375) 169 + 0xcd5f, // (12, 3423) 170 + 0xc35f, // (12, 863) 171 + 0xcb5f, // (12, 2911) 172 + 0xc75f, // (12, 1887) 173 + 0xcf5f, // (12, 3935) 174 + 0xb52f, // (11, 1327) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 5 of 22) (steady 5 of 16) (phase = 0.343750000 = 11.0 / 32.0) + // entropy: 4.4308578632493116345 + // avg_length: 4.4996166821663301505; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0xc12f, // (12, 303) 0 + 0x601d, // ( 6, 29) 1 + 0x9057, // ( 9, 87) 2 + 0x3002, // ( 3, 2) 3 + 0xb14f, // (11, 335) 4 + 0x5005, // ( 5, 5) 5 + 0x807b, // ( 8, 123) 6 + 0x2000, // ( 2, 0) 7 + 0xc92f, // (12, 2351) 8 + 0x700b, // ( 7, 11) 9 + 0xa1f7, // (10, 503) 10 + 0x4001, // ( 4, 1) 11 + 0xc52f, // (12, 1327) 12 + 0x603d, // ( 6, 61) 13 + 0x9157, // ( 9, 343) 14 + 0x3006, // ( 3, 6) 15 + 0xcd2f, // (12, 3375) 16 + 0x80fb, // ( 8, 251) 17 + 0xb54f, // (11, 1359) 18 + 0x5015, // ( 5, 21) 19 + 0xc32f, // (12, 815) 20 + 0x704b, // ( 7, 75) 21 + 0xa3f7, // (10, 1015) 22 + 0x4009, // ( 4, 9) 23 + 0xcb2f, // (12, 2863) 24 + 0x8007, // ( 8, 7) 25 + 0xc72f, // (12, 1839) 26 + 0x6003, // ( 6, 3) 27 + 0xcf2f, // (12, 3887) 28 + 0x8087, // ( 8, 135) 29 + 0xb34f, // (11, 847) 30 + 0x500d, // ( 5, 13) 31 + 0xc0af, // (12, 175) 32 + 0x90d7, // ( 9, 215) 33 + 0xc8af, // (12, 2223) 34 + 0x6023, // ( 6, 35) 35 + 0xc4af, // (12, 1199) 36 + 0x8047, // ( 8, 71) 37 + 0xb74f, // (11, 1871) 38 + 0x6013, // ( 6, 19) 39 + 0xccaf, // (12, 3247) 40 + 0xa00f, // (10, 15) 41 + 0xc2af, // (12, 687) 42 + 0x702b, // ( 7, 43) 43 + 0xcaaf, // (12, 2735) 44 + 0x91d7, // ( 9, 471) 45 + 0xc6af, // (12, 1711) 46 + 0x6033, // ( 6, 51) 47 + 0xceaf, // (12, 3759) 48 + 0xb0cf, // (11, 207) 49 + 0xc1af, // (12, 431) 50 + 0x80c7, // ( 8, 199) 51 + 0xc9af, // (12, 2479) 52 + 0xa20f, // (10, 527) 53 + 0xc5af, // (12, 1455) 54 + 0x706b, // ( 7, 107) 55 + 0xcdaf, // (12, 3503) 56 + 0xc3af, // (12, 943) 57 + 0xcbaf, // (12, 2991) 58 + 0x9037, // ( 9, 55) 59 + 0xc7af, // (12, 1967) 60 + 0xb4cf, // (11, 1231) 61 + 0xcfaf, // (12, 4015) 62 + 0x8027, // ( 8, 39) 63 + 0xc06f, // (12, 111) 64 + 0xa10f, // (10, 271) 65 + 0xc86f, // (12, 2159) 66 + 0x701b, // ( 7, 27) 67 + 0xc46f, // (12, 1135) 68 + 0x9137, // ( 9, 311) 69 + 0xcc6f, // (12, 3183) 70 + 0x705b, // ( 7, 91) 71 + 0xc26f, // (12, 623) 72 + 0xb2cf, // (11, 719) 73 + 0xca6f, // (12, 2671) 74 + 0x80a7, // ( 8, 167) 75 + 0xc66f, // (12, 1647) 76 + 0xa30f, // (10, 783) 77 + 0xce6f, // (12, 3695) 78 + 0x703b, // ( 7, 59) 79 + 0xc16f, // (12, 367) 80 + 0xc96f, // (12, 2415) 81 + 0xc56f, // (12, 1391) 82 + 0x90b7, // ( 9, 183) 83 + 0xcd6f, // (12, 3439) 84 + 0xb6cf, // (11, 1743) 85 + 0xc36f, // (12, 879) 86 + 0x8067, // ( 8, 103) 87 + 0xcb6f, // (12, 2927) 88 + 0xc76f, // (12, 1903) 89 + 0xcf6f, // (12, 3951) 90 + 0xa08f, // (10, 143) 91 + 0xc0ef, // (12, 239) 92 + 0xc8ef, // (12, 2287) 93 + 0xc4ef, // (12, 1263) 94 + 0x91b7, // ( 9, 439) 95 + 0xccef, // (12, 3311) 96 + 0xc2ef, // (12, 751) 97 + 0xcaef, // (12, 2799) 98 + 0xa28f, // (10, 655) 99 + 0xc6ef, // (12, 1775) 100 + 0xceef, // (12, 3823) 101 + 0xc1ef, // (12, 495) 102 + 0x9077, // ( 9, 119) 103 + 0xc9ef, // (12, 2543) 104 + 0xc5ef, // (12, 1519) 105 + 0xcdef, // (12, 3567) 106 + 0xb1cf, // (11, 463) 107 + 0xc3ef, // (12, 1007) 108 + 0xcbef, // (12, 3055) 109 + 0xc7ef, // (12, 2031) 110 + 0xa18f, // (10, 399) 111 + 0xcfef, // (12, 4079) 112 + 0xc01f, // (12, 31) 113 + 0xc81f, // (12, 2079) 114 + 0xc41f, // (12, 1055) 115 + 0xcc1f, // (12, 3103) 116 + 0xc21f, // (12, 543) 117 + 0xca1f, // (12, 2591) 118 + 0xb5cf, // (11, 1487) 119 + 0xc61f, // (12, 1567) 120 + 0xce1f, // (12, 3615) 121 + 0xc11f, // (12, 287) 122 + 0xc91f, // (12, 2335) 123 + 0xc51f, // (12, 1311) 124 + 0xcd1f, // (12, 3359) 125 + 0xc31f, // (12, 799) 126 + 0xcb1f, // (12, 2847) 127 + 0xc71f, // (12, 1823) 128 + 0xb3cf, // (11, 975) 129 + 0xcf1f, // (12, 3871) 130 + 0x80e7, // ( 8, 231) 131 + 0xc09f, // (12, 159) 132 + 0xa38f, // (10, 911) 133 + 0xc89f, // (12, 2207) 134 + 0x8017, // ( 8, 23) 135 + 0xc49f, // (12, 1183) 136 + 0xcc9f, // (12, 3231) 137 + 0xc29f, // (12, 671) 138 + 0x9177, // ( 9, 375) 139 + 0xca9f, // (12, 2719) 140 + 0xb7cf, // (11, 1999) 141 + 0xc69f, // (12, 1695) 142 + 0x8097, // ( 8, 151) 143 + 0xce9f, // (12, 3743) 144 + 0xc19f, // (12, 415) 145 + 0xc99f, // (12, 2463) 146 + 0xa04f, // (10, 79) 147 + 0xc59f, // (12, 1439) 148 + 0xcd9f, // (12, 3487) 149 + 0xc39f, // (12, 927) 150 + 0x90f7, // ( 9, 247) 151 + 0xcb9f, // (12, 2975) 152 + 0xc79f, // (12, 1951) 153 + 0xcf9f, // (12, 3999) 154 + 0xb02f, // (11, 47) 155 + 0xc05f, // (12, 95) 156 + 0xc85f, // (12, 2143) 157 + 0xc45f, // (12, 1119) 158 + 0xa24f, // (10, 591) 159 + 0xcc5f, // (12, 3167) 160 + 0xc25f, // (12, 607) 161 + 0xca5f, // (12, 2655) 162 + 0xb42f, // (11, 1071) 163 + 0xc65f, // (12, 1631) 164 + 0xce5f, // (12, 3679) 165 + 0xc15f, // (12, 351) 166 + 0xb22f, // (11, 559) 167 + 0xc95f, // (12, 2399) 168 + 0xc55f, // (12, 1375) 169 + 0xcd5f, // (12, 3423) 170 + 0xc35f, // (12, 863) 171 + 0xcb5f, // (12, 2911) 172 + 0xc75f, // (12, 1887) 173 + 0xcf5f, // (12, 3935) 174 + 0xb62f, // (11, 1583) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 6 of 22) (steady 6 of 16) (phase = 0.406250000 = 13.0 / 32.0) + // entropy: 4.4310364988500126060 + // avg_length: 4.5051134111084252254; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x601d, // ( 6, 29) 0 + 0x3002, // ( 3, 2) 1 + 0x603d, // ( 6, 61) 2 + 0x2000, // ( 2, 0) 3 + 0x700b, // ( 7, 11) 4 + 0x4001, // ( 4, 1) 5 + 0x6003, // ( 6, 3) 6 + 0x3006, // ( 3, 6) 7 + 0x807b, // ( 8, 123) 8 + 0x5005, // ( 5, 5) 9 + 0x704b, // ( 7, 75) 10 + 0x4009, // ( 4, 9) 11 + 0x9097, // ( 9, 151) 12 + 0x6023, // ( 6, 35) 13 + 0x80fb, // ( 8, 251) 14 + 0x5015, // ( 5, 21) 15 + 0x9197, // ( 9, 407) 16 + 0x6013, // ( 6, 19) 17 + 0x8007, // ( 8, 7) 18 + 0x500d, // ( 5, 13) 19 + 0xa0f7, // (10, 247) 20 + 0x702b, // ( 7, 43) 21 + 0x9057, // ( 9, 87) 22 + 0x6033, // ( 6, 51) 23 + 0xb14f, // (11, 335) 24 + 0x8087, // ( 8, 135) 25 + 0xa2f7, // (10, 759) 26 + 0x706b, // ( 7, 107) 27 + 0xb54f, // (11, 1359) 28 + 0x9157, // ( 9, 343) 29 + 0xa1f7, // (10, 503) 30 + 0x8047, // ( 8, 71) 31 + 0xa3f7, // (10, 1015) 32 + 0x701b, // ( 7, 27) 33 + 0x90d7, // ( 9, 215) 34 + 0x705b, // ( 7, 91) 35 + 0xb34f, // (11, 847) 36 + 0x80c7, // ( 8, 199) 37 + 0xa00f, // (10, 15) 38 + 0x703b, // ( 7, 59) 39 + 0xc32f, // (12, 815) 40 + 0x91d7, // ( 9, 471) 41 + 0xb74f, // (11, 1871) 42 + 0x8027, // ( 8, 39) 43 + 0xcb2f, // (12, 2863) 44 + 0xa20f, // (10, 527) 45 + 0xb0cf, // (11, 207) 46 + 0x9037, // ( 9, 55) 47 + 0xc72f, // (12, 1839) 48 + 0xa10f, // (10, 271) 49 + 0xcf2f, // (12, 3887) 50 + 0x9137, // ( 9, 311) 51 + 0xc0af, // (12, 175) 52 + 0xb4cf, // (11, 1231) 53 + 0xc8af, // (12, 2223) 54 + 0xa30f, // (10, 783) 55 + 0xc4af, // (12, 1199) 56 + 0xccaf, // (12, 3247) 57 + 0xc2af, // (12, 687) 58 + 0xb2cf, // (11, 719) 59 + 0xcaaf, // (12, 2735) 60 + 0xc6af, // (12, 1711) 61 + 0xceaf, // (12, 3759) 62 + 0xb6cf, // (11, 1743) 63 + 0xb1cf, // (11, 463) 64 + 0x80a7, // ( 8, 167) 65 + 0xa08f, // (10, 143) 66 + 0x8067, // ( 8, 103) 67 + 0xc1af, // (12, 431) 68 + 0x90b7, // ( 9, 183) 69 + 0xb5cf, // (11, 1487) 70 + 0x80e7, // ( 8, 231) 71 + 0xc9af, // (12, 2479) 72 + 0xa28f, // (10, 655) 73 + 0xc5af, // (12, 1455) 74 + 0x91b7, // ( 9, 439) 75 + 0xcdaf, // (12, 3503) 76 + 0xb3cf, // (11, 975) 77 + 0xc3af, // (12, 943) 78 + 0xa18f, // (10, 399) 79 + 0xcbaf, // (12, 2991) 80 + 0xb7cf, // (11, 1999) 81 + 0xc7af, // (12, 1967) 82 + 0xa38f, // (10, 911) 83 + 0xcfaf, // (12, 4015) 84 + 0xc06f, // (12, 111) 85 + 0xc86f, // (12, 2159) 86 + 0xb02f, // (11, 47) 87 + 0xc46f, // (12, 1135) 88 + 0xcc6f, // (12, 3183) 89 + 0xc26f, // (12, 623) 90 + 0xca6f, // (12, 2671) 91 + 0xc66f, // (12, 1647) 92 + 0xce6f, // (12, 3695) 93 + 0xc16f, // (12, 367) 94 + 0xc96f, // (12, 2415) 95 + 0xc56f, // (12, 1391) 96 + 0xcd6f, // (12, 3439) 97 + 0xc36f, // (12, 879) 98 + 0xb42f, // (11, 1071) 99 + 0xcb6f, // (12, 2927) 100 + 0xc76f, // (12, 1903) 101 + 0xcf6f, // (12, 3951) 102 + 0xc0ef, // (12, 239) 103 + 0xc8ef, // (12, 2287) 104 + 0xc4ef, // (12, 1263) 105 + 0xccef, // (12, 3311) 106 + 0xc2ef, // (12, 751) 107 + 0xcaef, // (12, 2799) 108 + 0xc6ef, // (12, 1775) 109 + 0xceef, // (12, 3823) 110 + 0xc1ef, // (12, 495) 111 + 0xc9ef, // (12, 2543) 112 + 0xc5ef, // (12, 1519) 113 + 0xcdef, // (12, 3567) 114 + 0xc3ef, // (12, 1007) 115 + 0xcbef, // (12, 3055) 116 + 0xc7ef, // (12, 2031) 117 + 0xcfef, // (12, 4079) 118 + 0xc01f, // (12, 31) 119 + 0xc81f, // (12, 2079) 120 + 0xc41f, // (12, 1055) 121 + 0xcc1f, // (12, 3103) 122 + 0xc21f, // (12, 543) 123 + 0xca1f, // (12, 2591) 124 + 0xc61f, // (12, 1567) 125 + 0xce1f, // (12, 3615) 126 + 0xc11f, // (12, 287) 127 + 0xc91f, // (12, 2335) 128 + 0x9077, // ( 9, 119) 129 + 0xb22f, // (11, 559) 130 + 0x8017, // ( 8, 23) 131 + 0xc51f, // (12, 1311) 132 + 0xa04f, // (10, 79) 133 + 0xcd1f, // (12, 3359) 134 + 0x9177, // ( 9, 375) 135 + 0xc31f, // (12, 799) 136 + 0xb62f, // (11, 1583) 137 + 0xcb1f, // (12, 2847) 138 + 0xa24f, // (10, 591) 139 + 0xc71f, // (12, 1823) 140 + 0xcf1f, // (12, 3871) 141 + 0xc09f, // (12, 159) 142 + 0xb12f, // (11, 303) 143 + 0xc89f, // (12, 2207) 144 + 0xc49f, // (12, 1183) 145 + 0xcc9f, // (12, 3231) 146 + 0xb52f, // (11, 1327) 147 + 0xc29f, // (12, 671) 148 + 0xca9f, // (12, 2719) 149 + 0xc69f, // (12, 1695) 150 + 0xce9f, // (12, 3743) 151 + 0xc19f, // (12, 415) 152 + 0xc99f, // (12, 2463) 153 + 0xc59f, // (12, 1439) 154 + 0xcd9f, // (12, 3487) 155 + 0xc39f, // (12, 927) 156 + 0xcb9f, // (12, 2975) 157 + 0xc79f, // (12, 1951) 158 + 0xcf9f, // (12, 3999) 159 + 0xc05f, // (12, 95) 160 + 0xc85f, // (12, 2143) 161 + 0xc45f, // (12, 1119) 162 + 0xcc5f, // (12, 3167) 163 + 0xc25f, // (12, 607) 164 + 0xca5f, // (12, 2655) 165 + 0xc65f, // (12, 1631) 166 + 0xce5f, // (12, 3679) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 7 of 22) (steady 7 of 16) (phase = 0.468750000 = 15.0 / 32.0) + // entropy: 4.4417871821766841123 + // avg_length: 4.5206419191518980583; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x700b, // ( 7, 11) 0 + 0x3002, // ( 3, 2) 1 + 0x601d, // ( 6, 29) 2 + 0x2000, // ( 2, 0) 3 + 0x704b, // ( 7, 75) 4 + 0x4001, // ( 4, 1) 5 + 0x603d, // ( 6, 61) 6 + 0x3006, // ( 3, 6) 7 + 0x8007, // ( 8, 7) 8 + 0x5005, // ( 5, 5) 9 + 0x702b, // ( 7, 43) 10 + 0x4009, // ( 4, 9) 11 + 0x9097, // ( 9, 151) 12 + 0x6003, // ( 6, 3) 13 + 0x8087, // ( 8, 135) 14 + 0x5015, // ( 5, 21) 15 + 0x9197, // ( 9, 407) 16 + 0x6023, // ( 6, 35) 17 + 0x8047, // ( 8, 71) 18 + 0x500d, // ( 5, 13) 19 + 0xa0f7, // (10, 247) 20 + 0x706b, // ( 7, 107) 21 + 0x9057, // ( 9, 87) 22 + 0x6013, // ( 6, 19) 23 + 0xb14f, // (11, 335) 24 + 0x80c7, // ( 8, 199) 25 + 0xa2f7, // (10, 759) 26 + 0x701b, // ( 7, 27) 27 + 0xc52f, // (12, 1327) 28 + 0x9157, // ( 9, 343) 29 + 0xb54f, // (11, 1359) 30 + 0x8027, // ( 8, 39) 31 + 0xa1f7, // (10, 503) 32 + 0x705b, // ( 7, 91) 33 + 0x90d7, // ( 9, 215) 34 + 0x6033, // ( 6, 51) 35 + 0xb34f, // (11, 847) 36 + 0x80a7, // ( 8, 167) 37 + 0xa3f7, // (10, 1015) 38 + 0x703b, // ( 7, 59) 39 + 0xcd2f, // (12, 3375) 40 + 0x91d7, // ( 9, 471) 41 + 0xb74f, // (11, 1871) 42 + 0x8067, // ( 8, 103) 43 + 0xc32f, // (12, 815) 44 + 0xa00f, // (10, 15) 45 + 0xcb2f, // (12, 2863) 46 + 0x9037, // ( 9, 55) 47 + 0xc72f, // (12, 1839) 48 + 0xa20f, // (10, 527) 49 + 0xcf2f, // (12, 3887) 50 + 0x9137, // ( 9, 311) 51 + 0xc0af, // (12, 175) 52 + 0xb0cf, // (11, 207) 53 + 0xc8af, // (12, 2223) 54 + 0xa10f, // (10, 271) 55 + 0xc4af, // (12, 1199) 56 + 0xccaf, // (12, 3247) 57 + 0xc2af, // (12, 687) 58 + 0xb4cf, // (11, 1231) 59 + 0xcaaf, // (12, 2735) 60 + 0xc6af, // (12, 1711) 61 + 0xceaf, // (12, 3759) 62 + 0xb2cf, // (11, 719) 63 + 0xb6cf, // (11, 1743) 64 + 0x80e7, // ( 8, 231) 65 + 0xa30f, // (10, 783) 66 + 0x707b, // ( 7, 123) 67 + 0xc1af, // (12, 431) 68 + 0x90b7, // ( 9, 183) 69 + 0xb1cf, // (11, 463) 70 + 0x8017, // ( 8, 23) 71 + 0xc9af, // (12, 2479) 72 + 0xa08f, // (10, 143) 73 + 0xc5af, // (12, 1455) 74 + 0x91b7, // ( 9, 439) 75 + 0xcdaf, // (12, 3503) 76 + 0xb5cf, // (11, 1487) 77 + 0xc3af, // (12, 943) 78 + 0xa28f, // (10, 655) 79 + 0xcbaf, // (12, 2991) 80 + 0xb3cf, // (11, 975) 81 + 0xc7af, // (12, 1967) 82 + 0xa18f, // (10, 399) 83 + 0xcfaf, // (12, 4015) 84 + 0xc06f, // (12, 111) 85 + 0xc86f, // (12, 2159) 86 + 0xb7cf, // (11, 1999) 87 + 0xc46f, // (12, 1135) 88 + 0xcc6f, // (12, 3183) 89 + 0xc26f, // (12, 623) 90 + 0xca6f, // (12, 2671) 91 + 0xc66f, // (12, 1647) 92 + 0xce6f, // (12, 3695) 93 + 0xc16f, // (12, 367) 94 + 0xc96f, // (12, 2415) 95 + 0xc56f, // (12, 1391) 96 + 0xcd6f, // (12, 3439) 97 + 0xc36f, // (12, 879) 98 + 0xb02f, // (11, 47) 99 + 0xcb6f, // (12, 2927) 100 + 0xc76f, // (12, 1903) 101 + 0xcf6f, // (12, 3951) 102 + 0xc0ef, // (12, 239) 103 + 0xc8ef, // (12, 2287) 104 + 0xc4ef, // (12, 1263) 105 + 0xccef, // (12, 3311) 106 + 0xc2ef, // (12, 751) 107 + 0xcaef, // (12, 2799) 108 + 0xc6ef, // (12, 1775) 109 + 0xceef, // (12, 3823) 110 + 0xc1ef, // (12, 495) 111 + 0xc9ef, // (12, 2543) 112 + 0xc5ef, // (12, 1519) 113 + 0xcdef, // (12, 3567) 114 + 0xc3ef, // (12, 1007) 115 + 0xcbef, // (12, 3055) 116 + 0xc7ef, // (12, 2031) 117 + 0xcfef, // (12, 4079) 118 + 0xc01f, // (12, 31) 119 + 0xc81f, // (12, 2079) 120 + 0xc41f, // (12, 1055) 121 + 0xcc1f, // (12, 3103) 122 + 0xc21f, // (12, 543) 123 + 0xca1f, // (12, 2591) 124 + 0xc61f, // (12, 1567) 125 + 0xce1f, // (12, 3615) 126 + 0xc11f, // (12, 287) 127 + 0xc91f, // (12, 2335) 128 + 0xa38f, // (10, 911) 129 + 0xb42f, // (11, 1071) 130 + 0x9077, // ( 9, 119) 131 + 0xc51f, // (12, 1311) 132 + 0xa04f, // (10, 79) 133 + 0xcd1f, // (12, 3359) 134 + 0x9177, // ( 9, 375) 135 + 0xc31f, // (12, 799) 136 + 0xb22f, // (11, 559) 137 + 0xcb1f, // (12, 2847) 138 + 0xa24f, // (10, 591) 139 + 0xc71f, // (12, 1823) 140 + 0xcf1f, // (12, 3871) 141 + 0xc09f, // (12, 159) 142 + 0xb62f, // (11, 1583) 143 + 0xc89f, // (12, 2207) 144 + 0xc49f, // (12, 1183) 145 + 0xcc9f, // (12, 3231) 146 + 0xb12f, // (11, 303) 147 + 0xc29f, // (12, 671) 148 + 0xca9f, // (12, 2719) 149 + 0xc69f, // (12, 1695) 150 + 0xce9f, // (12, 3743) 151 + 0xc19f, // (12, 415) 152 + 0xc99f, // (12, 2463) 153 + 0xc59f, // (12, 1439) 154 + 0xcd9f, // (12, 3487) 155 + 0xc39f, // (12, 927) 156 + 0xcb9f, // (12, 2975) 157 + 0xc79f, // (12, 1951) 158 + 0xcf9f, // (12, 3999) 159 + 0xc05f, // (12, 95) 160 + 0xc85f, // (12, 2143) 161 + 0xc45f, // (12, 1119) 162 + 0xcc5f, // (12, 3167) 163 + 0xc25f, // (12, 607) 164 + 0xca5f, // (12, 2655) 165 + 0xc65f, // (12, 1631) 166 + 0xce5f, // (12, 3679) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 8 of 22) (steady 8 of 16) (phase = 0.531250000 = 17.0 / 32.0) + // entropy: 4.4505873338397474726 + // avg_length: 4.5270058771550303334; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x7033, // ( 7, 51) 0 + 0x4006, // ( 4, 6) 1 + 0x601d, // ( 6, 29) 2 + 0x2000, // ( 2, 0) 3 + 0x7073, // ( 7, 115) 4 + 0x400e, // ( 4, 14) 5 + 0x603d, // ( 6, 61) 6 + 0x3002, // ( 3, 2) 7 + 0x807b, // ( 8, 123) 8 + 0x5005, // ( 5, 5) 9 + 0x700b, // ( 7, 11) 10 + 0x4001, // ( 4, 1) 11 + 0x9097, // ( 9, 151) 12 + 0x5015, // ( 5, 21) 13 + 0x80fb, // ( 8, 251) 14 + 0x4009, // ( 4, 9) 15 + 0xa0f7, // (10, 247) 16 + 0x6003, // ( 6, 3) 17 + 0x8007, // ( 8, 7) 18 + 0x500d, // ( 5, 13) 19 + 0xa2f7, // (10, 759) 20 + 0x704b, // ( 7, 75) 21 + 0x9197, // ( 9, 407) 22 + 0x6023, // ( 6, 35) 23 + 0xb34f, // (11, 847) 24 + 0x8087, // ( 8, 135) 25 + 0xa1f7, // (10, 503) 26 + 0x702b, // ( 7, 43) 27 + 0xb74f, // (11, 1871) 28 + 0x8047, // ( 8, 71) 29 + 0xa3f7, // (10, 1015) 30 + 0x706b, // ( 7, 107) 31 + 0xb0cf, // (11, 207) 32 + 0x701b, // ( 7, 27) 33 + 0x9057, // ( 9, 87) 34 + 0x6013, // ( 6, 19) 35 + 0xb4cf, // (11, 1231) 36 + 0x80c7, // ( 8, 199) 37 + 0xa00f, // (10, 15) 38 + 0x705b, // ( 7, 91) 39 + 0xc72f, // (12, 1839) 40 + 0x9157, // ( 9, 343) 41 + 0xb2cf, // (11, 719) 42 + 0x8027, // ( 8, 39) 43 + 0xcf2f, // (12, 3887) 44 + 0x90d7, // ( 9, 215) 45 + 0xb6cf, // (11, 1743) 46 + 0x80a7, // ( 8, 167) 47 + 0xc0af, // (12, 175) 48 + 0xa20f, // (10, 527) 49 + 0xc8af, // (12, 2223) 50 + 0x91d7, // ( 9, 471) 51 + 0xc4af, // (12, 1199) 52 + 0xa10f, // (10, 271) 53 + 0xccaf, // (12, 3247) 54 + 0x9037, // ( 9, 55) 55 + 0xc2af, // (12, 687) 56 + 0xcaaf, // (12, 2735) 57 + 0xc6af, // (12, 1711) 58 + 0xb1cf, // (11, 463) 59 + 0xceaf, // (12, 3759) 60 + 0xc1af, // (12, 431) 61 + 0xc9af, // (12, 2479) 62 + 0xb5cf, // (11, 1487) 63 + 0xc5af, // (12, 1455) 64 + 0x8067, // ( 8, 103) 65 + 0xa30f, // (10, 783) 66 + 0x703b, // ( 7, 59) 67 + 0xcdaf, // (12, 3503) 68 + 0x9137, // ( 9, 311) 69 + 0xb3cf, // (11, 975) 70 + 0x80e7, // ( 8, 231) 71 + 0xc3af, // (12, 943) 72 + 0xa08f, // (10, 143) 73 + 0xcbaf, // (12, 2991) 74 + 0x90b7, // ( 9, 183) 75 + 0xc7af, // (12, 1967) 76 + 0xa28f, // (10, 655) 77 + 0xcfaf, // (12, 4015) 78 + 0x91b7, // ( 9, 439) 79 + 0xc06f, // (12, 111) 80 + 0xb7cf, // (11, 1999) 81 + 0xc86f, // (12, 2159) 82 + 0xa18f, // (10, 399) 83 + 0xc46f, // (12, 1135) 84 + 0xb02f, // (11, 47) 85 + 0xcc6f, // (12, 3183) 86 + 0xa38f, // (10, 911) 87 + 0xc26f, // (12, 623) 88 + 0xca6f, // (12, 2671) 89 + 0xc66f, // (12, 1647) 90 + 0xce6f, // (12, 3695) 91 + 0xc16f, // (12, 367) 92 + 0xc96f, // (12, 2415) 93 + 0xc56f, // (12, 1391) 94 + 0xcd6f, // (12, 3439) 95 + 0xc36f, // (12, 879) 96 + 0xcb6f, // (12, 2927) 97 + 0xc76f, // (12, 1903) 98 + 0xb42f, // (11, 1071) 99 + 0xcf6f, // (12, 3951) 100 + 0xc0ef, // (12, 239) 101 + 0xc8ef, // (12, 2287) 102 + 0xb22f, // (11, 559) 103 + 0xc4ef, // (12, 1263) 104 + 0xccef, // (12, 3311) 105 + 0xc2ef, // (12, 751) 106 + 0xcaef, // (12, 2799) 107 + 0xc6ef, // (12, 1775) 108 + 0xceef, // (12, 3823) 109 + 0xc1ef, // (12, 495) 110 + 0xc9ef, // (12, 2543) 111 + 0xc5ef, // (12, 1519) 112 + 0xcdef, // (12, 3567) 113 + 0xc3ef, // (12, 1007) 114 + 0xcbef, // (12, 3055) 115 + 0xc7ef, // (12, 2031) 116 + 0xcfef, // (12, 4079) 117 + 0xc01f, // (12, 31) 118 + 0xc81f, // (12, 2079) 119 + 0xc41f, // (12, 1055) 120 + 0xcc1f, // (12, 3103) 121 + 0xc21f, // (12, 543) 122 + 0xca1f, // (12, 2591) 123 + 0xc61f, // (12, 1567) 124 + 0xce1f, // (12, 3615) 125 + 0xc11f, // (12, 287) 126 + 0xc91f, // (12, 2335) 127 + 0xc51f, // (12, 1311) 128 + 0x9077, // ( 9, 119) 129 + 0xcd1f, // (12, 3359) 130 + 0x8017, // ( 8, 23) 131 + 0xc31f, // (12, 799) 132 + 0xa04f, // (10, 79) 133 + 0xcb1f, // (12, 2847) 134 + 0x9177, // ( 9, 375) 135 + 0xc71f, // (12, 1823) 136 + 0xb62f, // (11, 1583) 137 + 0xcf1f, // (12, 3871) 138 + 0xa24f, // (10, 591) 139 + 0xc09f, // (12, 159) 140 + 0xb12f, // (11, 303) 141 + 0xc89f, // (12, 2207) 142 + 0xa14f, // (10, 335) 143 + 0xc49f, // (12, 1183) 144 + 0xcc9f, // (12, 3231) 145 + 0xc29f, // (12, 671) 146 + 0xb52f, // (11, 1327) 147 + 0xca9f, // (12, 2719) 148 + 0xc69f, // (12, 1695) 149 + 0xce9f, // (12, 3743) 150 + 0xb32f, // (11, 815) 151 + 0xc19f, // (12, 415) 152 + 0xc99f, // (12, 2463) 153 + 0xc59f, // (12, 1439) 154 + 0xcd9f, // (12, 3487) 155 + 0xc39f, // (12, 927) 156 + 0xcb9f, // (12, 2975) 157 + 0xc79f, // (12, 1951) 158 + 0xcf9f, // (12, 3999) 159 + 0xc05f, // (12, 95) 160 + 0xc85f, // (12, 2143) 161 + 0xc45f, // (12, 1119) 162 + 0xcc5f, // (12, 3167) 163 + 0xc25f, // (12, 607) 164 + 0xca5f, // (12, 2655) 165 + 0xc65f, // (12, 1631) 166 + 0xce5f, // (12, 3679) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 9 of 22) (steady 9 of 16) (phase = 0.593750000 = 19.0 / 32.0) + // entropy: 4.4575203029748040606 + // avg_length: 4.5315465600684730063; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x7033, // ( 7, 51) 0 + 0x4006, // ( 4, 6) 1 + 0x601d, // ( 6, 29) 2 + 0x2000, // ( 2, 0) 3 + 0x7073, // ( 7, 115) 4 + 0x400e, // ( 4, 14) 5 + 0x603d, // ( 6, 61) 6 + 0x3002, // ( 3, 2) 7 + 0x9097, // ( 9, 151) 8 + 0x5005, // ( 5, 5) 9 + 0x700b, // ( 7, 11) 10 + 0x4001, // ( 4, 1) 11 + 0x9197, // ( 9, 407) 12 + 0x6003, // ( 6, 3) 13 + 0x807b, // ( 8, 123) 14 + 0x4009, // ( 4, 9) 15 + 0xa0f7, // (10, 247) 16 + 0x6023, // ( 6, 35) 17 + 0x80fb, // ( 8, 251) 18 + 0x5015, // ( 5, 21) 19 + 0xa2f7, // (10, 759) 20 + 0x704b, // ( 7, 75) 21 + 0x9057, // ( 9, 87) 22 + 0x500d, // ( 5, 13) 23 + 0xb34f, // (11, 847) 24 + 0x8007, // ( 8, 7) 25 + 0xa1f7, // (10, 503) 26 + 0x702b, // ( 7, 43) 27 + 0xc72f, // (12, 1839) 28 + 0x8087, // ( 8, 135) 29 + 0xa3f7, // (10, 1015) 30 + 0x706b, // ( 7, 107) 31 + 0xb74f, // (11, 1871) 32 + 0x701b, // ( 7, 27) 33 + 0x9157, // ( 9, 343) 34 + 0x6013, // ( 6, 19) 35 + 0xb0cf, // (11, 207) 36 + 0x8047, // ( 8, 71) 37 + 0xa00f, // (10, 15) 38 + 0x705b, // ( 7, 91) 39 + 0xcf2f, // (12, 3887) 40 + 0x90d7, // ( 9, 215) 41 + 0xb4cf, // (11, 1231) 42 + 0x80c7, // ( 8, 199) 43 + 0xc0af, // (12, 175) 44 + 0x91d7, // ( 9, 471) 45 + 0xb2cf, // (11, 719) 46 + 0x8027, // ( 8, 39) 47 + 0xc8af, // (12, 2223) 48 + 0xa20f, // (10, 527) 49 + 0xc4af, // (12, 1199) 50 + 0x9037, // ( 9, 55) 51 + 0xccaf, // (12, 3247) 52 + 0xa10f, // (10, 271) 53 + 0xc2af, // (12, 687) 54 + 0x9137, // ( 9, 311) 55 + 0xcaaf, // (12, 2735) 56 + 0xc6af, // (12, 1711) 57 + 0xceaf, // (12, 3759) 58 + 0xa30f, // (10, 783) 59 + 0xc1af, // (12, 431) 60 + 0xc9af, // (12, 2479) 61 + 0xc5af, // (12, 1455) 62 + 0xb6cf, // (11, 1743) 63 + 0xcdaf, // (12, 3503) 64 + 0x80a7, // ( 8, 167) 65 + 0xb1cf, // (11, 463) 66 + 0x703b, // ( 7, 59) 67 + 0xc3af, // (12, 943) 68 + 0x90b7, // ( 9, 183) 69 + 0xb5cf, // (11, 1487) 70 + 0x8067, // ( 8, 103) 71 + 0xcbaf, // (12, 2991) 72 + 0xa08f, // (10, 143) 73 + 0xc7af, // (12, 1967) 74 + 0x91b7, // ( 9, 439) 75 + 0xcfaf, // (12, 4015) 76 + 0xa28f, // (10, 655) 77 + 0xc06f, // (12, 111) 78 + 0x9077, // ( 9, 119) 79 + 0xc86f, // (12, 2159) 80 + 0xb3cf, // (11, 975) 81 + 0xc46f, // (12, 1135) 82 + 0xa18f, // (10, 399) 83 + 0xcc6f, // (12, 3183) 84 + 0xb7cf, // (11, 1999) 85 + 0xc26f, // (12, 623) 86 + 0xa38f, // (10, 911) 87 + 0xca6f, // (12, 2671) 88 + 0xc66f, // (12, 1647) 89 + 0xce6f, // (12, 3695) 90 + 0xb02f, // (11, 47) 91 + 0xc16f, // (12, 367) 92 + 0xc96f, // (12, 2415) 93 + 0xc56f, // (12, 1391) 94 + 0xcd6f, // (12, 3439) 95 + 0xc36f, // (12, 879) 96 + 0xcb6f, // (12, 2927) 97 + 0xc76f, // (12, 1903) 98 + 0xb42f, // (11, 1071) 99 + 0xcf6f, // (12, 3951) 100 + 0xc0ef, // (12, 239) 101 + 0xc8ef, // (12, 2287) 102 + 0xb22f, // (11, 559) 103 + 0xc4ef, // (12, 1263) 104 + 0xccef, // (12, 3311) 105 + 0xc2ef, // (12, 751) 106 + 0xcaef, // (12, 2799) 107 + 0xc6ef, // (12, 1775) 108 + 0xceef, // (12, 3823) 109 + 0xc1ef, // (12, 495) 110 + 0xc9ef, // (12, 2543) 111 + 0xc5ef, // (12, 1519) 112 + 0xcdef, // (12, 3567) 113 + 0xc3ef, // (12, 1007) 114 + 0xcbef, // (12, 3055) 115 + 0xc7ef, // (12, 2031) 116 + 0xcfef, // (12, 4079) 117 + 0xc01f, // (12, 31) 118 + 0xc81f, // (12, 2079) 119 + 0xc41f, // (12, 1055) 120 + 0xcc1f, // (12, 3103) 121 + 0xc21f, // (12, 543) 122 + 0xca1f, // (12, 2591) 123 + 0xc61f, // (12, 1567) 124 + 0xce1f, // (12, 3615) 125 + 0xc11f, // (12, 287) 126 + 0xc91f, // (12, 2335) 127 + 0xc51f, // (12, 1311) 128 + 0x9177, // ( 9, 375) 129 + 0xcd1f, // (12, 3359) 130 + 0x80e7, // ( 8, 231) 131 + 0xc31f, // (12, 799) 132 + 0xa04f, // (10, 79) 133 + 0xcb1f, // (12, 2847) 134 + 0x8017, // ( 8, 23) 135 + 0xc71f, // (12, 1823) 136 + 0xb62f, // (11, 1583) 137 + 0xcf1f, // (12, 3871) 138 + 0xa24f, // (10, 591) 139 + 0xc09f, // (12, 159) 140 + 0xb12f, // (11, 303) 141 + 0xc89f, // (12, 2207) 142 + 0xa14f, // (10, 335) 143 + 0xc49f, // (12, 1183) 144 + 0xcc9f, // (12, 3231) 145 + 0xc29f, // (12, 671) 146 + 0xb52f, // (11, 1327) 147 + 0xca9f, // (12, 2719) 148 + 0xc69f, // (12, 1695) 149 + 0xce9f, // (12, 3743) 150 + 0xb32f, // (11, 815) 151 + 0xc19f, // (12, 415) 152 + 0xc99f, // (12, 2463) 153 + 0xc59f, // (12, 1439) 154 + 0xcd9f, // (12, 3487) 155 + 0xc39f, // (12, 927) 156 + 0xcb9f, // (12, 2975) 157 + 0xc79f, // (12, 1951) 158 + 0xcf9f, // (12, 3999) 159 + 0xc05f, // (12, 95) 160 + 0xc85f, // (12, 2143) 161 + 0xc45f, // (12, 1119) 162 + 0xcc5f, // (12, 3167) 163 + 0xc25f, // (12, 607) 164 + 0xca5f, // (12, 2655) 165 + 0xc65f, // (12, 1631) 166 + 0xce5f, // (12, 3679) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 10 of 22) (steady 10 of 16) (phase = 0.656250000 = 21.0 / 32.0) + // entropy: 4.4626765653088611430 + // avg_length: 4.5373141251902122661; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x700b, // ( 7, 11) 0 + 0x4006, // ( 4, 6) 1 + 0x601d, // ( 6, 29) 2 + 0x2000, // ( 2, 0) 3 + 0x807b, // ( 8, 123) 4 + 0x400e, // ( 4, 14) 5 + 0x603d, // ( 6, 61) 6 + 0x3002, // ( 3, 2) 7 + 0x9017, // ( 9, 23) 8 + 0x5005, // ( 5, 5) 9 + 0x704b, // ( 7, 75) 10 + 0x4001, // ( 4, 1) 11 + 0x9117, // ( 9, 279) 12 + 0x6003, // ( 6, 3) 13 + 0x80fb, // ( 8, 251) 14 + 0x4009, // ( 4, 9) 15 + 0xa177, // (10, 375) 16 + 0x6023, // ( 6, 35) 17 + 0x9097, // ( 9, 151) 18 + 0x5015, // ( 5, 21) 19 + 0xa377, // (10, 887) 20 + 0x702b, // ( 7, 43) 21 + 0x9197, // ( 9, 407) 22 + 0x500d, // ( 5, 13) 23 + 0xb34f, // (11, 847) 24 + 0x8007, // ( 8, 7) 25 + 0xa0f7, // (10, 247) 26 + 0x706b, // ( 7, 107) 27 + 0xc0af, // (12, 175) 28 + 0x8087, // ( 8, 135) 29 + 0xa2f7, // (10, 759) 30 + 0x701b, // ( 7, 27) 31 + 0xb74f, // (11, 1871) 32 + 0x8047, // ( 8, 71) 33 + 0xa1f7, // (10, 503) 34 + 0x6013, // ( 6, 19) 35 + 0xb0cf, // (11, 207) 36 + 0x80c7, // ( 8, 199) 37 + 0xa3f7, // (10, 1015) 38 + 0x6033, // ( 6, 51) 39 + 0xc8af, // (12, 2223) 40 + 0x9057, // ( 9, 87) 41 + 0xb4cf, // (11, 1231) 42 + 0x8027, // ( 8, 39) 43 + 0xc4af, // (12, 1199) 44 + 0x9157, // ( 9, 343) 45 + 0xb2cf, // (11, 719) 46 + 0x80a7, // ( 8, 167) 47 + 0xccaf, // (12, 3247) 48 + 0xa00f, // (10, 15) 49 + 0xc2af, // (12, 687) 50 + 0x90d7, // ( 9, 215) 51 + 0xcaaf, // (12, 2735) 52 + 0xa20f, // (10, 527) 53 + 0xc6af, // (12, 1711) 54 + 0x91d7, // ( 9, 471) 55 + 0xceaf, // (12, 3759) 56 + 0xb6cf, // (11, 1743) 57 + 0xc1af, // (12, 431) 58 + 0xa10f, // (10, 271) 59 + 0xc9af, // (12, 2479) 60 + 0xc5af, // (12, 1455) 61 + 0xcdaf, // (12, 3503) 62 + 0xa30f, // (10, 783) 63 + 0xc3af, // (12, 943) 64 + 0x9037, // ( 9, 55) 65 + 0xb1cf, // (11, 463) 66 + 0x705b, // ( 7, 91) 67 + 0xcbaf, // (12, 2991) 68 + 0x9137, // ( 9, 311) 69 + 0xb5cf, // (11, 1487) 70 + 0x703b, // ( 7, 59) 71 + 0xc7af, // (12, 1967) 72 + 0xa08f, // (10, 143) 73 + 0xcfaf, // (12, 4015) 74 + 0x90b7, // ( 9, 183) 75 + 0xc06f, // (12, 111) 76 + 0xa28f, // (10, 655) 77 + 0xc86f, // (12, 2159) 78 + 0x91b7, // ( 9, 439) 79 + 0xc46f, // (12, 1135) 80 + 0xb3cf, // (11, 975) 81 + 0xcc6f, // (12, 3183) 82 + 0xa18f, // (10, 399) 83 + 0xc26f, // (12, 623) 84 + 0xb7cf, // (11, 1999) 85 + 0xca6f, // (12, 2671) 86 + 0xa38f, // (10, 911) 87 + 0xc66f, // (12, 1647) 88 + 0xce6f, // (12, 3695) 89 + 0xc16f, // (12, 367) 90 + 0xb02f, // (11, 47) 91 + 0xc96f, // (12, 2415) 92 + 0xc56f, // (12, 1391) 93 + 0xcd6f, // (12, 3439) 94 + 0xb42f, // (11, 1071) 95 + 0xc36f, // (12, 879) 96 + 0xcb6f, // (12, 2927) 97 + 0xc76f, // (12, 1903) 98 + 0xb22f, // (11, 559) 99 + 0xcf6f, // (12, 3951) 100 + 0xc0ef, // (12, 239) 101 + 0xc8ef, // (12, 2287) 102 + 0xb62f, // (11, 1583) 103 + 0xc4ef, // (12, 1263) 104 + 0xccef, // (12, 3311) 105 + 0xc2ef, // (12, 751) 106 + 0xcaef, // (12, 2799) 107 + 0xc6ef, // (12, 1775) 108 + 0xceef, // (12, 3823) 109 + 0xc1ef, // (12, 495) 110 + 0xc9ef, // (12, 2543) 111 + 0xc5ef, // (12, 1519) 112 + 0xcdef, // (12, 3567) 113 + 0xc3ef, // (12, 1007) 114 + 0xcbef, // (12, 3055) 115 + 0xc7ef, // (12, 2031) 116 + 0xcfef, // (12, 4079) 117 + 0xc01f, // (12, 31) 118 + 0xc81f, // (12, 2079) 119 + 0xc41f, // (12, 1055) 120 + 0xcc1f, // (12, 3103) 121 + 0xc21f, // (12, 543) 122 + 0xca1f, // (12, 2591) 123 + 0xc61f, // (12, 1567) 124 + 0xce1f, // (12, 3615) 125 + 0xc11f, // (12, 287) 126 + 0xc91f, // (12, 2335) 127 + 0xc51f, // (12, 1311) 128 + 0x9077, // ( 9, 119) 129 + 0xcd1f, // (12, 3359) 130 + 0x8067, // ( 8, 103) 131 + 0xc31f, // (12, 799) 132 + 0xa04f, // (10, 79) 133 + 0xcb1f, // (12, 2847) 134 + 0x80e7, // ( 8, 231) 135 + 0xc71f, // (12, 1823) 136 + 0xb12f, // (11, 303) 137 + 0xcf1f, // (12, 3871) 138 + 0xa24f, // (10, 591) 139 + 0xc09f, // (12, 159) 140 + 0xb52f, // (11, 1327) 141 + 0xc89f, // (12, 2207) 142 + 0xa14f, // (10, 335) 143 + 0xc49f, // (12, 1183) 144 + 0xcc9f, // (12, 3231) 145 + 0xc29f, // (12, 671) 146 + 0xb32f, // (11, 815) 147 + 0xca9f, // (12, 2719) 148 + 0xc69f, // (12, 1695) 149 + 0xce9f, // (12, 3743) 150 + 0xb72f, // (11, 1839) 151 + 0xc19f, // (12, 415) 152 + 0xc99f, // (12, 2463) 153 + 0xc59f, // (12, 1439) 154 + 0xcd9f, // (12, 3487) 155 + 0xc39f, // (12, 927) 156 + 0xcb9f, // (12, 2975) 157 + 0xc79f, // (12, 1951) 158 + 0xcf9f, // (12, 3999) 159 + 0xc05f, // (12, 95) 160 + 0xc85f, // (12, 2143) 161 + 0xc45f, // (12, 1119) 162 + 0xcc5f, // (12, 3167) 163 + 0xc25f, // (12, 607) 164 + 0xca5f, // (12, 2655) 165 + 0xc65f, // (12, 1631) 166 + 0xce5f, // (12, 3679) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 11 of 22) (steady 11 of 16) (phase = 0.718750000 = 23.0 / 32.0) + // entropy: 4.4661524304421691411 + // avg_length: 4.5443750890419041255; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x803b, // ( 8, 59) 0 + 0x4006, // ( 4, 6) 1 + 0x601d, // ( 6, 29) 2 + 0x2000, // ( 2, 0) 3 + 0x80bb, // ( 8, 187) 4 + 0x400e, // ( 4, 14) 5 + 0x603d, // ( 6, 61) 6 + 0x3002, // ( 3, 2) 7 + 0x9017, // ( 9, 23) 8 + 0x5005, // ( 5, 5) 9 + 0x807b, // ( 8, 123) 10 + 0x4001, // ( 4, 1) 11 + 0x9117, // ( 9, 279) 12 + 0x6003, // ( 6, 3) 13 + 0x80fb, // ( 8, 251) 14 + 0x4009, // ( 4, 9) 15 + 0xa177, // (10, 375) 16 + 0x6023, // ( 6, 35) 17 + 0x9097, // ( 9, 151) 18 + 0x5015, // ( 5, 21) 19 + 0xa377, // (10, 887) 20 + 0x702b, // ( 7, 43) 21 + 0x9197, // ( 9, 407) 22 + 0x500d, // ( 5, 13) 23 + 0xb34f, // (11, 847) 24 + 0x8007, // ( 8, 7) 25 + 0xa0f7, // (10, 247) 26 + 0x6013, // ( 6, 19) 27 + 0xc0af, // (12, 175) 28 + 0x8087, // ( 8, 135) 29 + 0xa2f7, // (10, 759) 30 + 0x706b, // ( 7, 107) 31 + 0xb74f, // (11, 1871) 32 + 0x8047, // ( 8, 71) 33 + 0xa1f7, // (10, 503) 34 + 0x6033, // ( 6, 51) 35 + 0xb0cf, // (11, 207) 36 + 0x80c7, // ( 8, 199) 37 + 0xa3f7, // (10, 1015) 38 + 0x600b, // ( 6, 11) 39 + 0xc8af, // (12, 2223) 40 + 0x9057, // ( 9, 87) 41 + 0xb4cf, // (11, 1231) 42 + 0x8027, // ( 8, 39) 43 + 0xc4af, // (12, 1199) 44 + 0x9157, // ( 9, 343) 45 + 0xb2cf, // (11, 719) 46 + 0x80a7, // ( 8, 167) 47 + 0xccaf, // (12, 3247) 48 + 0xa00f, // (10, 15) 49 + 0xc2af, // (12, 687) 50 + 0x90d7, // ( 9, 215) 51 + 0xcaaf, // (12, 2735) 52 + 0xa20f, // (10, 527) 53 + 0xc6af, // (12, 1711) 54 + 0x91d7, // ( 9, 471) 55 + 0xceaf, // (12, 3759) 56 + 0xb6cf, // (11, 1743) 57 + 0xc1af, // (12, 431) 58 + 0xa10f, // (10, 271) 59 + 0xc9af, // (12, 2479) 60 + 0xc5af, // (12, 1455) 61 + 0xcdaf, // (12, 3503) 62 + 0xa30f, // (10, 783) 63 + 0xc3af, // (12, 943) 64 + 0x9037, // ( 9, 55) 65 + 0xb1cf, // (11, 463) 66 + 0x701b, // ( 7, 27) 67 + 0xcbaf, // (12, 2991) 68 + 0x9137, // ( 9, 311) 69 + 0xb5cf, // (11, 1487) 70 + 0x705b, // ( 7, 91) 71 + 0xc7af, // (12, 1967) 72 + 0xa08f, // (10, 143) 73 + 0xcfaf, // (12, 4015) 74 + 0x90b7, // ( 9, 183) 75 + 0xc06f, // (12, 111) 76 + 0xa28f, // (10, 655) 77 + 0xc86f, // (12, 2159) 78 + 0x91b7, // ( 9, 439) 79 + 0xc46f, // (12, 1135) 80 + 0xb3cf, // (11, 975) 81 + 0xcc6f, // (12, 3183) 82 + 0xa18f, // (10, 399) 83 + 0xc26f, // (12, 623) 84 + 0xb7cf, // (11, 1999) 85 + 0xca6f, // (12, 2671) 86 + 0xa38f, // (10, 911) 87 + 0xc66f, // (12, 1647) 88 + 0xce6f, // (12, 3695) 89 + 0xc16f, // (12, 367) 90 + 0xb02f, // (11, 47) 91 + 0xc96f, // (12, 2415) 92 + 0xc56f, // (12, 1391) 93 + 0xcd6f, // (12, 3439) 94 + 0xb42f, // (11, 1071) 95 + 0xc36f, // (12, 879) 96 + 0xcb6f, // (12, 2927) 97 + 0xc76f, // (12, 1903) 98 + 0xb22f, // (11, 559) 99 + 0xcf6f, // (12, 3951) 100 + 0xc0ef, // (12, 239) 101 + 0xc8ef, // (12, 2287) 102 + 0xb62f, // (11, 1583) 103 + 0xc4ef, // (12, 1263) 104 + 0xccef, // (12, 3311) 105 + 0xc2ef, // (12, 751) 106 + 0xcaef, // (12, 2799) 107 + 0xc6ef, // (12, 1775) 108 + 0xceef, // (12, 3823) 109 + 0xc1ef, // (12, 495) 110 + 0xc9ef, // (12, 2543) 111 + 0xc5ef, // (12, 1519) 112 + 0xcdef, // (12, 3567) 113 + 0xc3ef, // (12, 1007) 114 + 0xcbef, // (12, 3055) 115 + 0xc7ef, // (12, 2031) 116 + 0xcfef, // (12, 4079) 117 + 0xc01f, // (12, 31) 118 + 0xc81f, // (12, 2079) 119 + 0xc41f, // (12, 1055) 120 + 0xcc1f, // (12, 3103) 121 + 0xc21f, // (12, 543) 122 + 0xca1f, // (12, 2591) 123 + 0xc61f, // (12, 1567) 124 + 0xce1f, // (12, 3615) 125 + 0xc11f, // (12, 287) 126 + 0xc91f, // (12, 2335) 127 + 0xc51f, // (12, 1311) 128 + 0xa04f, // (10, 79) 129 + 0xcd1f, // (12, 3359) 130 + 0x8067, // ( 8, 103) 131 + 0xc31f, // (12, 799) 132 + 0xa24f, // (10, 591) 133 + 0xcb1f, // (12, 2847) 134 + 0x80e7, // ( 8, 231) 135 + 0xc71f, // (12, 1823) 136 + 0xb12f, // (11, 303) 137 + 0xcf1f, // (12, 3871) 138 + 0x9077, // ( 9, 119) 139 + 0xc09f, // (12, 159) 140 + 0xb52f, // (11, 1327) 141 + 0xc89f, // (12, 2207) 142 + 0xa14f, // (10, 335) 143 + 0xc49f, // (12, 1183) 144 + 0xcc9f, // (12, 3231) 145 + 0xc29f, // (12, 671) 146 + 0xb32f, // (11, 815) 147 + 0xca9f, // (12, 2719) 148 + 0xc69f, // (12, 1695) 149 + 0xce9f, // (12, 3743) 150 + 0xb72f, // (11, 1839) 151 + 0xc19f, // (12, 415) 152 + 0xc99f, // (12, 2463) 153 + 0xc59f, // (12, 1439) 154 + 0xcd9f, // (12, 3487) 155 + 0xc39f, // (12, 927) 156 + 0xcb9f, // (12, 2975) 157 + 0xc79f, // (12, 1951) 158 + 0xcf9f, // (12, 3999) 159 + 0xc05f, // (12, 95) 160 + 0xc85f, // (12, 2143) 161 + 0xc45f, // (12, 1119) 162 + 0xcc5f, // (12, 3167) 163 + 0xc25f, // (12, 607) 164 + 0xca5f, // (12, 2655) 165 + 0xc65f, // (12, 1631) 166 + 0xce5f, // (12, 3679) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 12 of 22) (steady 12 of 16) (phase = 0.781250000 = 25.0 / 32.0) + // entropy: 4.4680486273043946710 + // avg_length: 4.5521643785256946657; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x807b, // ( 8, 123) 0 + 0x4006, // ( 4, 6) 1 + 0x601d, // ( 6, 29) 2 + 0x2000, // ( 2, 0) 3 + 0x80fb, // ( 8, 251) 4 + 0x400e, // ( 4, 14) 5 + 0x700b, // ( 7, 11) 6 + 0x3002, // ( 3, 2) 7 + 0x9097, // ( 9, 151) 8 + 0x5005, // ( 5, 5) 9 + 0x8007, // ( 8, 7) 10 + 0x4001, // ( 4, 1) 11 + 0x9197, // ( 9, 407) 12 + 0x603d, // ( 6, 61) 13 + 0x8087, // ( 8, 135) 14 + 0x4009, // ( 4, 9) 15 + 0xa177, // (10, 375) 16 + 0x704b, // ( 7, 75) 17 + 0x9057, // ( 9, 87) 18 + 0x5015, // ( 5, 21) 19 + 0xb34f, // (11, 847) 20 + 0x702b, // ( 7, 43) 21 + 0x9157, // ( 9, 343) 22 + 0x500d, // ( 5, 13) 23 + 0xc72f, // (12, 1839) 24 + 0x8047, // ( 8, 71) 25 + 0xa377, // (10, 887) 26 + 0x6003, // ( 6, 3) 27 + 0xcf2f, // (12, 3887) 28 + 0x80c7, // ( 8, 199) 29 + 0xa0f7, // (10, 247) 30 + 0x6023, // ( 6, 35) 31 + 0xc0af, // (12, 175) 32 + 0x8027, // ( 8, 39) 33 + 0xa2f7, // (10, 759) 34 + 0x6013, // ( 6, 19) 35 + 0xc8af, // (12, 2223) 36 + 0x80a7, // ( 8, 167) 37 + 0xa1f7, // (10, 503) 38 + 0x6033, // ( 6, 51) 39 + 0xc4af, // (12, 1199) 40 + 0x90d7, // ( 9, 215) 41 + 0xb74f, // (11, 1871) 42 + 0x706b, // ( 7, 107) 43 + 0xccaf, // (12, 3247) 44 + 0x91d7, // ( 9, 471) 45 + 0xb0cf, // (11, 207) 46 + 0x701b, // ( 7, 27) 47 + 0xc2af, // (12, 687) 48 + 0xa3f7, // (10, 1015) 49 + 0xcaaf, // (12, 2735) 50 + 0x9037, // ( 9, 55) 51 + 0xc6af, // (12, 1711) 52 + 0xa00f, // (10, 15) 53 + 0xceaf, // (12, 3759) 54 + 0x9137, // ( 9, 311) 55 + 0xc1af, // (12, 431) 56 + 0xb4cf, // (11, 1231) 57 + 0xc9af, // (12, 2479) 58 + 0xa20f, // (10, 527) 59 + 0xc5af, // (12, 1455) 60 + 0xb2cf, // (11, 719) 61 + 0xcdaf, // (12, 3503) 62 + 0xa10f, // (10, 271) 63 + 0xc3af, // (12, 943) 64 + 0x90b7, // ( 9, 183) 65 + 0xb6cf, // (11, 1743) 66 + 0x705b, // ( 7, 91) 67 + 0xcbaf, // (12, 2991) 68 + 0x91b7, // ( 9, 439) 69 + 0xb1cf, // (11, 463) 70 + 0x703b, // ( 7, 59) 71 + 0xc7af, // (12, 1967) 72 + 0xa30f, // (10, 783) 73 + 0xcfaf, // (12, 4015) 74 + 0x8067, // ( 8, 103) 75 + 0xc06f, // (12, 111) 76 + 0xa08f, // (10, 143) 77 + 0xc86f, // (12, 2159) 78 + 0x9077, // ( 9, 119) 79 + 0xc46f, // (12, 1135) 80 + 0xb5cf, // (11, 1487) 81 + 0xcc6f, // (12, 3183) 82 + 0xa28f, // (10, 655) 83 + 0xc26f, // (12, 623) 84 + 0xb3cf, // (11, 975) 85 + 0xca6f, // (12, 2671) 86 + 0xa18f, // (10, 399) 87 + 0xc66f, // (12, 1647) 88 + 0xce6f, // (12, 3695) 89 + 0xc16f, // (12, 367) 90 + 0xb7cf, // (11, 1999) 91 + 0xc96f, // (12, 2415) 92 + 0xc56f, // (12, 1391) 93 + 0xcd6f, // (12, 3439) 94 + 0xb02f, // (11, 47) 95 + 0xc36f, // (12, 879) 96 + 0xcb6f, // (12, 2927) 97 + 0xc76f, // (12, 1903) 98 + 0xb42f, // (11, 1071) 99 + 0xcf6f, // (12, 3951) 100 + 0xc0ef, // (12, 239) 101 + 0xc8ef, // (12, 2287) 102 + 0xb22f, // (11, 559) 103 + 0xc4ef, // (12, 1263) 104 + 0xccef, // (12, 3311) 105 + 0xc2ef, // (12, 751) 106 + 0xcaef, // (12, 2799) 107 + 0xc6ef, // (12, 1775) 108 + 0xceef, // (12, 3823) 109 + 0xc1ef, // (12, 495) 110 + 0xc9ef, // (12, 2543) 111 + 0xc5ef, // (12, 1519) 112 + 0xcdef, // (12, 3567) 113 + 0xc3ef, // (12, 1007) 114 + 0xcbef, // (12, 3055) 115 + 0xc7ef, // (12, 2031) 116 + 0xcfef, // (12, 4079) 117 + 0xc01f, // (12, 31) 118 + 0xc81f, // (12, 2079) 119 + 0xc41f, // (12, 1055) 120 + 0xcc1f, // (12, 3103) 121 + 0xc21f, // (12, 543) 122 + 0xca1f, // (12, 2591) 123 + 0xc61f, // (12, 1567) 124 + 0xce1f, // (12, 3615) 125 + 0xc11f, // (12, 287) 126 + 0xc91f, // (12, 2335) 127 + 0xc51f, // (12, 1311) 128 + 0xa38f, // (10, 911) 129 + 0xcd1f, // (12, 3359) 130 + 0x80e7, // ( 8, 231) 131 + 0xc31f, // (12, 799) 132 + 0xa04f, // (10, 79) 133 + 0xcb1f, // (12, 2847) 134 + 0x8017, // ( 8, 23) 135 + 0xc71f, // (12, 1823) 136 + 0xb62f, // (11, 1583) 137 + 0xcf1f, // (12, 3871) 138 + 0xa24f, // (10, 591) 139 + 0xc09f, // (12, 159) 140 + 0xb12f, // (11, 303) 141 + 0xc89f, // (12, 2207) 142 + 0xa14f, // (10, 335) 143 + 0xc49f, // (12, 1183) 144 + 0xcc9f, // (12, 3231) 145 + 0xc29f, // (12, 671) 146 + 0xb52f, // (11, 1327) 147 + 0xca9f, // (12, 2719) 148 + 0xc69f, // (12, 1695) 149 + 0xce9f, // (12, 3743) 150 + 0xb32f, // (11, 815) 151 + 0xc19f, // (12, 415) 152 + 0xc99f, // (12, 2463) 153 + 0xc59f, // (12, 1439) 154 + 0xcd9f, // (12, 3487) 155 + 0xc39f, // (12, 927) 156 + 0xcb9f, // (12, 2975) 157 + 0xc79f, // (12, 1951) 158 + 0xcf9f, // (12, 3999) 159 + 0xc05f, // (12, 95) 160 + 0xc85f, // (12, 2143) 161 + 0xc45f, // (12, 1119) 162 + 0xcc5f, // (12, 3167) 163 + 0xc25f, // (12, 607) 164 + 0xca5f, // (12, 2655) 165 + 0xc65f, // (12, 1631) 166 + 0xce5f, // (12, 3679) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 13 of 22) (steady 13 of 16) (phase = 0.843750000 = 27.0 / 32.0) + // entropy: 4.4684687952964843305 + // avg_length: 4.5509169030369793774; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x803b, // ( 8, 59) 0 + 0x4006, // ( 4, 6) 1 + 0x7033, // ( 7, 51) 2 + 0x3002, // ( 3, 2) 3 + 0x80bb, // ( 8, 187) 4 + 0x400e, // ( 4, 14) 5 + 0x7073, // ( 7, 115) 6 + 0x2000, // ( 2, 0) 7 + 0xa0f7, // (10, 247) 8 + 0x601d, // ( 6, 29) 9 + 0x807b, // ( 8, 123) 10 + 0x4001, // ( 4, 1) 11 + 0xa2f7, // (10, 759) 12 + 0x5005, // ( 5, 5) 13 + 0x80fb, // ( 8, 251) 14 + 0x4009, // ( 4, 9) 15 + 0xb34f, // (11, 847) 16 + 0x700b, // ( 7, 11) 17 + 0x9057, // ( 9, 87) 18 + 0x5015, // ( 5, 21) 19 + 0xb74f, // (11, 1871) 20 + 0x704b, // ( 7, 75) 21 + 0x9157, // ( 9, 343) 22 + 0x500d, // ( 5, 13) 23 + 0xc72f, // (12, 1839) 24 + 0x8007, // ( 8, 7) 25 + 0xa1f7, // (10, 503) 26 + 0x603d, // ( 6, 61) 27 + 0xcf2f, // (12, 3887) 28 + 0x8087, // ( 8, 135) 29 + 0xa3f7, // (10, 1015) 30 + 0x6003, // ( 6, 3) 31 + 0xc0af, // (12, 175) 32 + 0x8047, // ( 8, 71) 33 + 0xa00f, // (10, 15) 34 + 0x6023, // ( 6, 35) 35 + 0xc8af, // (12, 2223) 36 + 0x80c7, // ( 8, 199) 37 + 0xa20f, // (10, 527) 38 + 0x6013, // ( 6, 19) 39 + 0xc4af, // (12, 1199) 40 + 0x90d7, // ( 9, 215) 41 + 0xb0cf, // (11, 207) 42 + 0x702b, // ( 7, 43) 43 + 0xccaf, // (12, 3247) 44 + 0x91d7, // ( 9, 471) 45 + 0xb4cf, // (11, 1231) 46 + 0x706b, // ( 7, 107) 47 + 0xc2af, // (12, 687) 48 + 0xa10f, // (10, 271) 49 + 0xcaaf, // (12, 2735) 50 + 0x8027, // ( 8, 39) 51 + 0xc6af, // (12, 1711) 52 + 0xa30f, // (10, 783) 53 + 0xceaf, // (12, 3759) 54 + 0x80a7, // ( 8, 167) 55 + 0xc1af, // (12, 431) 56 + 0xb2cf, // (11, 719) 57 + 0xc9af, // (12, 2479) 58 + 0xa08f, // (10, 143) 59 + 0xc5af, // (12, 1455) 60 + 0xb6cf, // (11, 1743) 61 + 0xcdaf, // (12, 3503) 62 + 0xa28f, // (10, 655) 63 + 0xc3af, // (12, 943) 64 + 0x9037, // ( 9, 55) 65 + 0xb1cf, // (11, 463) 66 + 0x701b, // ( 7, 27) 67 + 0xcbaf, // (12, 2991) 68 + 0x9137, // ( 9, 311) 69 + 0xb5cf, // (11, 1487) 70 + 0x705b, // ( 7, 91) 71 + 0xc7af, // (12, 1967) 72 + 0xa18f, // (10, 399) 73 + 0xcfaf, // (12, 4015) 74 + 0x8067, // ( 8, 103) 75 + 0xc06f, // (12, 111) 76 + 0xa38f, // (10, 911) 77 + 0xc86f, // (12, 2159) 78 + 0x80e7, // ( 8, 231) 79 + 0xc46f, // (12, 1135) 80 + 0xb3cf, // (11, 975) 81 + 0xcc6f, // (12, 3183) 82 + 0x90b7, // ( 9, 183) 83 + 0xc26f, // (12, 623) 84 + 0xb7cf, // (11, 1999) 85 + 0xca6f, // (12, 2671) 86 + 0x91b7, // ( 9, 439) 87 + 0xc66f, // (12, 1647) 88 + 0xce6f, // (12, 3695) 89 + 0xc16f, // (12, 367) 90 + 0xb02f, // (11, 47) 91 + 0xc96f, // (12, 2415) 92 + 0xc56f, // (12, 1391) 93 + 0xcd6f, // (12, 3439) 94 + 0xb42f, // (11, 1071) 95 + 0xc36f, // (12, 879) 96 + 0xcb6f, // (12, 2927) 97 + 0xc76f, // (12, 1903) 98 + 0xb22f, // (11, 559) 99 + 0xcf6f, // (12, 3951) 100 + 0xc0ef, // (12, 239) 101 + 0xc8ef, // (12, 2287) 102 + 0xb62f, // (11, 1583) 103 + 0xc4ef, // (12, 1263) 104 + 0xccef, // (12, 3311) 105 + 0xc2ef, // (12, 751) 106 + 0xcaef, // (12, 2799) 107 + 0xc6ef, // (12, 1775) 108 + 0xceef, // (12, 3823) 109 + 0xc1ef, // (12, 495) 110 + 0xc9ef, // (12, 2543) 111 + 0xc5ef, // (12, 1519) 112 + 0xcdef, // (12, 3567) 113 + 0xc3ef, // (12, 1007) 114 + 0xcbef, // (12, 3055) 115 + 0xc7ef, // (12, 2031) 116 + 0xcfef, // (12, 4079) 117 + 0xc01f, // (12, 31) 118 + 0xc81f, // (12, 2079) 119 + 0xc41f, // (12, 1055) 120 + 0xcc1f, // (12, 3103) 121 + 0xc21f, // (12, 543) 122 + 0xca1f, // (12, 2591) 123 + 0xc61f, // (12, 1567) 124 + 0xce1f, // (12, 3615) 125 + 0xc11f, // (12, 287) 126 + 0xc91f, // (12, 2335) 127 + 0xc51f, // (12, 1311) 128 + 0xa04f, // (10, 79) 129 + 0xcd1f, // (12, 3359) 130 + 0x8017, // ( 8, 23) 131 + 0xc31f, // (12, 799) 132 + 0xa24f, // (10, 591) 133 + 0xcb1f, // (12, 2847) 134 + 0x8097, // ( 8, 151) 135 + 0xc71f, // (12, 1823) 136 + 0xb12f, // (11, 303) 137 + 0xcf1f, // (12, 3871) 138 + 0x9077, // ( 9, 119) 139 + 0xc09f, // (12, 159) 140 + 0xb52f, // (11, 1327) 141 + 0xc89f, // (12, 2207) 142 + 0x9177, // ( 9, 375) 143 + 0xc49f, // (12, 1183) 144 + 0xcc9f, // (12, 3231) 145 + 0xc29f, // (12, 671) 146 + 0xb32f, // (11, 815) 147 + 0xca9f, // (12, 2719) 148 + 0xc69f, // (12, 1695) 149 + 0xce9f, // (12, 3743) 150 + 0xa14f, // (10, 335) 151 + 0xc19f, // (12, 415) 152 + 0xc99f, // (12, 2463) 153 + 0xc59f, // (12, 1439) 154 + 0xcd9f, // (12, 3487) 155 + 0xc39f, // (12, 927) 156 + 0xcb9f, // (12, 2975) 157 + 0xc79f, // (12, 1951) 158 + 0xcf9f, // (12, 3999) 159 + 0xc05f, // (12, 95) 160 + 0xc85f, // (12, 2143) 161 + 0xc45f, // (12, 1119) 162 + 0xcc5f, // (12, 3167) 163 + 0xc25f, // (12, 607) 164 + 0xca5f, // (12, 2655) 165 + 0xc65f, // (12, 1631) 166 + 0xce5f, // (12, 3679) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 14 of 22) (steady 14 of 16) (phase = 0.906250000 = 29.0 / 32.0) + // entropy: 4.4675179140944036860 + // avg_length: 4.5477235350841240802; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x9017, // ( 9, 23) 0 + 0x4006, // ( 4, 6) 1 + 0x7033, // ( 7, 51) 2 + 0x3002, // ( 3, 2) 3 + 0x9117, // ( 9, 279) 4 + 0x400e, // ( 4, 14) 5 + 0x7073, // ( 7, 115) 6 + 0x2000, // ( 2, 0) 7 + 0xa177, // (10, 375) 8 + 0x601d, // ( 6, 29) 9 + 0x803b, // ( 8, 59) 10 + 0x4001, // ( 4, 1) 11 + 0xa377, // (10, 887) 12 + 0x5005, // ( 5, 5) 13 + 0x80bb, // ( 8, 187) 14 + 0x4009, // ( 4, 9) 15 + 0xb0cf, // (11, 207) 16 + 0x700b, // ( 7, 11) 17 + 0x9097, // ( 9, 151) 18 + 0x5015, // ( 5, 21) 19 + 0xb4cf, // (11, 1231) 20 + 0x704b, // ( 7, 75) 21 + 0x9197, // ( 9, 407) 22 + 0x500d, // ( 5, 13) 23 + 0xc4af, // (12, 1199) 24 + 0x807b, // ( 8, 123) 25 + 0xa0f7, // (10, 247) 26 + 0x603d, // ( 6, 61) 27 + 0xccaf, // (12, 3247) 28 + 0x80fb, // ( 8, 251) 29 + 0xa2f7, // (10, 759) 30 + 0x6003, // ( 6, 3) 31 + 0xc2af, // (12, 687) 32 + 0x8007, // ( 8, 7) 33 + 0xa1f7, // (10, 503) 34 + 0x6023, // ( 6, 35) 35 + 0xcaaf, // (12, 2735) 36 + 0x8087, // ( 8, 135) 37 + 0xa3f7, // (10, 1015) 38 + 0x6013, // ( 6, 19) 39 + 0xc6af, // (12, 1711) 40 + 0x9057, // ( 9, 87) 41 + 0xb2cf, // (11, 719) 42 + 0x702b, // ( 7, 43) 43 + 0xceaf, // (12, 3759) 44 + 0x9157, // ( 9, 343) 45 + 0xb6cf, // (11, 1743) 46 + 0x706b, // ( 7, 107) 47 + 0xc1af, // (12, 431) 48 + 0xa00f, // (10, 15) 49 + 0xc9af, // (12, 2479) 50 + 0x8047, // ( 8, 71) 51 + 0xc5af, // (12, 1455) 52 + 0xa20f, // (10, 527) 53 + 0xcdaf, // (12, 3503) 54 + 0x80c7, // ( 8, 199) 55 + 0xc3af, // (12, 943) 56 + 0xb1cf, // (11, 463) 57 + 0xcbaf, // (12, 2991) 58 + 0xa10f, // (10, 271) 59 + 0xc7af, // (12, 1967) 60 + 0xb5cf, // (11, 1487) 61 + 0xcfaf, // (12, 4015) 62 + 0x90d7, // ( 9, 215) 63 + 0xc06f, // (12, 111) 64 + 0x91d7, // ( 9, 471) 65 + 0xb3cf, // (11, 975) 66 + 0x701b, // ( 7, 27) 67 + 0xc86f, // (12, 2159) 68 + 0x9037, // ( 9, 55) 69 + 0xb7cf, // (11, 1999) 70 + 0x705b, // ( 7, 91) 71 + 0xc46f, // (12, 1135) 72 + 0xa30f, // (10, 783) 73 + 0xcc6f, // (12, 3183) 74 + 0x8027, // ( 8, 39) 75 + 0xc26f, // (12, 623) 76 + 0xa08f, // (10, 143) 77 + 0xca6f, // (12, 2671) 78 + 0x80a7, // ( 8, 167) 79 + 0xc66f, // (12, 1647) 80 + 0xb02f, // (11, 47) 81 + 0xce6f, // (12, 3695) 82 + 0x9137, // ( 9, 311) 83 + 0xc16f, // (12, 367) 84 + 0xb42f, // (11, 1071) 85 + 0xc96f, // (12, 2415) 86 + 0x90b7, // ( 9, 183) 87 + 0xc56f, // (12, 1391) 88 + 0xcd6f, // (12, 3439) 89 + 0xc36f, // (12, 879) 90 + 0xb22f, // (11, 559) 91 + 0xcb6f, // (12, 2927) 92 + 0xc76f, // (12, 1903) 93 + 0xcf6f, // (12, 3951) 94 + 0xa28f, // (10, 655) 95 + 0xc0ef, // (12, 239) 96 + 0xc8ef, // (12, 2287) 97 + 0xc4ef, // (12, 1263) 98 + 0xa18f, // (10, 399) 99 + 0xccef, // (12, 3311) 100 + 0xc2ef, // (12, 751) 101 + 0xcaef, // (12, 2799) 102 + 0xa38f, // (10, 911) 103 + 0xc6ef, // (12, 1775) 104 + 0xceef, // (12, 3823) 105 + 0xc1ef, // (12, 495) 106 + 0xc9ef, // (12, 2543) 107 + 0xc5ef, // (12, 1519) 108 + 0xcdef, // (12, 3567) 109 + 0xc3ef, // (12, 1007) 110 + 0xb62f, // (11, 1583) 111 + 0xcbef, // (12, 3055) 112 + 0xc7ef, // (12, 2031) 113 + 0xcfef, // (12, 4079) 114 + 0xc01f, // (12, 31) 115 + 0xc81f, // (12, 2079) 116 + 0xc41f, // (12, 1055) 117 + 0xcc1f, // (12, 3103) 118 + 0xc21f, // (12, 543) 119 + 0xca1f, // (12, 2591) 120 + 0xc61f, // (12, 1567) 121 + 0xce1f, // (12, 3615) 122 + 0xc11f, // (12, 287) 123 + 0xc91f, // (12, 2335) 124 + 0xc51f, // (12, 1311) 125 + 0xcd1f, // (12, 3359) 126 + 0xc31f, // (12, 799) 127 + 0xcb1f, // (12, 2847) 128 + 0xa04f, // (10, 79) 129 + 0xc71f, // (12, 1823) 130 + 0x8067, // ( 8, 103) 131 + 0xcf1f, // (12, 3871) 132 + 0xa24f, // (10, 591) 133 + 0xc09f, // (12, 159) 134 + 0x80e7, // ( 8, 231) 135 + 0xc89f, // (12, 2207) 136 + 0xb12f, // (11, 303) 137 + 0xc49f, // (12, 1183) 138 + 0x91b7, // ( 9, 439) 139 + 0xcc9f, // (12, 3231) 140 + 0xb52f, // (11, 1327) 141 + 0xc29f, // (12, 671) 142 + 0x9077, // ( 9, 119) 143 + 0xca9f, // (12, 2719) 144 + 0xc69f, // (12, 1695) 145 + 0xce9f, // (12, 3743) 146 + 0xa14f, // (10, 335) 147 + 0xc19f, // (12, 415) 148 + 0xc99f, // (12, 2463) 149 + 0xc59f, // (12, 1439) 150 + 0xa34f, // (10, 847) 151 + 0xcd9f, // (12, 3487) 152 + 0xc39f, // (12, 927) 153 + 0xcb9f, // (12, 2975) 154 + 0xc79f, // (12, 1951) 155 + 0xcf9f, // (12, 3999) 156 + 0xc05f, // (12, 95) 157 + 0xc85f, // (12, 2143) 158 + 0xb32f, // (11, 815) 159 + 0xc45f, // (12, 1119) 160 + 0xcc5f, // (12, 3167) 161 + 0xc25f, // (12, 607) 162 + 0xb72f, // (11, 1839) 163 + 0xca5f, // (12, 2655) 164 + 0xc65f, // (12, 1631) 165 + 0xce5f, // (12, 3679) 166 + 0xb0af, // (11, 175) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 15 of 22) (steady 15 of 16) (phase = 0.968750000 = 31.0 / 32.0) + // entropy: 4.4653007097343397902 + // avg_length: 4.5480722016259509388; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x9017, // ( 9, 23) 0 + 0x4006, // ( 4, 6) 1 + 0x7033, // ( 7, 51) 2 + 0x3002, // ( 3, 2) 3 + 0x9117, // ( 9, 279) 4 + 0x400e, // ( 4, 14) 5 + 0x7073, // ( 7, 115) 6 + 0x2000, // ( 2, 0) 7 + 0xa0f7, // (10, 247) 8 + 0x601d, // ( 6, 29) 9 + 0x803b, // ( 8, 59) 10 + 0x4001, // ( 4, 1) 11 + 0xa2f7, // (10, 759) 12 + 0x5005, // ( 5, 5) 13 + 0x80bb, // ( 8, 187) 14 + 0x4009, // ( 4, 9) 15 + 0xb0cf, // (11, 207) 16 + 0x700b, // ( 7, 11) 17 + 0x9097, // ( 9, 151) 18 + 0x5015, // ( 5, 21) 19 + 0xb4cf, // (11, 1231) 20 + 0x704b, // ( 7, 75) 21 + 0x9197, // ( 9, 407) 22 + 0x500d, // ( 5, 13) 23 + 0xc0af, // (12, 175) 24 + 0x807b, // ( 8, 123) 25 + 0xb2cf, // (11, 719) 26 + 0x603d, // ( 6, 61) 27 + 0xc8af, // (12, 2223) 28 + 0x80fb, // ( 8, 251) 29 + 0xa1f7, // (10, 503) 30 + 0x6003, // ( 6, 3) 31 + 0xc4af, // (12, 1199) 32 + 0x8007, // ( 8, 7) 33 + 0xb6cf, // (11, 1743) 34 + 0x6023, // ( 6, 35) 35 + 0xccaf, // (12, 3247) 36 + 0x8087, // ( 8, 135) 37 + 0xa3f7, // (10, 1015) 38 + 0x6013, // ( 6, 19) 39 + 0xc2af, // (12, 687) 40 + 0x9057, // ( 9, 87) 41 + 0xcaaf, // (12, 2735) 42 + 0x702b, // ( 7, 43) 43 + 0xc6af, // (12, 1711) 44 + 0x9157, // ( 9, 343) 45 + 0xb1cf, // (11, 463) 46 + 0x706b, // ( 7, 107) 47 + 0xceaf, // (12, 3759) 48 + 0xa00f, // (10, 15) 49 + 0xc1af, // (12, 431) 50 + 0x8047, // ( 8, 71) 51 + 0xc9af, // (12, 2479) 52 + 0xa20f, // (10, 527) 53 + 0xc5af, // (12, 1455) 54 + 0x80c7, // ( 8, 199) 55 + 0xcdaf, // (12, 3503) 56 + 0xb5cf, // (11, 1487) 57 + 0xc3af, // (12, 943) 58 + 0x90d7, // ( 9, 215) 59 + 0xcbaf, // (12, 2991) 60 + 0xb3cf, // (11, 975) 61 + 0xc7af, // (12, 1967) 62 + 0x91d7, // ( 9, 471) 63 + 0xcfaf, // (12, 4015) 64 + 0x9037, // ( 9, 55) 65 + 0xc06f, // (12, 111) 66 + 0x701b, // ( 7, 27) 67 + 0xc86f, // (12, 2159) 68 + 0x9137, // ( 9, 311) 69 + 0xb7cf, // (11, 1999) 70 + 0x705b, // ( 7, 91) 71 + 0xc46f, // (12, 1135) 72 + 0xa10f, // (10, 271) 73 + 0xcc6f, // (12, 3183) 74 + 0x8027, // ( 8, 39) 75 + 0xc26f, // (12, 623) 76 + 0xa30f, // (10, 783) 77 + 0xca6f, // (12, 2671) 78 + 0x80a7, // ( 8, 167) 79 + 0xc66f, // (12, 1647) 80 + 0xb02f, // (11, 47) 81 + 0xce6f, // (12, 3695) 82 + 0x90b7, // ( 9, 183) 83 + 0xc16f, // (12, 367) 84 + 0xb42f, // (11, 1071) 85 + 0xc96f, // (12, 2415) 86 + 0x91b7, // ( 9, 439) 87 + 0xc56f, // (12, 1391) 88 + 0xcd6f, // (12, 3439) 89 + 0xc36f, // (12, 879) 90 + 0xa08f, // (10, 143) 91 + 0xcb6f, // (12, 2927) 92 + 0xc76f, // (12, 1903) 93 + 0xcf6f, // (12, 3951) 94 + 0xa28f, // (10, 655) 95 + 0xc0ef, // (12, 239) 96 + 0xc8ef, // (12, 2287) 97 + 0xc4ef, // (12, 1263) 98 + 0xa18f, // (10, 399) 99 + 0xccef, // (12, 3311) 100 + 0xc2ef, // (12, 751) 101 + 0xcaef, // (12, 2799) 102 + 0xa38f, // (10, 911) 103 + 0xc6ef, // (12, 1775) 104 + 0xceef, // (12, 3823) 105 + 0xc1ef, // (12, 495) 106 + 0xc9ef, // (12, 2543) 107 + 0xc5ef, // (12, 1519) 108 + 0xcdef, // (12, 3567) 109 + 0xc3ef, // (12, 1007) 110 + 0xb22f, // (11, 559) 111 + 0xcbef, // (12, 3055) 112 + 0xc7ef, // (12, 2031) 113 + 0xcfef, // (12, 4079) 114 + 0xc01f, // (12, 31) 115 + 0xc81f, // (12, 2079) 116 + 0xc41f, // (12, 1055) 117 + 0xcc1f, // (12, 3103) 118 + 0xc21f, // (12, 543) 119 + 0xca1f, // (12, 2591) 120 + 0xc61f, // (12, 1567) 121 + 0xce1f, // (12, 3615) 122 + 0xc11f, // (12, 287) 123 + 0xc91f, // (12, 2335) 124 + 0xc51f, // (12, 1311) 125 + 0xcd1f, // (12, 3359) 126 + 0xc31f, // (12, 799) 127 + 0xcb1f, // (12, 2847) 128 + 0xa04f, // (10, 79) 129 + 0xc71f, // (12, 1823) 130 + 0x8067, // ( 8, 103) 131 + 0xcf1f, // (12, 3871) 132 + 0xa24f, // (10, 591) 133 + 0xc09f, // (12, 159) 134 + 0x80e7, // ( 8, 231) 135 + 0xc89f, // (12, 2207) 136 + 0xb62f, // (11, 1583) 137 + 0xc49f, // (12, 1183) 138 + 0x9077, // ( 9, 119) 139 + 0xcc9f, // (12, 3231) 140 + 0xb12f, // (11, 303) 141 + 0xc29f, // (12, 671) 142 + 0x9177, // ( 9, 375) 143 + 0xca9f, // (12, 2719) 144 + 0xc69f, // (12, 1695) 145 + 0xce9f, // (12, 3743) 146 + 0xa14f, // (10, 335) 147 + 0xc19f, // (12, 415) 148 + 0xc99f, // (12, 2463) 149 + 0xc59f, // (12, 1439) 150 + 0xa34f, // (10, 847) 151 + 0xcd9f, // (12, 3487) 152 + 0xc39f, // (12, 927) 153 + 0xcb9f, // (12, 2975) 154 + 0xc79f, // (12, 1951) 155 + 0xcf9f, // (12, 3999) 156 + 0xc05f, // (12, 95) 157 + 0xc85f, // (12, 2143) 158 + 0xb52f, // (11, 1327) 159 + 0xc45f, // (12, 1119) 160 + 0xcc5f, // (12, 3167) 161 + 0xc25f, // (12, 607) 162 + 0xb32f, // (11, 815) 163 + 0xca5f, // (12, 2655) 164 + 0xc65f, // (12, 1631) 165 + 0xce5f, // (12, 3679) 166 + 0xb72f, // (11, 1839) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // Six Encoding Tables for the Midrange. + + // (table 16 of 22) (midrange 0 of 6) (c/k = 0.500000000 = 3.0 / 6.0) + // entropy: 2.1627885076675394949 + // avg_length: 2.2704182849800043087; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x1000, // ( 1, 0) 0 + 0x2001, // ( 2, 1) 1 + 0x4003, // ( 4, 3) 2 + 0x500b, // ( 5, 11) 3 + 0x501b, // ( 5, 27) 4 + 0x6007, // ( 6, 7) 5 + 0x8057, // ( 8, 87) 6 + 0x9077, // ( 9, 119) 7 + 0x6027, // ( 6, 39) 8 + 0x80d7, // ( 8, 215) 9 + 0x9177, // ( 9, 375) 10 + 0xa1f7, // (10, 503) 11 + 0xa3f7, // (10, 1015) 12 + 0xb08f, // (11, 143) 13 + 0xc58f, // (12, 1423) 14 + 0xcd8f, // (12, 3471) 15 + 0x7017, // ( 7, 23) 16 + 0x8037, // ( 8, 55) 17 + 0xa00f, // (10, 15) 18 + 0xb48f, // (11, 1167) 19 + 0xb28f, // (11, 655) 20 + 0xc38f, // (12, 911) 21 + 0xcb8f, // (12, 2959) 22 + 0xc78f, // (12, 1935) 23 + 0xcf8f, // (12, 3983) 24 + 0xc04f, // (12, 79) 25 + 0xc84f, // (12, 2127) 26 + 0xc44f, // (12, 1103) 27 + 0xcc4f, // (12, 3151) 28 + 0xc24f, // (12, 591) 29 + 0xca4f, // (12, 2639) 30 + 0xc64f, // (12, 1615) 31 + 0x80b7, // ( 8, 183) 32 + 0xa20f, // (10, 527) 33 + 0xb68f, // (11, 1679) 34 + 0xce4f, // (12, 3663) 35 + 0xc14f, // (12, 335) 36 + 0xc94f, // (12, 2383) 37 + 0xc54f, // (12, 1359) 38 + 0xcd4f, // (12, 3407) 39 + 0xc34f, // (12, 847) 40 + 0xcb4f, // (12, 2895) 41 + 0xc74f, // (12, 1871) 42 + 0xcf4f, // (12, 3919) 43 + 0xc0cf, // (12, 207) 44 + 0xc8cf, // (12, 2255) 45 + 0xc4cf, // (12, 1231) 46 + 0xcccf, // (12, 3279) 47 + 0xc2cf, // (12, 719) 48 + 0xcacf, // (12, 2767) 49 + 0xc6cf, // (12, 1743) 50 + 0xcecf, // (12, 3791) 51 + 0xc1cf, // (12, 463) 52 + 0xc9cf, // (12, 2511) 53 + 0xc5cf, // (12, 1487) 54 + 0xcdcf, // (12, 3535) 55 + 0xc3cf, // (12, 975) 56 + 0xcbcf, // (12, 3023) 57 + 0xc7cf, // (12, 1999) 58 + 0xcfcf, // (12, 4047) 59 + 0xc02f, // (12, 47) 60 + 0xc82f, // (12, 2095) 61 + 0xc42f, // (12, 1071) 62 + 0xcc2f, // (12, 3119) 63 + 0x90f7, // ( 9, 247) 64 + 0xa10f, // (10, 271) 65 + 0xc22f, // (12, 559) 66 + 0xca2f, // (12, 2607) 67 + 0xc62f, // (12, 1583) 68 + 0xce2f, // (12, 3631) 69 + 0xc12f, // (12, 303) 70 + 0xc92f, // (12, 2351) 71 + 0xc52f, // (12, 1327) 72 + 0xcd2f, // (12, 3375) 73 + 0xc32f, // (12, 815) 74 + 0xcb2f, // (12, 2863) 75 + 0xc72f, // (12, 1839) 76 + 0xcf2f, // (12, 3887) 77 + 0xc0af, // (12, 175) 78 + 0xc8af, // (12, 2223) 79 + 0xc4af, // (12, 1199) 80 + 0xccaf, // (12, 3247) 81 + 0xc2af, // (12, 687) 82 + 0xcaaf, // (12, 2735) 83 + 0xc6af, // (12, 1711) 84 + 0xceaf, // (12, 3759) 85 + 0xc1af, // (12, 431) 86 + 0xc9af, // (12, 2479) 87 + 0xc5af, // (12, 1455) 88 + 0xcdaf, // (12, 3503) 89 + 0xc3af, // (12, 943) 90 + 0xcbaf, // (12, 2991) 91 + 0xc7af, // (12, 1967) 92 + 0xcfaf, // (12, 4015) 93 + 0xc06f, // (12, 111) 94 + 0xc86f, // (12, 2159) 95 + 0xc46f, // (12, 1135) 96 + 0xcc6f, // (12, 3183) 97 + 0xc26f, // (12, 623) 98 + 0xca6f, // (12, 2671) 99 + 0xc66f, // (12, 1647) 100 + 0xce6f, // (12, 3695) 101 + 0xc16f, // (12, 367) 102 + 0xc96f, // (12, 2415) 103 + 0xc56f, // (12, 1391) 104 + 0xcd6f, // (12, 3439) 105 + 0xc36f, // (12, 879) 106 + 0xcb6f, // (12, 2927) 107 + 0xc76f, // (12, 1903) 108 + 0xcf6f, // (12, 3951) 109 + 0xc0ef, // (12, 239) 110 + 0xc8ef, // (12, 2287) 111 + 0xc4ef, // (12, 1263) 112 + 0xccef, // (12, 3311) 113 + 0xc2ef, // (12, 751) 114 + 0xcaef, // (12, 2799) 115 + 0xc6ef, // (12, 1775) 116 + 0xceef, // (12, 3823) 117 + 0xc1ef, // (12, 495) 118 + 0xc9ef, // (12, 2543) 119 + 0xc5ef, // (12, 1519) 120 + 0xcdef, // (12, 3567) 121 + 0xc3ef, // (12, 1007) 122 + 0xcbef, // (12, 3055) 123 + 0xc7ef, // (12, 2031) 124 + 0xcfef, // (12, 4079) 125 + 0xc01f, // (12, 31) 126 + 0xc81f, // (12, 2079) 127 + 0xa30f, // (10, 783) 128 + 0xb18f, // (11, 399) 129 + 0xc41f, // (12, 1055) 130 + 0xcc1f, // (12, 3103) 131 + 0xc21f, // (12, 543) 132 + 0xca1f, // (12, 2591) 133 + 0xc61f, // (12, 1567) 134 + 0xce1f, // (12, 3615) 135 + 0xc11f, // (12, 287) 136 + 0xc91f, // (12, 2335) 137 + 0xc51f, // (12, 1311) 138 + 0xcd1f, // (12, 3359) 139 + 0xc31f, // (12, 799) 140 + 0xcb1f, // (12, 2847) 141 + 0xc71f, // (12, 1823) 142 + 0xcf1f, // (12, 3871) 143 + 0xc09f, // (12, 159) 144 + 0xc89f, // (12, 2207) 145 + 0xc49f, // (12, 1183) 146 + 0xcc9f, // (12, 3231) 147 + 0xc29f, // (12, 671) 148 + 0xca9f, // (12, 2719) 149 + 0xc69f, // (12, 1695) 150 + 0xce9f, // (12, 3743) 151 + 0xc19f, // (12, 415) 152 + 0xc99f, // (12, 2463) 153 + 0xc59f, // (12, 1439) 154 + 0xcd9f, // (12, 3487) 155 + 0xc39f, // (12, 927) 156 + 0xcb9f, // (12, 2975) 157 + 0xc79f, // (12, 1951) 158 + 0xcf9f, // (12, 3999) 159 + 0xc05f, // (12, 95) 160 + 0xc85f, // (12, 2143) 161 + 0xc45f, // (12, 1119) 162 + 0xcc5f, // (12, 3167) 163 + 0xc25f, // (12, 607) 164 + 0xca5f, // (12, 2655) 165 + 0xc65f, // (12, 1631) 166 + 0xce5f, // (12, 3679) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 17 of 22) (midrange 1 of 6) (c/k = 0.833333333 = 5.0 / 6.0) + // entropy: 2.9553294756640680063 + // avg_length: 3.0766035704232641557; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x2000, // ( 2, 0) 0 + 0x2002, // ( 2, 2) 1 + 0x3001, // ( 3, 1) 2 + 0x4005, // ( 4, 5) 3 + 0x400d, // ( 4, 13) 4 + 0x5003, // ( 5, 3) 5 + 0x600b, // ( 6, 11) 6 + 0x602b, // ( 6, 43) 7 + 0x5013, // ( 5, 19) 8 + 0x601b, // ( 6, 27) 9 + 0x7007, // ( 7, 7) 10 + 0x7047, // ( 7, 71) 11 + 0x8017, // ( 8, 23) 12 + 0x90b7, // ( 9, 183) 13 + 0xa1f7, // (10, 503) 14 + 0xa3f7, // (10, 1015) 15 + 0x603b, // ( 6, 59) 16 + 0x7027, // ( 7, 39) 17 + 0x8097, // ( 8, 151) 18 + 0x8057, // ( 8, 87) 19 + 0x91b7, // ( 9, 439) 20 + 0xa00f, // (10, 15) 21 + 0xb18f, // (11, 399) 22 + 0xb58f, // (11, 1423) 23 + 0xa20f, // (10, 527) 24 + 0xb38f, // (11, 911) 25 + 0xc54f, // (12, 1359) 26 + 0xcd4f, // (12, 3407) 27 + 0xc34f, // (12, 847) 28 + 0xcb4f, // (12, 2895) 29 + 0xc74f, // (12, 1871) 30 + 0xcf4f, // (12, 3919) 31 + 0x7067, // ( 7, 103) 32 + 0x80d7, // ( 8, 215) 33 + 0x9077, // ( 9, 119) 34 + 0xa10f, // (10, 271) 35 + 0xa30f, // (10, 783) 36 + 0xb78f, // (11, 1935) 37 + 0xc0cf, // (12, 207) 38 + 0xc8cf, // (12, 2255) 39 + 0xb04f, // (11, 79) 40 + 0xc4cf, // (12, 1231) 41 + 0xcccf, // (12, 3279) 42 + 0xc2cf, // (12, 719) 43 + 0xcacf, // (12, 2767) 44 + 0xc6cf, // (12, 1743) 45 + 0xcecf, // (12, 3791) 46 + 0xc1cf, // (12, 463) 47 + 0xc9cf, // (12, 2511) 48 + 0xc5cf, // (12, 1487) 49 + 0xcdcf, // (12, 3535) 50 + 0xc3cf, // (12, 975) 51 + 0xcbcf, // (12, 3023) 52 + 0xc7cf, // (12, 1999) 53 + 0xcfcf, // (12, 4047) 54 + 0xc02f, // (12, 47) 55 + 0xc82f, // (12, 2095) 56 + 0xc42f, // (12, 1071) 57 + 0xcc2f, // (12, 3119) 58 + 0xc22f, // (12, 559) 59 + 0xca2f, // (12, 2607) 60 + 0xc62f, // (12, 1583) 61 + 0xce2f, // (12, 3631) 62 + 0xc12f, // (12, 303) 63 + 0x8037, // ( 8, 55) 64 + 0x9177, // ( 9, 375) 65 + 0xa08f, // (10, 143) 66 + 0xb44f, // (11, 1103) 67 + 0xb24f, // (11, 591) 68 + 0xc92f, // (12, 2351) 69 + 0xc52f, // (12, 1327) 70 + 0xcd2f, // (12, 3375) 71 + 0xc32f, // (12, 815) 72 + 0xcb2f, // (12, 2863) 73 + 0xc72f, // (12, 1839) 74 + 0xcf2f, // (12, 3887) 75 + 0xc0af, // (12, 175) 76 + 0xc8af, // (12, 2223) 77 + 0xc4af, // (12, 1199) 78 + 0xccaf, // (12, 3247) 79 + 0xc2af, // (12, 687) 80 + 0xcaaf, // (12, 2735) 81 + 0xc6af, // (12, 1711) 82 + 0xceaf, // (12, 3759) 83 + 0xc1af, // (12, 431) 84 + 0xc9af, // (12, 2479) 85 + 0xc5af, // (12, 1455) 86 + 0xcdaf, // (12, 3503) 87 + 0xc3af, // (12, 943) 88 + 0xcbaf, // (12, 2991) 89 + 0xc7af, // (12, 1967) 90 + 0xcfaf, // (12, 4015) 91 + 0xc06f, // (12, 111) 92 + 0xc86f, // (12, 2159) 93 + 0xc46f, // (12, 1135) 94 + 0xcc6f, // (12, 3183) 95 + 0xc26f, // (12, 623) 96 + 0xca6f, // (12, 2671) 97 + 0xc66f, // (12, 1647) 98 + 0xce6f, // (12, 3695) 99 + 0xc16f, // (12, 367) 100 + 0xc96f, // (12, 2415) 101 + 0xc56f, // (12, 1391) 102 + 0xcd6f, // (12, 3439) 103 + 0xc36f, // (12, 879) 104 + 0xcb6f, // (12, 2927) 105 + 0xc76f, // (12, 1903) 106 + 0xcf6f, // (12, 3951) 107 + 0xc0ef, // (12, 239) 108 + 0xc8ef, // (12, 2287) 109 + 0xc4ef, // (12, 1263) 110 + 0xccef, // (12, 3311) 111 + 0xc2ef, // (12, 751) 112 + 0xcaef, // (12, 2799) 113 + 0xc6ef, // (12, 1775) 114 + 0xceef, // (12, 3823) 115 + 0xc1ef, // (12, 495) 116 + 0xc9ef, // (12, 2543) 117 + 0xc5ef, // (12, 1519) 118 + 0xcdef, // (12, 3567) 119 + 0xc3ef, // (12, 1007) 120 + 0xcbef, // (12, 3055) 121 + 0xc7ef, // (12, 2031) 122 + 0xcfef, // (12, 4079) 123 + 0xc01f, // (12, 31) 124 + 0xc81f, // (12, 2079) 125 + 0xc41f, // (12, 1055) 126 + 0xcc1f, // (12, 3103) 127 + 0x90f7, // ( 9, 247) 128 + 0xa28f, // (10, 655) 129 + 0xb64f, // (11, 1615) 130 + 0xb14f, // (11, 335) 131 + 0xc21f, // (12, 543) 132 + 0xca1f, // (12, 2591) 133 + 0xc61f, // (12, 1567) 134 + 0xce1f, // (12, 3615) 135 + 0xc11f, // (12, 287) 136 + 0xc91f, // (12, 2335) 137 + 0xc51f, // (12, 1311) 138 + 0xcd1f, // (12, 3359) 139 + 0xc31f, // (12, 799) 140 + 0xcb1f, // (12, 2847) 141 + 0xc71f, // (12, 1823) 142 + 0xcf1f, // (12, 3871) 143 + 0xc09f, // (12, 159) 144 + 0xc89f, // (12, 2207) 145 + 0xc49f, // (12, 1183) 146 + 0xcc9f, // (12, 3231) 147 + 0xc29f, // (12, 671) 148 + 0xca9f, // (12, 2719) 149 + 0xc69f, // (12, 1695) 150 + 0xce9f, // (12, 3743) 151 + 0xc19f, // (12, 415) 152 + 0xc99f, // (12, 2463) 153 + 0xc59f, // (12, 1439) 154 + 0xcd9f, // (12, 3487) 155 + 0xc39f, // (12, 927) 156 + 0xcb9f, // (12, 2975) 157 + 0xc79f, // (12, 1951) 158 + 0xcf9f, // (12, 3999) 159 + 0xc05f, // (12, 95) 160 + 0xc85f, // (12, 2143) 161 + 0xc45f, // (12, 1119) 162 + 0xcc5f, // (12, 3167) 163 + 0xc25f, // (12, 607) 164 + 0xca5f, // (12, 2655) 165 + 0xc65f, // (12, 1631) 166 + 0xce5f, // (12, 3679) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 18 of 22) (midrange 2 of 6) (c/k = 1.166666667 = 7.0 / 6.0) + // entropy: 3.5218672531711128215 + // avg_length: 3.6153551492375441967; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x2000, // ( 2, 0) 0 + 0x2002, // ( 2, 2) 1 + 0x4005, // ( 4, 5) 2 + 0x3001, // ( 3, 1) 3 + 0x5003, // ( 5, 3) 4 + 0x400d, // ( 4, 13) 5 + 0x600b, // ( 6, 11) 6 + 0x602b, // ( 6, 43) 7 + 0x601b, // ( 6, 27) 8 + 0x5013, // ( 5, 19) 9 + 0x703b, // ( 7, 59) 10 + 0x707b, // ( 7, 123) 11 + 0x8067, // ( 8, 103) 12 + 0x80e7, // ( 8, 231) 13 + 0x90d7, // ( 9, 215) 14 + 0x91d7, // ( 9, 471) 15 + 0x7007, // ( 7, 7) 16 + 0x7047, // ( 7, 71) 17 + 0x8017, // ( 8, 23) 18 + 0x8097, // ( 8, 151) 19 + 0x9037, // ( 9, 55) 20 + 0x9137, // ( 9, 311) 21 + 0xa1f7, // (10, 503) 22 + 0xa3f7, // (10, 1015) 23 + 0xa00f, // (10, 15) 24 + 0xa20f, // (10, 527) 25 + 0xb38f, // (11, 911) 26 + 0xb78f, // (11, 1935) 27 + 0xc0cf, // (12, 207) 28 + 0xc8cf, // (12, 2255) 29 + 0xc4cf, // (12, 1231) 30 + 0xcccf, // (12, 3279) 31 + 0x8057, // ( 8, 87) 32 + 0x7027, // ( 7, 39) 33 + 0x90b7, // ( 9, 183) 34 + 0x91b7, // ( 9, 439) 35 + 0xa10f, // (10, 271) 36 + 0xa30f, // (10, 783) 37 + 0xb04f, // (11, 79) 38 + 0xb44f, // (11, 1103) 39 + 0xb24f, // (11, 591) 40 + 0xb64f, // (11, 1615) 41 + 0xc2cf, // (12, 719) 42 + 0xcacf, // (12, 2767) 43 + 0xc6cf, // (12, 1743) 44 + 0xcecf, // (12, 3791) 45 + 0xc1cf, // (12, 463) 46 + 0xc9cf, // (12, 2511) 47 + 0xc5cf, // (12, 1487) 48 + 0xcdcf, // (12, 3535) 49 + 0xc3cf, // (12, 975) 50 + 0xcbcf, // (12, 3023) 51 + 0xc7cf, // (12, 1999) 52 + 0xcfcf, // (12, 4047) 53 + 0xc02f, // (12, 47) 54 + 0xc82f, // (12, 2095) 55 + 0xc42f, // (12, 1071) 56 + 0xcc2f, // (12, 3119) 57 + 0xc22f, // (12, 559) 58 + 0xca2f, // (12, 2607) 59 + 0xc62f, // (12, 1583) 60 + 0xce2f, // (12, 3631) 61 + 0xc12f, // (12, 303) 62 + 0xc92f, // (12, 2351) 63 + 0x9077, // ( 9, 119) 64 + 0x9177, // ( 9, 375) 65 + 0xa08f, // (10, 143) 66 + 0xa28f, // (10, 655) 67 + 0xb14f, // (11, 335) 68 + 0xb54f, // (11, 1359) 69 + 0xc52f, // (12, 1327) 70 + 0xcd2f, // (12, 3375) 71 + 0xc32f, // (12, 815) 72 + 0xcb2f, // (12, 2863) 73 + 0xc72f, // (12, 1839) 74 + 0xcf2f, // (12, 3887) 75 + 0xc0af, // (12, 175) 76 + 0xc8af, // (12, 2223) 77 + 0xc4af, // (12, 1199) 78 + 0xccaf, // (12, 3247) 79 + 0xc2af, // (12, 687) 80 + 0xcaaf, // (12, 2735) 81 + 0xc6af, // (12, 1711) 82 + 0xceaf, // (12, 3759) 83 + 0xc1af, // (12, 431) 84 + 0xc9af, // (12, 2479) 85 + 0xc5af, // (12, 1455) 86 + 0xcdaf, // (12, 3503) 87 + 0xc3af, // (12, 943) 88 + 0xcbaf, // (12, 2991) 89 + 0xc7af, // (12, 1967) 90 + 0xcfaf, // (12, 4015) 91 + 0xc06f, // (12, 111) 92 + 0xc86f, // (12, 2159) 93 + 0xc46f, // (12, 1135) 94 + 0xcc6f, // (12, 3183) 95 + 0xc26f, // (12, 623) 96 + 0xca6f, // (12, 2671) 97 + 0xc66f, // (12, 1647) 98 + 0xce6f, // (12, 3695) 99 + 0xc16f, // (12, 367) 100 + 0xc96f, // (12, 2415) 101 + 0xc56f, // (12, 1391) 102 + 0xcd6f, // (12, 3439) 103 + 0xc36f, // (12, 879) 104 + 0xcb6f, // (12, 2927) 105 + 0xc76f, // (12, 1903) 106 + 0xcf6f, // (12, 3951) 107 + 0xc0ef, // (12, 239) 108 + 0xc8ef, // (12, 2287) 109 + 0xc4ef, // (12, 1263) 110 + 0xccef, // (12, 3311) 111 + 0xc2ef, // (12, 751) 112 + 0xcaef, // (12, 2799) 113 + 0xc6ef, // (12, 1775) 114 + 0xceef, // (12, 3823) 115 + 0xc1ef, // (12, 495) 116 + 0xc9ef, // (12, 2543) 117 + 0xc5ef, // (12, 1519) 118 + 0xcdef, // (12, 3567) 119 + 0xc3ef, // (12, 1007) 120 + 0xcbef, // (12, 3055) 121 + 0xc7ef, // (12, 2031) 122 + 0xcfef, // (12, 4079) 123 + 0xc01f, // (12, 31) 124 + 0xc81f, // (12, 2079) 125 + 0xc41f, // (12, 1055) 126 + 0xcc1f, // (12, 3103) 127 + 0xa18f, // (10, 399) 128 + 0x90f7, // ( 9, 247) 129 + 0xb34f, // (11, 847) 130 + 0xb74f, // (11, 1871) 131 + 0xc21f, // (12, 543) 132 + 0xca1f, // (12, 2591) 133 + 0xc61f, // (12, 1567) 134 + 0xce1f, // (12, 3615) 135 + 0xc11f, // (12, 287) 136 + 0xc91f, // (12, 2335) 137 + 0xc51f, // (12, 1311) 138 + 0xcd1f, // (12, 3359) 139 + 0xc31f, // (12, 799) 140 + 0xcb1f, // (12, 2847) 141 + 0xc71f, // (12, 1823) 142 + 0xcf1f, // (12, 3871) 143 + 0xc09f, // (12, 159) 144 + 0xc89f, // (12, 2207) 145 + 0xc49f, // (12, 1183) 146 + 0xcc9f, // (12, 3231) 147 + 0xc29f, // (12, 671) 148 + 0xca9f, // (12, 2719) 149 + 0xc69f, // (12, 1695) 150 + 0xce9f, // (12, 3743) 151 + 0xc19f, // (12, 415) 152 + 0xc99f, // (12, 2463) 153 + 0xc59f, // (12, 1439) 154 + 0xcd9f, // (12, 3487) 155 + 0xc39f, // (12, 927) 156 + 0xcb9f, // (12, 2975) 157 + 0xc79f, // (12, 1951) 158 + 0xcf9f, // (12, 3999) 159 + 0xc05f, // (12, 95) 160 + 0xc85f, // (12, 2143) 161 + 0xc45f, // (12, 1119) 162 + 0xcc5f, // (12, 3167) 163 + 0xc25f, // (12, 607) 164 + 0xca5f, // (12, 2655) 165 + 0xc65f, // (12, 1631) 166 + 0xce5f, // (12, 3679) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 19 of 22) (midrange 3 of 6) (c/k = 1.500000000 = 9.0 / 6.0) + // entropy: 3.9228873257934386842 + // avg_length: 3.9989687586992346269; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x3002, // ( 3, 2) 0 + 0x2000, // ( 2, 0) 1 + 0x4001, // ( 4, 1) 2 + 0x3006, // ( 3, 6) 3 + 0x500d, // ( 5, 13) 4 + 0x4009, // ( 4, 9) 5 + 0x501d, // ( 5, 29) 6 + 0x4005, // ( 4, 5) 7 + 0x6013, // ( 6, 19) 8 + 0x5003, // ( 5, 3) 9 + 0x6033, // ( 6, 51) 10 + 0x600b, // ( 6, 11) 11 + 0x8027, // ( 8, 39) 12 + 0x701b, // ( 7, 27) 13 + 0x80a7, // ( 8, 167) 14 + 0x705b, // ( 7, 91) 15 + 0x703b, // ( 7, 59) 16 + 0x602b, // ( 6, 43) 17 + 0x707b, // ( 7, 123) 18 + 0x7007, // ( 7, 7) 19 + 0x90d7, // ( 9, 215) 20 + 0x8067, // ( 8, 103) 21 + 0x91d7, // ( 9, 471) 22 + 0x80e7, // ( 8, 231) 23 + 0xa1f7, // (10, 503) 24 + 0x9037, // ( 9, 55) 25 + 0xa3f7, // (10, 1015) 26 + 0xa00f, // (10, 15) 27 + 0xc5cf, // (12, 1487) 28 + 0xb04f, // (11, 79) 29 + 0xcdcf, // (12, 3535) 30 + 0xb44f, // (11, 1103) 31 + 0x8017, // ( 8, 23) 32 + 0x7047, // ( 7, 71) 33 + 0x9137, // ( 9, 311) 34 + 0x8097, // ( 8, 151) 35 + 0xa20f, // (10, 527) 36 + 0x90b7, // ( 9, 183) 37 + 0xa10f, // (10, 271) 38 + 0x91b7, // ( 9, 439) 39 + 0xb24f, // (11, 591) 40 + 0xa30f, // (10, 783) 41 + 0xb64f, // (11, 1615) 42 + 0xb14f, // (11, 335) 43 + 0xc3cf, // (12, 975) 44 + 0xcbcf, // (12, 3023) 45 + 0xc7cf, // (12, 1999) 46 + 0xcfcf, // (12, 4047) 47 + 0xc02f, // (12, 47) 48 + 0xb54f, // (11, 1359) 49 + 0xc82f, // (12, 2095) 50 + 0xc42f, // (12, 1071) 51 + 0xcc2f, // (12, 3119) 52 + 0xc22f, // (12, 559) 53 + 0xca2f, // (12, 2607) 54 + 0xc62f, // (12, 1583) 55 + 0xce2f, // (12, 3631) 56 + 0xc12f, // (12, 303) 57 + 0xc92f, // (12, 2351) 58 + 0xc52f, // (12, 1327) 59 + 0xcd2f, // (12, 3375) 60 + 0xc32f, // (12, 815) 61 + 0xcb2f, // (12, 2863) 62 + 0xc72f, // (12, 1839) 63 + 0x9077, // ( 9, 119) 64 + 0x8057, // ( 8, 87) 65 + 0xa08f, // (10, 143) 66 + 0x9177, // ( 9, 375) 67 + 0xb34f, // (11, 847) 68 + 0xa28f, // (10, 655) 69 + 0xb74f, // (11, 1871) 70 + 0xb0cf, // (11, 207) 71 + 0xcf2f, // (12, 3887) 72 + 0xb4cf, // (11, 1231) 73 + 0xc0af, // (12, 175) 74 + 0xc8af, // (12, 2223) 75 + 0xc4af, // (12, 1199) 76 + 0xccaf, // (12, 3247) 77 + 0xc2af, // (12, 687) 78 + 0xcaaf, // (12, 2735) 79 + 0xc6af, // (12, 1711) 80 + 0xceaf, // (12, 3759) 81 + 0xc1af, // (12, 431) 82 + 0xc9af, // (12, 2479) 83 + 0xc5af, // (12, 1455) 84 + 0xcdaf, // (12, 3503) 85 + 0xc3af, // (12, 943) 86 + 0xcbaf, // (12, 2991) 87 + 0xc7af, // (12, 1967) 88 + 0xcfaf, // (12, 4015) 89 + 0xc06f, // (12, 111) 90 + 0xc86f, // (12, 2159) 91 + 0xc46f, // (12, 1135) 92 + 0xcc6f, // (12, 3183) 93 + 0xc26f, // (12, 623) 94 + 0xca6f, // (12, 2671) 95 + 0xc66f, // (12, 1647) 96 + 0xce6f, // (12, 3695) 97 + 0xc16f, // (12, 367) 98 + 0xc96f, // (12, 2415) 99 + 0xc56f, // (12, 1391) 100 + 0xcd6f, // (12, 3439) 101 + 0xc36f, // (12, 879) 102 + 0xcb6f, // (12, 2927) 103 + 0xc76f, // (12, 1903) 104 + 0xcf6f, // (12, 3951) 105 + 0xc0ef, // (12, 239) 106 + 0xc8ef, // (12, 2287) 107 + 0xc4ef, // (12, 1263) 108 + 0xccef, // (12, 3311) 109 + 0xc2ef, // (12, 751) 110 + 0xcaef, // (12, 2799) 111 + 0xc6ef, // (12, 1775) 112 + 0xceef, // (12, 3823) 113 + 0xc1ef, // (12, 495) 114 + 0xc9ef, // (12, 2543) 115 + 0xc5ef, // (12, 1519) 116 + 0xcdef, // (12, 3567) 117 + 0xc3ef, // (12, 1007) 118 + 0xcbef, // (12, 3055) 119 + 0xc7ef, // (12, 2031) 120 + 0xcfef, // (12, 4079) 121 + 0xc01f, // (12, 31) 122 + 0xc81f, // (12, 2079) 123 + 0xc41f, // (12, 1055) 124 + 0xcc1f, // (12, 3103) 125 + 0xc21f, // (12, 543) 126 + 0xca1f, // (12, 2591) 127 + 0xa18f, // (10, 399) 128 + 0x90f7, // ( 9, 247) 129 + 0xb2cf, // (11, 719) 130 + 0xa38f, // (10, 911) 131 + 0xc61f, // (12, 1567) 132 + 0xb6cf, // (11, 1743) 133 + 0xce1f, // (12, 3615) 134 + 0xb1cf, // (11, 463) 135 + 0xc11f, // (12, 287) 136 + 0xc91f, // (12, 2335) 137 + 0xc51f, // (12, 1311) 138 + 0xcd1f, // (12, 3359) 139 + 0xc31f, // (12, 799) 140 + 0xcb1f, // (12, 2847) 141 + 0xc71f, // (12, 1823) 142 + 0xcf1f, // (12, 3871) 143 + 0xc09f, // (12, 159) 144 + 0xc89f, // (12, 2207) 145 + 0xc49f, // (12, 1183) 146 + 0xcc9f, // (12, 3231) 147 + 0xc29f, // (12, 671) 148 + 0xca9f, // (12, 2719) 149 + 0xc69f, // (12, 1695) 150 + 0xce9f, // (12, 3743) 151 + 0xc19f, // (12, 415) 152 + 0xc99f, // (12, 2463) 153 + 0xc59f, // (12, 1439) 154 + 0xcd9f, // (12, 3487) 155 + 0xc39f, // (12, 927) 156 + 0xcb9f, // (12, 2975) 157 + 0xc79f, // (12, 1951) 158 + 0xcf9f, // (12, 3999) 159 + 0xc05f, // (12, 95) 160 + 0xc85f, // (12, 2143) 161 + 0xc45f, // (12, 1119) 162 + 0xcc5f, // (12, 3167) 163 + 0xc25f, // (12, 607) 164 + 0xca5f, // (12, 2655) 165 + 0xc65f, // (12, 1631) 166 + 0xce5f, // (12, 3679) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 20 of 22) (midrange 4 of 6) (c/k = 1.833333333 = 11.0 / 6.0) + // entropy: 4.1937026483207340277 + // avg_length: 4.2809622975207295426; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x4006, // ( 4, 6) 0 + 0x2000, // ( 2, 0) 1 + 0x400e, // ( 4, 14) 2 + 0x3002, // ( 3, 2) 3 + 0x5005, // ( 5, 5) 4 + 0x4001, // ( 4, 1) 5 + 0x5015, // ( 5, 21) 6 + 0x4009, // ( 4, 9) 7 + 0x6003, // ( 6, 3) 8 + 0x500d, // ( 5, 13) 9 + 0x6023, // ( 6, 35) 10 + 0x501d, // ( 5, 29) 11 + 0x8047, // ( 8, 71) 12 + 0x6013, // ( 6, 19) 13 + 0x80c7, // ( 8, 199) 14 + 0x6033, // ( 6, 51) 15 + 0x701b, // ( 7, 27) 16 + 0x600b, // ( 6, 11) 17 + 0x8027, // ( 8, 39) 18 + 0x602b, // ( 6, 43) 19 + 0x90d7, // ( 9, 215) 20 + 0x705b, // ( 7, 91) 21 + 0x91d7, // ( 9, 471) 22 + 0x703b, // ( 7, 59) 23 + 0xa1f7, // (10, 503) 24 + 0x80a7, // ( 8, 167) 25 + 0xa3f7, // (10, 1015) 26 + 0x8067, // ( 8, 103) 27 + 0xb24f, // (11, 591) 28 + 0xa00f, // (10, 15) 29 + 0xb64f, // (11, 1615) 30 + 0xa20f, // (10, 527) 31 + 0x9037, // ( 9, 55) 32 + 0x707b, // ( 7, 123) 33 + 0x9137, // ( 9, 311) 34 + 0x7007, // ( 7, 7) 35 + 0xa10f, // (10, 271) 36 + 0x80e7, // ( 8, 231) 37 + 0xa30f, // (10, 783) 38 + 0x8017, // ( 8, 23) 39 + 0xb14f, // (11, 335) 40 + 0x90b7, // ( 9, 183) 41 + 0xb54f, // (11, 1359) 42 + 0xa08f, // (10, 143) 43 + 0xc02f, // (12, 47) 44 + 0xb34f, // (11, 847) 45 + 0xc82f, // (12, 2095) 46 + 0xb74f, // (11, 1871) 47 + 0xc42f, // (12, 1071) 48 + 0xb0cf, // (11, 207) 49 + 0xcc2f, // (12, 3119) 50 + 0xb4cf, // (11, 1231) 51 + 0xc22f, // (12, 559) 52 + 0xca2f, // (12, 2607) 53 + 0xc62f, // (12, 1583) 54 + 0xce2f, // (12, 3631) 55 + 0xc12f, // (12, 303) 56 + 0xc92f, // (12, 2351) 57 + 0xc52f, // (12, 1327) 58 + 0xcd2f, // (12, 3375) 59 + 0xc32f, // (12, 815) 60 + 0xcb2f, // (12, 2863) 61 + 0xc72f, // (12, 1839) 62 + 0xcf2f, // (12, 3887) 63 + 0xa28f, // (10, 655) 64 + 0x8097, // ( 8, 151) 65 + 0xa18f, // (10, 399) 66 + 0x8057, // ( 8, 87) 67 + 0xb2cf, // (11, 719) 68 + 0x91b7, // ( 9, 439) 69 + 0xb6cf, // (11, 1743) 70 + 0x9077, // ( 9, 119) 71 + 0xc0af, // (12, 175) 72 + 0xb1cf, // (11, 463) 73 + 0xc8af, // (12, 2223) 74 + 0xb5cf, // (11, 1487) 75 + 0xc4af, // (12, 1199) 76 + 0xccaf, // (12, 3247) 77 + 0xc2af, // (12, 687) 78 + 0xcaaf, // (12, 2735) 79 + 0xc6af, // (12, 1711) 80 + 0xceaf, // (12, 3759) 81 + 0xc1af, // (12, 431) 82 + 0xc9af, // (12, 2479) 83 + 0xc5af, // (12, 1455) 84 + 0xcdaf, // (12, 3503) 85 + 0xc3af, // (12, 943) 86 + 0xcbaf, // (12, 2991) 87 + 0xc7af, // (12, 1967) 88 + 0xcfaf, // (12, 4015) 89 + 0xc06f, // (12, 111) 90 + 0xc86f, // (12, 2159) 91 + 0xc46f, // (12, 1135) 92 + 0xcc6f, // (12, 3183) 93 + 0xc26f, // (12, 623) 94 + 0xca6f, // (12, 2671) 95 + 0xc66f, // (12, 1647) 96 + 0xce6f, // (12, 3695) 97 + 0xc16f, // (12, 367) 98 + 0xc96f, // (12, 2415) 99 + 0xc56f, // (12, 1391) 100 + 0xcd6f, // (12, 3439) 101 + 0xc36f, // (12, 879) 102 + 0xcb6f, // (12, 2927) 103 + 0xc76f, // (12, 1903) 104 + 0xcf6f, // (12, 3951) 105 + 0xc0ef, // (12, 239) 106 + 0xc8ef, // (12, 2287) 107 + 0xc4ef, // (12, 1263) 108 + 0xccef, // (12, 3311) 109 + 0xc2ef, // (12, 751) 110 + 0xcaef, // (12, 2799) 111 + 0xc6ef, // (12, 1775) 112 + 0xceef, // (12, 3823) 113 + 0xc1ef, // (12, 495) 114 + 0xc9ef, // (12, 2543) 115 + 0xc5ef, // (12, 1519) 116 + 0xcdef, // (12, 3567) 117 + 0xc3ef, // (12, 1007) 118 + 0xcbef, // (12, 3055) 119 + 0xc7ef, // (12, 2031) 120 + 0xcfef, // (12, 4079) 121 + 0xc01f, // (12, 31) 122 + 0xc81f, // (12, 2079) 123 + 0xc41f, // (12, 1055) 124 + 0xcc1f, // (12, 3103) 125 + 0xc21f, // (12, 543) 126 + 0xca1f, // (12, 2591) 127 + 0xb3cf, // (11, 975) 128 + 0x9177, // ( 9, 375) 129 + 0xb7cf, // (11, 1999) 130 + 0x90f7, // ( 9, 247) 131 + 0xc61f, // (12, 1567) 132 + 0xa38f, // (10, 911) 133 + 0xce1f, // (12, 3615) 134 + 0xa04f, // (10, 79) 135 + 0xc11f, // (12, 287) 136 + 0xc91f, // (12, 2335) 137 + 0xc51f, // (12, 1311) 138 + 0xcd1f, // (12, 3359) 139 + 0xc31f, // (12, 799) 140 + 0xcb1f, // (12, 2847) 141 + 0xc71f, // (12, 1823) 142 + 0xcf1f, // (12, 3871) 143 + 0xc09f, // (12, 159) 144 + 0xc89f, // (12, 2207) 145 + 0xc49f, // (12, 1183) 146 + 0xcc9f, // (12, 3231) 147 + 0xc29f, // (12, 671) 148 + 0xca9f, // (12, 2719) 149 + 0xc69f, // (12, 1695) 150 + 0xce9f, // (12, 3743) 151 + 0xc19f, // (12, 415) 152 + 0xc99f, // (12, 2463) 153 + 0xc59f, // (12, 1439) 154 + 0xcd9f, // (12, 3487) 155 + 0xc39f, // (12, 927) 156 + 0xcb9f, // (12, 2975) 157 + 0xc79f, // (12, 1951) 158 + 0xcf9f, // (12, 3999) 159 + 0xc05f, // (12, 95) 160 + 0xc85f, // (12, 2143) 161 + 0xc45f, // (12, 1119) 162 + 0xcc5f, // (12, 3167) 163 + 0xc25f, // (12, 607) 164 + 0xca5f, // (12, 2655) 165 + 0xc65f, // (12, 1631) 166 + 0xce5f, // (12, 3679) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +}, + + // (table 21 of 22) (midrange 5 of 6) (c/k = 2.166666667 = 13.0 / 6.0) + // entropy: 4.3601926041863263706 + // avg_length: 4.4384101723259572481; max_length = 12; num_symbols = 256 +{ +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x5009, // ( 5, 9) 0 + 0x3002, // ( 3, 2) 1 + 0x5019, // ( 5, 25) 2 + 0x2000, // ( 2, 0) 3 + 0x6003, // ( 6, 3) 4 + 0x4001, // ( 4, 1) 5 + 0x5005, // ( 5, 5) 6 + 0x3006, // ( 3, 6) 7 + 0x702b, // ( 7, 43) 8 + 0x5015, // ( 5, 21) 9 + 0x706b, // ( 7, 107) 10 + 0x500d, // ( 5, 13) 11 + 0x8007, // ( 8, 7) 12 + 0x6023, // ( 6, 35) 13 + 0x8087, // ( 8, 135) 14 + 0x501d, // ( 5, 29) 15 + 0x8047, // ( 8, 71) 16 + 0x6013, // ( 6, 19) 17 + 0x80c7, // ( 8, 199) 18 + 0x6033, // ( 6, 51) 19 + 0x9097, // ( 9, 151) 20 + 0x701b, // ( 7, 27) 21 + 0x9197, // ( 9, 407) 22 + 0x600b, // ( 6, 11) 23 + 0xa0f7, // (10, 247) 24 + 0x8027, // ( 8, 39) 25 + 0xa2f7, // (10, 759) 26 + 0x80a7, // ( 8, 167) 27 + 0xb14f, // (11, 335) 28 + 0x9057, // ( 9, 87) 29 + 0xb54f, // (11, 1359) 30 + 0x9157, // ( 9, 343) 31 + 0x90d7, // ( 9, 215) 32 + 0x705b, // ( 7, 91) 33 + 0x91d7, // ( 9, 471) 34 + 0x703b, // ( 7, 59) 35 + 0xa1f7, // (10, 503) 36 + 0x8067, // ( 8, 103) 37 + 0xa3f7, // (10, 1015) 38 + 0x707b, // ( 7, 123) 39 + 0xb34f, // (11, 847) 40 + 0x9037, // ( 9, 55) 41 + 0xb74f, // (11, 1871) 42 + 0x9137, // ( 9, 311) 43 + 0xc12f, // (12, 303) 44 + 0xa00f, // (10, 15) 45 + 0xc92f, // (12, 2351) 46 + 0xa20f, // (10, 527) 47 + 0xc52f, // (12, 1327) 48 + 0xa10f, // (10, 271) 49 + 0xcd2f, // (12, 3375) 50 + 0xa30f, // (10, 783) 51 + 0xc32f, // (12, 815) 52 + 0xb0cf, // (11, 207) 53 + 0xcb2f, // (12, 2863) 54 + 0xb4cf, // (11, 1231) 55 + 0xc72f, // (12, 1839) 56 + 0xcf2f, // (12, 3887) 57 + 0xc0af, // (12, 175) 58 + 0xc8af, // (12, 2223) 59 + 0xc4af, // (12, 1199) 60 + 0xccaf, // (12, 3247) 61 + 0xc2af, // (12, 687) 62 + 0xcaaf, // (12, 2735) 63 + 0xa08f, // (10, 143) 64 + 0x80e7, // ( 8, 231) 65 + 0xa28f, // (10, 655) 66 + 0x8017, // ( 8, 23) 67 + 0xb2cf, // (11, 719) 68 + 0x90b7, // ( 9, 183) 69 + 0xb6cf, // (11, 1743) 70 + 0x91b7, // ( 9, 439) 71 + 0xc6af, // (12, 1711) 72 + 0xa18f, // (10, 399) 73 + 0xceaf, // (12, 3759) 74 + 0xa38f, // (10, 911) 75 + 0xc1af, // (12, 431) 76 + 0xb1cf, // (11, 463) 77 + 0xc9af, // (12, 2479) 78 + 0xb5cf, // (11, 1487) 79 + 0xc5af, // (12, 1455) 80 + 0xb3cf, // (11, 975) 81 + 0xcdaf, // (12, 3503) 82 + 0xb7cf, // (11, 1999) 83 + 0xc3af, // (12, 943) 84 + 0xcbaf, // (12, 2991) 85 + 0xc7af, // (12, 1967) 86 + 0xcfaf, // (12, 4015) 87 + 0xc06f, // (12, 111) 88 + 0xc86f, // (12, 2159) 89 + 0xc46f, // (12, 1135) 90 + 0xcc6f, // (12, 3183) 91 + 0xc26f, // (12, 623) 92 + 0xca6f, // (12, 2671) 93 + 0xc66f, // (12, 1647) 94 + 0xce6f, // (12, 3695) 95 + 0xc16f, // (12, 367) 96 + 0xc96f, // (12, 2415) 97 + 0xc56f, // (12, 1391) 98 + 0xcd6f, // (12, 3439) 99 + 0xc36f, // (12, 879) 100 + 0xcb6f, // (12, 2927) 101 + 0xc76f, // (12, 1903) 102 + 0xcf6f, // (12, 3951) 103 + 0xc0ef, // (12, 239) 104 + 0xc8ef, // (12, 2287) 105 + 0xc4ef, // (12, 1263) 106 + 0xccef, // (12, 3311) 107 + 0xc2ef, // (12, 751) 108 + 0xcaef, // (12, 2799) 109 + 0xc6ef, // (12, 1775) 110 + 0xceef, // (12, 3823) 111 + 0xc1ef, // (12, 495) 112 + 0xc9ef, // (12, 2543) 113 + 0xc5ef, // (12, 1519) 114 + 0xcdef, // (12, 3567) 115 + 0xc3ef, // (12, 1007) 116 + 0xcbef, // (12, 3055) 117 + 0xc7ef, // (12, 2031) 118 + 0xcfef, // (12, 4079) 119 + 0xc01f, // (12, 31) 120 + 0xc81f, // (12, 2079) 121 + 0xc41f, // (12, 1055) 122 + 0xcc1f, // (12, 3103) 123 + 0xc21f, // (12, 543) 124 + 0xca1f, // (12, 2591) 125 + 0xc61f, // (12, 1567) 126 + 0xce1f, // (12, 3615) 127 + 0xb02f, // (11, 47) 128 + 0x9077, // ( 9, 119) 129 + 0xb42f, // (11, 1071) 130 + 0x9177, // ( 9, 375) 131 + 0xc11f, // (12, 287) 132 + 0xa04f, // (10, 79) 133 + 0xc91f, // (12, 2335) 134 + 0xa24f, // (10, 591) 135 + 0xc51f, // (12, 1311) 136 + 0xb22f, // (11, 559) 137 + 0xcd1f, // (12, 3359) 138 + 0xb62f, // (11, 1583) 139 + 0xc31f, // (12, 799) 140 + 0xcb1f, // (12, 2847) 141 + 0xc71f, // (12, 1823) 142 + 0xcf1f, // (12, 3871) 143 + 0xc09f, // (12, 159) 144 + 0xc89f, // (12, 2207) 145 + 0xc49f, // (12, 1183) 146 + 0xcc9f, // (12, 3231) 147 + 0xc29f, // (12, 671) 148 + 0xca9f, // (12, 2719) 149 + 0xc69f, // (12, 1695) 150 + 0xce9f, // (12, 3743) 151 + 0xc19f, // (12, 415) 152 + 0xc99f, // (12, 2463) 153 + 0xc59f, // (12, 1439) 154 + 0xcd9f, // (12, 3487) 155 + 0xc39f, // (12, 927) 156 + 0xcb9f, // (12, 2975) 157 + 0xc79f, // (12, 1951) 158 + 0xcf9f, // (12, 3999) 159 + 0xc05f, // (12, 95) 160 + 0xc85f, // (12, 2143) 161 + 0xc45f, // (12, 1119) 162 + 0xcc5f, // (12, 3167) 163 + 0xc25f, // (12, 607) 164 + 0xca5f, // (12, 2655) 165 + 0xc65f, // (12, 1631) 166 + 0xce5f, // (12, 3679) 167 + 0xc15f, // (12, 351) 168 + 0xc95f, // (12, 2399) 169 + 0xc55f, // (12, 1375) 170 + 0xcd5f, // (12, 3423) 171 + 0xc35f, // (12, 863) 172 + 0xcb5f, // (12, 2911) 173 + 0xc75f, // (12, 1887) 174 + 0xcf5f, // (12, 3935) 175 + 0xc0df, // (12, 223) 176 + 0xc8df, // (12, 2271) 177 + 0xc4df, // (12, 1247) 178 + 0xccdf, // (12, 3295) 179 + 0xc2df, // (12, 735) 180 + 0xcadf, // (12, 2783) 181 + 0xc6df, // (12, 1759) 182 + 0xcedf, // (12, 3807) 183 + 0xc1df, // (12, 479) 184 + 0xc9df, // (12, 2527) 185 + 0xc5df, // (12, 1503) 186 + 0xcddf, // (12, 3551) 187 + 0xc3df, // (12, 991) 188 + 0xcbdf, // (12, 3039) 189 + 0xc7df, // (12, 2015) 190 + 0xcfdf, // (12, 4063) 191 + 0xc03f, // (12, 63) 192 + 0xc83f, // (12, 2111) 193 + 0xc43f, // (12, 1087) 194 + 0xcc3f, // (12, 3135) 195 + 0xc23f, // (12, 575) 196 + 0xca3f, // (12, 2623) 197 + 0xc63f, // (12, 1599) 198 + 0xce3f, // (12, 3647) 199 + 0xc13f, // (12, 319) 200 + 0xc93f, // (12, 2367) 201 + 0xc53f, // (12, 1343) 202 + 0xcd3f, // (12, 3391) 203 + 0xc33f, // (12, 831) 204 + 0xcb3f, // (12, 2879) 205 + 0xc73f, // (12, 1855) 206 + 0xcf3f, // (12, 3903) 207 + 0xc0bf, // (12, 191) 208 + 0xc8bf, // (12, 2239) 209 + 0xc4bf, // (12, 1215) 210 + 0xccbf, // (12, 3263) 211 + 0xc2bf, // (12, 703) 212 + 0xcabf, // (12, 2751) 213 + 0xc6bf, // (12, 1727) 214 + 0xcebf, // (12, 3775) 215 + 0xc1bf, // (12, 447) 216 + 0xc9bf, // (12, 2495) 217 + 0xc5bf, // (12, 1471) 218 + 0xcdbf, // (12, 3519) 219 + 0xc3bf, // (12, 959) 220 + 0xcbbf, // (12, 3007) 221 + 0xc7bf, // (12, 1983) 222 + 0xcfbf, // (12, 4031) 223 + 0xc07f, // (12, 127) 224 + 0xc87f, // (12, 2175) 225 + 0xc47f, // (12, 1151) 226 + 0xcc7f, // (12, 3199) 227 + 0xc27f, // (12, 639) 228 + 0xca7f, // (12, 2687) 229 + 0xc67f, // (12, 1663) 230 + 0xce7f, // (12, 3711) 231 + 0xc17f, // (12, 383) 232 + 0xc97f, // (12, 2431) 233 + 0xc57f, // (12, 1407) 234 + 0xcd7f, // (12, 3455) 235 + 0xc37f, // (12, 895) 236 + 0xcb7f, // (12, 2943) 237 + 0xc77f, // (12, 1919) 238 + 0xcf7f, // (12, 3967) 239 + 0xc0ff, // (12, 255) 240 + 0xc8ff, // (12, 2303) 241 + 0xc4ff, // (12, 1279) 242 + 0xccff, // (12, 3327) 243 + 0xc2ff, // (12, 767) 244 + 0xcaff, // (12, 2815) 245 + 0xc6ff, // (12, 1791) 246 + 0xceff, // (12, 3839) 247 + 0xc1ff, // (12, 511) 248 + 0xc9ff, // (12, 2559) 249 + 0xc5ff, // (12, 1535) 250 + 0xcdff, // (12, 3583) 251 + 0xc3ff, // (12, 1023) 252 + 0xcbff, // (12, 3071) 253 + 0xc7ff, // (12, 2047) 254 + 0xcfff // (12, 4095) 255 +} +}; + +/************************************************************************************************************/ + +/* Notice that there are only 65 symbols here, which is different from our + usual 8->12 coding scheme which handles 256 symbols. */ + +static const uint16_t length_limited_unary_encoding_table65[65] = { + // Length-limited "unary" code with 65 symbols. + // entropy: 2.0 + // avg_length: 2.0249023437500000000; max_length = 12; num_symbols = 65 + +//table, // (4 bits, 12 bits) symbol +//entry, // (length, codeword) [byte] + 0x1000, // ( 1, 0) 0 + 0x2001, // ( 2, 1) 1 + 0x3003, // ( 3, 3) 2 + 0x4007, // ( 4, 7) 3 + 0x500f, // ( 5, 15) 4 + 0x701f, // ( 7, 31) 5 + 0x805f, // ( 8, 95) 6 + 0x80df, // ( 8, 223) 7 + 0xa03f, // (10, 63) 8 + 0xa23f, // (10, 575) 9 + 0xb13f, // (11, 319) 10 + 0xc53f, // (12, 1343) 11 + 0xcd3f, // (12, 3391) 12 + 0xc33f, // (12, 831) 13 + 0xcb3f, // (12, 2879) 14 + 0xc73f, // (12, 1855) 15 + 0xcf3f, // (12, 3903) 16 + 0xc0bf, // (12, 191) 17 + 0xc8bf, // (12, 2239) 18 + 0xc4bf, // (12, 1215) 19 + 0xccbf, // (12, 3263) 20 + 0xc2bf, // (12, 703) 21 + 0xcabf, // (12, 2751) 22 + 0xc6bf, // (12, 1727) 23 + 0xcebf, // (12, 3775) 24 + 0xc1bf, // (12, 447) 25 + 0xc9bf, // (12, 2495) 26 + 0xc5bf, // (12, 1471) 27 + 0xcdbf, // (12, 3519) 28 + 0xc3bf, // (12, 959) 29 + 0xcbbf, // (12, 3007) 30 + 0xc7bf, // (12, 1983) 31 + 0xcfbf, // (12, 4031) 32 + 0xc07f, // (12, 127) 33 + 0xc87f, // (12, 2175) 34 + 0xc47f, // (12, 1151) 35 + 0xcc7f, // (12, 3199) 36 + 0xc27f, // (12, 639) 37 + 0xca7f, // (12, 2687) 38 + 0xc67f, // (12, 1663) 39 + 0xce7f, // (12, 3711) 40 + 0xc17f, // (12, 383) 41 + 0xc97f, // (12, 2431) 42 + 0xc57f, // (12, 1407) 43 + 0xcd7f, // (12, 3455) 44 + 0xc37f, // (12, 895) 45 + 0xcb7f, // (12, 2943) 46 + 0xc77f, // (12, 1919) 47 + 0xcf7f, // (12, 3967) 48 + 0xc0ff, // (12, 255) 49 + 0xc8ff, // (12, 2303) 50 + 0xc4ff, // (12, 1279) 51 + 0xccff, // (12, 3327) 52 + 0xc2ff, // (12, 767) 53 + 0xcaff, // (12, 2815) 54 + 0xc6ff, // (12, 1791) 55 + 0xceff, // (12, 3839) 56 + 0xc1ff, // (12, 511) 57 + 0xc9ff, // (12, 2559) 58 + 0xc5ff, // (12, 1535) 59 + 0xcdff, // (12, 3583) 60 + 0xc3ff, // (12, 1023) 61 + 0xcbff, // (12, 3071) 62 + 0xc7ff, // (12, 2047) 63 + 0xcfff // (12, 4095) 64 +}; + +/* +Note: these column permutations are part of the encoding scheme for sketches where C >= 3.375 * K. +In each row, we identify the (0-based) column indices of all surprising bits +outside of the high-entropy byte. + +These indices are "rotated right" via the formula +new = (old - (8+shift_by) + 64) mod 64 = (old + 56 - shift_by) mod 64. +resulting in canonicalized indices between 0 and 55 inclusive. + +These are then mapped through the forwards permutation specified below (and selected +by the phase of C / K). Finally, the remapped indices are encoding with a unary code +(with delta encoding for rows containing more than one surprising bit). +*/ + +// These permutations were created by +// the ocaml program "generatePermutationsForSLIDING.ml". + +static const uint8_t column_permutations_for_encoding[16][56] = { + // for phase = 1 / 32 + {0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 34, 14, 4}, + // for phase = 3 / 32 + {0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 35, 15, 4}, + // for phase = 5 / 32 + {0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 37, 16, 5}, + // for phase = 7 / 32 + {0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 39, 17, 5}, + // for phase = 9 / 32 + {0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 41, 18, 6}, + // for phase = 11 / 32 + {0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 43, 19, 6}, + // for phase = 13 / 32 + {1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 45, 20, 7, 0}, + // for phase = 15 / 32 + {1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 53, 54, 55, 47, 21, 7, 0}, + // for phase = 17 / 32 + {1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 51, 52, 53, 54, 55, 50, 22, 8, 0}, + // for phase = 19 / 32 + {0, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 53, 54, 55, 52, 23, 9, 1}, + // for phase = 21 / 32 + {0, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 25, 9, 1}, + // for phase = 23 / 32 + {0, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 26, 10, 1}, + // for phase = 25 / 32 + {0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 27, 11, 2}, + // for phase = 27 / 32 + {0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 29, 11, 2}, + // for phase = 29 / 32 + {0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 30, 12, 3}, + // for phase = 31 / 32 + {0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 32, 13, 3} +}; + +} /* namespace datasketches */ + +#endif diff --git a/3rd/datasketches/datasketches/cpc/counter_of_zeros.hpp b/3rd/datasketches/datasketches/cpc/counter_of_zeros.hpp new file mode 100644 index 000000000..eb2036ed8 --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/counter_of_zeros.hpp @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef COUNT_ZEROS_HPP_ +#define COUNT_ZEROS_HPP_ + +#include + +#include + +namespace datasketches { + +static const uint8_t byte_leading_zeros_table[256] = { + 8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +static const uint8_t byte_trailing_zeros_table[256] = { + 8, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0 +}; + +static const uint64_t FCLZ_MASK_56 = 0x00ffffffffffffff; +static const uint64_t FCLZ_MASK_48 = 0x0000ffffffffffff; +static const uint64_t FCLZ_MASK_40 = 0x000000ffffffffff; +static const uint64_t FCLZ_MASK_32 = 0x00000000ffffffff; +static const uint64_t FCLZ_MASK_24 = 0x0000000000ffffff; +static const uint64_t FCLZ_MASK_16 = 0x000000000000ffff; +static const uint64_t FCLZ_MASK_08 = 0x00000000000000ff; + +static inline uint8_t count_leading_zeros_in_u64(uint64_t input) { + if (input > FCLZ_MASK_56) + return byte_leading_zeros_table[(input >> 56) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_48) + return 8 + byte_leading_zeros_table[(input >> 48) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_40) + return 16 + byte_leading_zeros_table[(input >> 40) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_32) + return 24 + byte_leading_zeros_table[(input >> 32) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_24) + return 32 + byte_leading_zeros_table[(input >> 24) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_16) + return 40 + byte_leading_zeros_table[(input >> 16) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_08) + return 48 + byte_leading_zeros_table[(input >> 8) & FCLZ_MASK_08]; + if (true) + return 56 + byte_leading_zeros_table[(input ) & FCLZ_MASK_08]; +} + +static inline uint8_t count_trailing_zeros_in_u64(uint64_t input) { + for (int i = 0; i < 8; i++) { + const int byte = input & 0xff; + if (byte != 0) return (i << 3) + byte_trailing_zeros_table[byte]; + input >>= 8; + } + return 64; +} + +} /* namespace datasketches */ + +#endif diff --git a/3rd/datasketches/datasketches/cpc/cpc_common.hpp b/3rd/datasketches/datasketches/cpc/cpc_common.hpp new file mode 100644 index 000000000..c6254df82 --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/cpc_common.hpp @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef CPC_COMMON_HPP_ +#define CPC_COMMON_HPP_ + +#include + +#include "MurmurHash3.h" + +namespace datasketches { + +static const uint8_t CPC_MIN_LG_K = 4; +static const uint8_t CPC_MAX_LG_K = 26; +static const uint8_t CPC_DEFAULT_LG_K = 11; +static const uint64_t DEFAULT_SEED = 9001; + +template using AllocU8 = typename std::allocator_traits::template rebind_alloc; +template using AllocU16 = typename std::allocator_traits::template rebind_alloc; +template using AllocU32 = typename std::allocator_traits::template rebind_alloc; +template using AllocU64 = typename std::allocator_traits::template rebind_alloc; + +template using vector_u8 = std::vector>; +template using vector_u32 = std::vector>; +template using vector_u64 = std::vector>; + +// forward declaration +template class u32_table; + +template +struct compressed_state { + vector_u32 table_data; + uint32_t table_data_words; + uint32_t table_num_entries; // can be different from the number of entries in the sketch in hybrid mode + vector_u32 window_data; + uint32_t window_data_words; +}; + +template +struct uncompressed_state { + u32_table table; + vector_u8 window; +}; + +} /* namespace datasketches */ + +#endif diff --git a/3rd/datasketches/datasketches/cpc/cpc_compressor.hpp b/3rd/datasketches/datasketches/cpc/cpc_compressor.hpp new file mode 100644 index 000000000..55fa3b879 --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/cpc_compressor.hpp @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// author Kevin Lang, Oath Research + +#ifndef CPC_COMPRESSOR_HPP_ +#define CPC_COMPRESSOR_HPP_ + +#include "cpc_common.hpp" + +namespace datasketches { + +/* + * This is a very efficient compressor specialized for use by the CPC Sketch. + * There are two very different compression schemes here: one for the sliding window + * and another for the table of so-called surprising values. + * These two compression schemes are designed for specific probability distributions of entries + * in these data structures and make some compromises for performance. As a result + * the compression is slightly less effective than theoretically achievable but is very fast. + */ + +// forward declarations +template class cpc_sketch_alloc; +template class cpc_compressor; + +// the compressor is not instantiated directly +// the sketch implementation uses this global function to statically allocate and construct on the first use +template +inline cpc_compressor& get_compressor(); + +template +class cpc_compressor { +public: + void compress(const cpc_sketch_alloc& source, compressed_state& target) const; + void uncompress(const compressed_state& source, uncompressed_state& target, uint8_t lg_k, uint64_t num_coupons) const; + + // methods below are public for testing + + // This returns the number of compressed words that were actually used. It is the caller's + // responsibility to ensure that the compressed_words array is long enough to prevent over-run. + size_t low_level_compress_bytes( + const uint8_t* byte_array, // input + size_t num_bytes_to_encode, + const uint16_t* encoding_table, + uint32_t* compressed_words // output + ) const; + + void low_level_uncompress_bytes( + uint8_t* byte_array, // output + size_t num_bytes_to_decode, + const uint16_t* decoding_table, + const uint32_t* compressed_words, + size_t num_compressed_words // input + ) const; + + // Here "pairs" refers to row-column pairs that specify + // the positions of surprising values in the bit matrix. + + // returns the number of compressedWords actually used + size_t low_level_compress_pairs( + const uint32_t* pair_array, // input + size_t num_pairs_to_encode, + size_t num_base_bits, + uint32_t* compressed_words // output + ) const; + + void low_level_uncompress_pairs( + uint32_t* pair_array, // output + size_t num_pairs_to_decode, + size_t num_base_bits, + const uint32_t* compressed_words, // input + size_t num_compressed_words // input + ) const; + +private: + // These decoding tables are created at library startup time by inverting the encoding tables + uint16_t* decoding_tables_for_high_entropy_byte[22] = { + // sixteen tables for the steady state (chosen based on the "phase" of C/K) + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + // six more tables for the gradual transition between warmup mode and the steady state. + NULL, NULL, NULL, NULL, NULL, NULL + }; + uint16_t* length_limited_unary_decoding_table65; + uint8_t* column_permutations_for_decoding[16] = { + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL + }; + + cpc_compressor(); + template friend cpc_compressor& get_compressor(); + ~cpc_compressor(); + + void make_decoding_tables(); // call this at startup + void free_decoding_tables(); // call this at the end + + void compress_sparse_flavor(const cpc_sketch_alloc& source, compressed_state& target) const; + void compress_hybrid_flavor(const cpc_sketch_alloc& source, compressed_state& target) const; + void compress_pinned_flavor(const cpc_sketch_alloc& source, compressed_state& target) const; + void compress_sliding_flavor(const cpc_sketch_alloc& source, compressed_state& target) const; + + void uncompress_sparse_flavor(const compressed_state& source, uncompressed_state& target, uint8_t lg_k) const; + void uncompress_hybrid_flavor(const compressed_state& source, uncompressed_state& target, uint8_t lg_k) const; + void uncompress_pinned_flavor(const compressed_state& source, uncompressed_state& target, uint8_t lg_k, uint32_t num_coupons) const; + void uncompress_sliding_flavor(const compressed_state& source, uncompressed_state& target, uint8_t lg_k, uint32_t num_coupons) const; + + uint8_t* make_inverse_permutation(const uint8_t* permu, int length); + uint16_t* make_decoding_table(const uint16_t* encoding_table, int num_byte_values); + void validate_decoding_table(const uint16_t* decoding_table, const uint16_t* encoding_table) const; + + void compress_surprising_values(const vector_u32& pairs, uint8_t lg_k, compressed_state& result) const; + void compress_sliding_window(const uint8_t* window, uint8_t lg_k, uint32_t num_coupons, compressed_state& target) const; + + vector_u32 uncompress_surprising_values(const uint32_t* data, size_t data_words, size_t num_pairs, uint8_t lg_k) const; + void uncompress_sliding_window(const uint32_t* data, size_t data_words, vector_u8& window, uint8_t lg_k, uint32_t num_coupons) const; + + static size_t safe_length_for_compressed_pair_buf(uint64_t k, size_t num_pairs, size_t num_base_bits); + static size_t safe_length_for_compressed_window_buf(uint64_t k); + static uint8_t determine_pseudo_phase(uint8_t lg_k, uint64_t c); + + static inline vector_u32 tricky_get_pairs_from_window(const uint8_t* window, uint32_t k, uint32_t num_pairs_to_get, uint32_t empty_space); + static inline uint64_t golomb_choose_number_of_base_bits(uint64_t k, uint64_t count); +}; + +} /* namespace datasketches */ + +#include "cpc_compressor_impl.hpp" + +#endif diff --git a/3rd/datasketches/datasketches/cpc/cpc_compressor_impl.hpp b/3rd/datasketches/datasketches/cpc/cpc_compressor_impl.hpp new file mode 100644 index 000000000..d5d928d17 --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/cpc_compressor_impl.hpp @@ -0,0 +1,743 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// author Kevin Lang, Oath Research + +#ifndef CPC_COMPRESSOR_IMPL_HPP_ +#define CPC_COMPRESSOR_IMPL_HPP_ + +#include + +#include "compression_data.hpp" +#include "counter_of_zeros.hpp" +#include "cpc_util.hpp" + +#include "cpc_common.hpp" + +namespace datasketches { + +// construct on first use +template +cpc_compressor& get_compressor() { + static cpc_compressor* instance = new cpc_compressor(); // use new for global initialization + return *instance; +} + +template +cpc_compressor::cpc_compressor() { + make_decoding_tables(); +} + +template +cpc_compressor::~cpc_compressor() { + free_decoding_tables(); +} + +template +uint8_t* cpc_compressor::make_inverse_permutation(const uint8_t* permu, int length) { + uint8_t* inverse = new uint8_t[length]; // use new for global initialization + for (int i = 0; i < length; i++) { + inverse[permu[i]] = i; + } + for (int i = 0; i < length; i++) { + if (permu[inverse[i]] != i) throw std::logic_error("inverse permutation error"); + } + return inverse; +} + +/* Given an encoding table that maps unsigned bytes to codewords + of length at most 12, this builds a size-4096 decoding table */ +// The second argument is typically 256, but can be other values such as 65. +template +uint16_t* cpc_compressor::make_decoding_table(const uint16_t* encoding_table, int num_byte_values) { + uint16_t* decoding_table = new uint16_t[4096]; // use new for global initialization + for (int byte_value = 0; byte_value < num_byte_values; byte_value++) { + const int encoding_entry = encoding_table[byte_value]; + const int code_value = encoding_entry & 0xfff; + const int code_length = encoding_entry >> 12; + const int decoding_entry = (code_length << 8) | byte_value; + const int garbage_length = 12 - code_length; + const int num_copies = 1 << garbage_length; + for (int garbage_bits = 0; garbage_bits < num_copies; garbage_bits++) { + const int extended_code_value = code_value | (garbage_bits << code_length); + decoding_table[extended_code_value & 0xfff] = decoding_entry; + } + } + return decoding_table; +} + +template +void cpc_compressor::validate_decoding_table(const uint16_t* decoding_table, const uint16_t* encoding_table) const { + for (int decode_this = 0; decode_this < 4096; decode_this++) { + const int tmp_d = decoding_table[decode_this]; + const int decoded_byte = tmp_d & 0xff; + const int decoded_length = tmp_d >> 8; + + const int tmp_e = encoding_table[decoded_byte]; + const int encoded_bit_pattern = tmp_e & 0xfff; + const int encoded_length = tmp_e >> 12; + + if (decoded_length != encoded_length) throw std::logic_error("decoded length error"); + if (encoded_bit_pattern != (decode_this & ((1 << decoded_length) - 1))) throw std::logic_error("bit pattern error"); + } +} + +template +void cpc_compressor::make_decoding_tables() { + length_limited_unary_decoding_table65 = make_decoding_table(length_limited_unary_encoding_table65, 65); + validate_decoding_table( + length_limited_unary_decoding_table65, + length_limited_unary_encoding_table65 + ); + + for (int i = 0; i < (16 + 6); i++) { + decoding_tables_for_high_entropy_byte[i] = make_decoding_table(encoding_tables_for_high_entropy_byte[i], 256); + validate_decoding_table( + decoding_tables_for_high_entropy_byte[i], + encoding_tables_for_high_entropy_byte[i] + ); + } + + for (int i = 0; i < 16; i++) { + column_permutations_for_decoding[i] = make_inverse_permutation(column_permutations_for_encoding[i], 56); + } +} + +template +void cpc_compressor::free_decoding_tables() { + delete[] length_limited_unary_decoding_table65; + for (int i = 0; i < (16 + 6); i++) { + delete[] decoding_tables_for_high_entropy_byte[i]; + } + for (int i = 0; i < 16; i++) { + delete[] column_permutations_for_decoding[i]; + } +} + +template +void cpc_compressor::compress(const cpc_sketch_alloc& source, compressed_state& result) const { + switch (source.determine_flavor()) { + case cpc_sketch_alloc::flavor::EMPTY: + break; + case cpc_sketch_alloc::flavor::SPARSE: + compress_sparse_flavor(source, result); + if (result.window_data.size() > 0) throw std::logic_error("window is not expected"); + if (result.table_data.size() == 0) throw std::logic_error("table is expected"); + break; + case cpc_sketch_alloc::flavor::HYBRID: + compress_hybrid_flavor(source, result); + if (result.window_data.size() > 0) throw std::logic_error("window is not expected"); + if (result.table_data.size() == 0) throw std::logic_error("table is expected"); + break; + case cpc_sketch_alloc::flavor::PINNED: + compress_pinned_flavor(source, result); + if (result.window_data.size() == 0) throw std::logic_error("window is not expected"); + break; + case cpc_sketch_alloc::flavor::SLIDING: + compress_sliding_flavor(source, result); + if (result.window_data.size() == 0) throw std::logic_error("window is expected"); + break; + default: throw std::logic_error("Unknown sketch flavor"); + } +} + +template +void cpc_compressor::uncompress(const compressed_state& source, uncompressed_state& target, uint8_t lg_k, uint64_t num_coupons) const { + switch (cpc_sketch_alloc::determine_flavor(lg_k, num_coupons)) { + case cpc_sketch_alloc::flavor::EMPTY: + target.table = u32_table(2, 6 + lg_k); + break; + case cpc_sketch_alloc::flavor::SPARSE: + uncompress_sparse_flavor(source, target, lg_k); + break; + case cpc_sketch_alloc::flavor::HYBRID: + uncompress_hybrid_flavor(source, target, lg_k); + break; + case cpc_sketch_alloc::flavor::PINNED: + if (source.window_data.size() == 0) throw std::logic_error("window is expected"); + uncompress_pinned_flavor(source, target, lg_k, num_coupons); + break; + case cpc_sketch_alloc::flavor::SLIDING: + uncompress_sliding_flavor(source, target, lg_k, num_coupons); + break; + default: std::logic_error("Unknown sketch flavor"); + } +} + +template +void cpc_compressor::compress_sparse_flavor(const cpc_sketch_alloc& source, compressed_state& result) const { + if (source.sliding_window.size() > 0) throw std::logic_error("unexpected sliding window"); + vector_u32 pairs = source.surprising_value_table.unwrapping_get_items(); + u32_table::introspective_insertion_sort(pairs.data(), 0, pairs.size()); + compress_surprising_values(pairs, source.get_lg_k(), result); +} + +template +void cpc_compressor::uncompress_sparse_flavor(const compressed_state& source, uncompressed_state& target, uint8_t lg_k) const { + if (source.window_data.size() > 0) throw std::logic_error("unexpected sliding window"); + if (source.table_data.size() == 0) throw std::logic_error("table is expected"); + vector_u32 pairs = uncompress_surprising_values(source.table_data.data(), source.table_data_words, source.table_num_entries, lg_k); + target.table = u32_table::make_from_pairs(pairs.data(), source.table_num_entries, lg_k); +} + +// This is complicated because it effectively builds a Sparse version +// of a Pinned sketch before compressing it. Hence the name Hybrid. +template +void cpc_compressor::compress_hybrid_flavor(const cpc_sketch_alloc& source, compressed_state& result) const { + if (source.sliding_window.size() == 0) throw std::logic_error("no sliding window"); + if (source.window_offset != 0) throw std::logic_error("window_offset != 0"); + const size_t k = 1 << source.get_lg_k(); + vector_u32 pairs_from_table = source.surprising_value_table.unwrapping_get_items(); + if (pairs_from_table.size() > 0) u32_table::introspective_insertion_sort(pairs_from_table.data(), 0, pairs_from_table.size()); + const size_t num_pairs_from_window = source.get_num_coupons() - pairs_from_table.size(); // because the window offset is zero + + vector_u32 all_pairs = tricky_get_pairs_from_window(source.sliding_window.data(), k, num_pairs_from_window, pairs_from_table.size()); + + u32_table::merge( + pairs_from_table.data(), 0, pairs_from_table.size(), + all_pairs.data(), pairs_from_table.size(), num_pairs_from_window, + all_pairs.data(), 0 + ); // note the overlapping subarray trick + + compress_surprising_values(all_pairs, source.get_lg_k(), result); +} + +template +void cpc_compressor::uncompress_hybrid_flavor(const compressed_state& source, uncompressed_state& target, uint8_t lg_k) const { + if (source.window_data.size() > 0) throw std::logic_error("window is not expected"); + if (source.table_data.size() == 0) throw std::logic_error("table is expected"); + vector_u32 pairs = uncompress_surprising_values(source.table_data.data(), source.table_data_words, source.table_num_entries, lg_k); + + // In the hybrid flavor, some of these pairs actually + // belong in the window, so we will separate them out, + // moving the "true" pairs to the bottom of the array. + const size_t k = 1 << lg_k; + target.window.resize(k, 0); // important: zero the memory + size_t next_true_pair = 0; + for (size_t i = 0; i < source.table_num_entries; i++) { + const uint32_t row_col = pairs[i]; + if (row_col == UINT32_MAX) throw std::logic_error("empty marker is not expected"); + const uint8_t col = row_col & 63; + if (col < 8) { + const size_t row = row_col >> 6; + target.window[row] |= 1 << col; // set the window bit + } else { + pairs[next_true_pair++] = row_col; // move true pair down + } + } + target.table = u32_table::make_from_pairs(pairs.data(), next_true_pair, lg_k); +} + +template +void cpc_compressor::compress_pinned_flavor(const cpc_sketch_alloc& source, compressed_state& result) const { + compress_sliding_window(source.sliding_window.data(), source.get_lg_k(), source.get_num_coupons(), result); + vector_u32 pairs = source.surprising_value_table.unwrapping_get_items(); + if (pairs.size() > 0) { + // Here we subtract 8 from the column indices. Because they are stored in the low 6 bits + // of each row_col pair, and because no column index is less than 8 for a "Pinned" sketch, + // we can simply subtract 8 from the pairs themselves. + + // shift the columns over by 8 positions before compressing (because of the window) + for (size_t i = 0; i < pairs.size(); i++) { + if ((pairs[i] & 63) < 8) throw std::logic_error("(pairs[i] & 63) < 8"); + pairs[i] -= 8; + } + + if (pairs.size() > 0) u32_table::introspective_insertion_sort(pairs.data(), 0, pairs.size()); + compress_surprising_values(pairs, source.get_lg_k(), result); + } +} + +template +void cpc_compressor::uncompress_pinned_flavor(const compressed_state& source, uncompressed_state& target, uint8_t lg_k, uint32_t num_coupons) const { + if (source.window_data.size() == 0) throw std::logic_error("window is expected"); + uncompress_sliding_window(source.window_data.data(), source.window_data_words, target.window, lg_k, num_coupons); + const size_t num_pairs = source.table_num_entries; + if (num_pairs == 0) { + target.table = u32_table(2, 6 + lg_k); + } else { + if (source.table_data.size() == 0) throw std::logic_error("table is expected"); + vector_u32 pairs = uncompress_surprising_values(source.table_data.data(), source.table_data_words, num_pairs, lg_k); + // undo the compressor's 8-column shift + for (size_t i = 0; i < num_pairs; i++) { + if ((pairs[i] & 63) >= 56) throw std::logic_error("(pairs[i] & 63) >= 56"); + pairs[i] += 8; + } + target.table = u32_table::make_from_pairs(pairs.data(), num_pairs, lg_k); + } +} + +template +void cpc_compressor::compress_sliding_flavor(const cpc_sketch_alloc& source, compressed_state& result) const { + compress_sliding_window(source.sliding_window.data(), source.get_lg_k(), source.get_num_coupons(), result); + vector_u32 pairs = source.surprising_value_table.unwrapping_get_items(); + if (pairs.size() > 0) { + // Here we apply a complicated transformation to the column indices, which + // changes the implied ordering of the pairs, so we must do it before sorting. + + const uint8_t pseudo_phase = determine_pseudo_phase(source.get_lg_k(), source.get_num_coupons()); + const uint8_t* permutation = column_permutations_for_encoding[pseudo_phase]; + + const uint8_t offset = source.window_offset; + if (offset > 56) throw std::out_of_range("offset out of range"); + + for (size_t i = 0; i < pairs.size(); i++) { + const uint32_t row_col = pairs[i]; + const size_t row = row_col >> 6; + uint8_t col = row_col & 63; + // first rotate the columns into a canonical configuration: new = ((old - (offset+8)) + 64) mod 64 + col = (col + 56 - offset) & 63; + if (col >= 56) throw std::out_of_range("col out of range"); + // then apply the permutation + col = permutation[col]; + pairs[i] = (row << 6) | col; + } + + if (pairs.size() > 0) u32_table::introspective_insertion_sort(pairs.data(), 0, pairs.size()); + compress_surprising_values(pairs, source.get_lg_k(), result); + } +} + +template +void cpc_compressor::uncompress_sliding_flavor(const compressed_state& source, uncompressed_state& target, uint8_t lg_k, uint32_t num_coupons) const { + if (source.window_data.size() == 0) throw std::logic_error("window is expected"); + uncompress_sliding_window(source.window_data.data(), source.window_data_words, target.window, lg_k, num_coupons); + const size_t num_pairs = source.table_num_entries; + if (num_pairs == 0) { + target.table = u32_table(2, 6 + lg_k); + } else { + if (source.table_data.size() == 0) throw std::logic_error("table is expected"); + vector_u32 pairs = uncompress_surprising_values(source.table_data.data(), source.table_data_words, num_pairs, lg_k); + + const uint8_t pseudo_phase = determine_pseudo_phase(lg_k, num_coupons); + if (pseudo_phase >= 16) throw std::logic_error("pseudo phase >= 16"); + const uint8_t* permutation = column_permutations_for_decoding[pseudo_phase]; + + uint8_t offset = cpc_sketch_alloc::determine_correct_offset(lg_k, num_coupons); + if (offset > 56) throw std::out_of_range("offset out of range"); + + for (size_t i = 0; i < num_pairs; i++) { + const uint32_t row_col = pairs[i]; + const size_t row = row_col >> 6; + uint8_t col = row_col & 63; + // first undo the permutation + col = permutation[col]; + // then undo the rotation: old = (new + (offset+8)) mod 64 + col = (col + (offset + 8)) & 63; + pairs[i] = (row << 6) | col; + } + + target.table = u32_table::make_from_pairs(pairs.data(), num_pairs, lg_k); + } +} + +template +void cpc_compressor::compress_surprising_values(const vector_u32& pairs, uint8_t lg_k, compressed_state& result) const { + const size_t k = 1 << lg_k; + const uint64_t num_base_bits = golomb_choose_number_of_base_bits(k + pairs.size(), pairs.size()); + const uint64_t table_len = safe_length_for_compressed_pair_buf(k, pairs.size(), num_base_bits); + result.table_data.resize(table_len); + + size_t csv_length = low_level_compress_pairs(pairs.data(), pairs.size(), num_base_bits, result.table_data.data()); + + // At this point we could free the unused portion of the compression output buffer, + // but it is not necessary if it is temporary + // Note: realloc caused strange timing spikes for lgK = 11 and 12. + + result.table_data_words = csv_length; + result.table_num_entries = pairs.size(); +} + +template +vector_u32 cpc_compressor::uncompress_surprising_values(const uint32_t* data, size_t data_words, size_t num_pairs, uint8_t lg_k) const { + const size_t k = 1 << lg_k; + vector_u32 pairs(num_pairs); + const uint8_t num_base_bits = golomb_choose_number_of_base_bits(k + num_pairs, num_pairs); + low_level_uncompress_pairs(pairs.data(), num_pairs, num_base_bits, data, data_words); + return pairs; +} + +template +void cpc_compressor::compress_sliding_window(const uint8_t* window, uint8_t lg_k, uint32_t num_coupons, compressed_state& target) const { + const size_t k = 1 << lg_k; + const size_t window_buf_len = safe_length_for_compressed_window_buf(k); + target.window_data.resize(window_buf_len); + const uint8_t pseudo_phase = determine_pseudo_phase(lg_k, num_coupons); + size_t data_words = low_level_compress_bytes(window, k, encoding_tables_for_high_entropy_byte[pseudo_phase], target.window_data.data()); + + // At this point we could free the unused portion of the compression output buffer, + // but it is not necessary if it is temporary + // Note: realloc caused strange timing spikes for lgK = 11 and 12. + + target.window_data_words = data_words; +} + +template +void cpc_compressor::uncompress_sliding_window(const uint32_t* data, size_t data_words, vector_u8& window, uint8_t lg_k, uint32_t num_coupons) const { + const size_t k = 1 << lg_k; + window.resize(k); // zeroing not needed here (unlike the Hybrid Flavor) + const uint8_t pseudo_phase = determine_pseudo_phase(lg_k, num_coupons); + low_level_uncompress_bytes(window.data(), k, decoding_tables_for_high_entropy_byte[pseudo_phase], data, data_words); +} + +template +size_t cpc_compressor::safe_length_for_compressed_pair_buf(uint64_t k, size_t num_pairs, size_t num_base_bits) { + // Long ybits = k + numPairs; // simpler and safer UB + // The following tighter UB on ybits is based on page 198 + // of the textbook "Managing Gigabytes" by Witten, Moffat, and Bell. + // Notice that if numBaseBits == 0 it coincides with (k + numPairs). + const size_t ybits = num_pairs * (1 + num_base_bits) + (k >> num_base_bits); + const size_t xbits = 12 * num_pairs; + const size_t padding = num_base_bits > 10 ? 0 : 10 - num_base_bits; + return divide_longs_rounding_up(xbits + ybits + padding, 32); +} + +// Explanation of padding: we write +// 1) xdelta (huffman, provides at least 1 bit, requires 12-bit lookahead) +// 2) ydeltaGolombHi (unary, provides at least 1 bit, requires 8-bit lookahead) +// 3) ydeltaGolombLo (straight B bits). +// So the 12-bit lookahead is the tight constraint, but there are at least (2 + B) bits emitted, +// so we would be safe with max (0, 10 - B) bits of padding at the end of the bitstream. +template +size_t cpc_compressor::safe_length_for_compressed_window_buf(uint64_t k) { // measured in 32-bit words + const size_t bits = 12 * k + 11; // 11 bits of padding, due to 12-bit lookahead, with 1 bit certainly present. + return divide_longs_rounding_up(bits, 32); +} + +template +uint8_t cpc_compressor::determine_pseudo_phase(uint8_t lg_k, uint64_t c) { + const size_t k = 1 << lg_k; + // This mid-range logic produces pseudo-phases. They are used to select encoding tables. + // The thresholds were chosen by hand after looking at plots of measured compression. + if (1000 * c < 2375 * k) { + if ( 4 * c < 3 * k) return 16 + 0; // mid-range table + else if ( 10 * c < 11 * k) return 16 + 1; // mid-range table + else if ( 100 * c < 132 * k) return 16 + 2; // mid-range table + else if ( 3 * c < 5 * k) return 16 + 3; // mid-range table + else if (1000 * c < 1965 * k) return 16 + 4; // mid-range table + else if (1000 * c < 2275 * k) return 16 + 5; // mid-range table + else return 6; // steady-state table employed before its actual phase + } else { // This steady-state logic produces true phases. They are used to select + // encoding tables, and also column permutations for the "Sliding" flavor. + if (lg_k < 4) throw std::logic_error("lgK < 4"); + const size_t tmp = c >> (lg_k - 4); + const uint8_t phase = tmp & 15; + if (phase < 0 or phase >= 16) throw std::out_of_range("wrong phase"); + return phase; + } +} + +static inline void maybe_flush_bitbuf(uint64_t& bitbuf, uint8_t& bufbits, uint32_t* wordarr, size_t& wordindex) { + if (bufbits >= 32) { + wordarr[wordindex++] = bitbuf & 0xffffffff; + bitbuf = bitbuf >> 32; + bufbits -= 32; + } +} + +static inline void maybe_fill_bitbuf(uint64_t& bitbuf, uint8_t& bufbits, const uint32_t* wordarr, size_t& wordindex, uint8_t minbits) { + if (bufbits < minbits) { + bitbuf |= static_cast(wordarr[wordindex++]) << bufbits; + bufbits += 32; + } +} + +// This returns the number of compressed words that were actually used. +// It is the caller's responsibility to ensure that the compressed_words array is long enough. +template +size_t cpc_compressor::low_level_compress_bytes( + const uint8_t* byte_array, // input + size_t num_bytes_to_encode, + const uint16_t* encoding_table, + uint32_t* compressed_words // output +) const { + uint64_t bitbuf = 0; // bits are packed into this first, then are flushed to compressed_words + uint8_t bufbits = 0; // number of bits currently in bitbuf; must be between 0 and 31 + size_t next_word_index = 0; + + for (size_t byte_index = 0; byte_index < num_bytes_to_encode; byte_index++) { + const uint64_t code_info = encoding_table[byte_array[byte_index]]; + const uint64_t code_val = code_info & 0xfff; + const int code_len = code_info >> 12; + bitbuf |= (code_val << bufbits); + bufbits += code_len; + maybe_flush_bitbuf(bitbuf, bufbits, compressed_words, next_word_index); + } + + // Pad the bitstream with 11 zero-bits so that the decompressor's 12-bit peek can't overrun its input. + bufbits += 11; + maybe_flush_bitbuf(bitbuf, bufbits, compressed_words, next_word_index); + + if (bufbits > 0) { // We are done encoding now, so we flush the bit buffer. + if (bufbits >= 32) throw std::logic_error("bufbits >= 32"); + compressed_words[next_word_index++] = bitbuf & 0xffffffff; + bitbuf = 0; bufbits = 0; // not really necessary + } + return next_word_index; +} + +template +void cpc_compressor::low_level_uncompress_bytes( + uint8_t* byte_array, // output + size_t num_bytes_to_decode, + const uint16_t* decoding_table, + const uint32_t* compressed_words, // input + size_t num_compressed_words +) const { + size_t word_index = 0; + uint64_t bitbuf = 0; + uint8_t bufbits = 0; + + if (byte_array == nullptr) throw std::logic_error("byte_array == NULL"); + if (decoding_table == nullptr) throw std::logic_error("decoding_table == NULL"); + if (compressed_words == nullptr) throw std::logic_error("compressed_words == NULL"); + + for (size_t byte_index = 0; byte_index < num_bytes_to_decode; byte_index++) { + maybe_fill_bitbuf(bitbuf, bufbits, compressed_words, word_index, 12); // ensure 12 bits in bit buffer + + const size_t peek12 = bitbuf & 0xfff; // These 12 bits will include an entire Huffman codeword. + const uint16_t lookup = decoding_table[peek12]; + const uint8_t code_word_length = lookup >> 8; + const uint8_t decoded_byte = lookup & 0xff; + byte_array[byte_index] = decoded_byte; + bitbuf >>= code_word_length; + bufbits -= code_word_length; + } + // Buffer over-run should be impossible unless there is a bug. + // However, we might as well check here. + if (word_index > num_compressed_words) throw std::logic_error("word_index > num_compressed_words"); +} + +static inline uint64_t read_unary( + const uint32_t* compressed_words, + size_t& next_word_index, + uint64_t& bitbuf, + uint8_t& bufbits +); + +static inline void write_unary( + uint32_t* compressed_words, + size_t& next_word_index_ptr, + uint64_t& bit_buf_ptr, + uint8_t& buf_bits_ptr, + uint64_t value +); + +// Here "pairs" refers to row/column pairs that specify +// the positions of surprising values in the bit matrix. + +// returns the number of compressed_words actually used +template +size_t cpc_compressor::low_level_compress_pairs( + const uint32_t* pair_array, // input + size_t num_pairs_to_encode, + size_t num_base_bits, + uint32_t* compressed_words // output +) const { + uint64_t bitbuf = 0; + uint8_t bufbits = 0; + size_t next_word_index = 0; + const uint64_t golomb_lo_mask = (1 << num_base_bits) - 1; + uint64_t predicted_row_index = 0; + uint16_t predicted_col_index = 0; + + for (size_t pair_index = 0; pair_index < num_pairs_to_encode; pair_index++) { + const uint32_t row_col = pair_array[pair_index]; + const uint64_t row_index = row_col >> 6; + const uint16_t col_index = row_col & 63; + + if (row_index != predicted_row_index) predicted_col_index = 0; + + if (row_index < predicted_row_index) throw std::logic_error("row_index < predicted_row_index"); + if (col_index < predicted_col_index) throw std::logic_error("col_index < predicted_col_index"); + + const uint64_t y_delta = row_index - predicted_row_index; + const uint16_t x_delta = col_index - predicted_col_index; + + predicted_row_index = row_index; + predicted_col_index = col_index + 1; + + const uint64_t code_info = length_limited_unary_encoding_table65[x_delta]; + const uint64_t code_val = code_info & 0xfff; + const uint8_t code_len = code_info >> 12; + bitbuf |= code_val << bufbits; + bufbits += code_len; + maybe_flush_bitbuf(bitbuf, bufbits, compressed_words, next_word_index); + + const uint64_t golomb_lo = y_delta & golomb_lo_mask; + const uint64_t golomb_hi = y_delta >> num_base_bits; + + write_unary(compressed_words, next_word_index, bitbuf, bufbits, golomb_hi); + + bitbuf |= golomb_lo << bufbits; + bufbits += num_base_bits; + maybe_flush_bitbuf(bitbuf, bufbits, compressed_words, next_word_index); + } + + // Pad the bitstream so that the decompressor's 12-bit peek can't overrun its input. + const uint8_t padding = (num_base_bits > 10) ? 0 : 10 - num_base_bits; + bufbits += padding; + maybe_flush_bitbuf(bitbuf, bufbits, compressed_words, next_word_index); + + if (bufbits > 0) { // We are done encoding now, so we flush the bit buffer + if (bufbits >= 32) throw std::logic_error("bufbits >= 32"); + compressed_words[next_word_index++] = bitbuf & 0xffffffff; + bitbuf = 0; bufbits = 0; // not really necessary + } + + return next_word_index; +} + +template +void cpc_compressor::low_level_uncompress_pairs( + uint32_t* pair_array, // output + size_t num_pairs_to_decode, + size_t num_base_bits, + const uint32_t* compressed_words, // input + size_t num_compressed_words +) const { + size_t word_index = 0; + uint64_t bitbuf = 0; + uint8_t bufbits = 0; + const uint64_t golomb_lo_mask = (1 << num_base_bits) - 1; + uint64_t predicted_row_index = 0; + uint16_t predicted_col_index = 0; + + // for each pair we need to read: + // x_delta (12-bit length-limited unary) + // y_delta_hi (unary) + // y_delta_lo (basebits) + + for (size_t pair_index = 0; pair_index < num_pairs_to_decode; pair_index++) { + maybe_fill_bitbuf(bitbuf, bufbits, compressed_words, word_index, 12); // ensure 12 bits in bit buffer + const size_t peek12 = bitbuf & 0xfff; + const uint16_t lookup = length_limited_unary_decoding_table65[peek12]; + const int code_word_length = lookup >> 8; + const int16_t x_delta = lookup & 0xff; + bitbuf >>= code_word_length; + bufbits -= code_word_length; + + const uint64_t golomb_hi = read_unary(compressed_words, word_index, bitbuf, bufbits); + + maybe_fill_bitbuf(bitbuf, bufbits, compressed_words, word_index, num_base_bits); // ensure num_base_bits in bit buffer + const uint64_t golomb_lo = bitbuf & golomb_lo_mask; + bitbuf >>= num_base_bits; + bufbits -= num_base_bits; + const int64_t y_delta = (golomb_hi << num_base_bits) | golomb_lo; + + // Now that we have x_delta and y_delta, we can compute the pair's row and column + if (y_delta > 0) predicted_col_index = 0; + const uint64_t row_index = predicted_row_index + y_delta; + const uint16_t col_index = predicted_col_index + x_delta; + const uint32_t row_col = (row_index << 6) | col_index; + pair_array[pair_index] = row_col; + predicted_row_index = row_index; + predicted_col_index = col_index + 1; + } + if (word_index > num_compressed_words) throw std::logic_error("word_index > num_compressed_words"); // check for buffer over-run +} + +uint64_t read_unary( + const uint32_t* compressed_words, + size_t& next_word_index, + uint64_t& bitbuf, + uint8_t& bufbits +) { + if (compressed_words == nullptr) throw std::logic_error("compressed_words == NULL"); + size_t subtotal = 0; + while (true) { + maybe_fill_bitbuf(bitbuf, bufbits, compressed_words, next_word_index, 8); // ensure 8 bits in bit buffer + + const uint8_t peek8 = bitbuf & 0xff; // These 8 bits include either all or part of the Unary codeword + const uint8_t trailing_zeros = byte_trailing_zeros_table[peek8]; + + if (trailing_zeros > 8) throw std::out_of_range("trailing_zeros out of range"); + if (trailing_zeros < 8) { + bufbits -= 1 + trailing_zeros; + bitbuf >>= 1 + trailing_zeros; + return subtotal + trailing_zeros; + } + // The codeword was partial, so read some more + subtotal += 8; + bufbits -= 8; + bitbuf >>= 8; + } +} + +void write_unary( + uint32_t* compressed_words, + size_t& next_word_index, + uint64_t& bitbuf, + uint8_t& bufbits, + uint64_t value +) { + if (compressed_words == nullptr) throw std::logic_error("compressed_words == NULL"); + if (bufbits > 31) throw std::out_of_range("bufbits out of range"); + + uint64_t remaining = value; + + while (remaining >= 16) { + remaining -= 16; + // Here we output 16 zeros, but we don't need to physically write them into bitbuf + // because it already contains zeros in that region. + bufbits += 16; // Record the fact that 16 bits of output have occurred. + maybe_flush_bitbuf(bitbuf, bufbits, compressed_words, next_word_index); + } + + if (remaining > 15) throw std::out_of_range("remaining out of range"); + + const uint64_t the_unary_code = 1 << remaining; + bitbuf |= the_unary_code << bufbits; + bufbits += 1 + remaining; + maybe_flush_bitbuf(bitbuf, bufbits, compressed_words, next_word_index); +} + +// The empty space that this leaves at the beginning of the output array +// will be filled in later by the caller. +template +vector_u32 cpc_compressor::tricky_get_pairs_from_window(const uint8_t* window, uint32_t k, uint32_t num_pairs_to_get, uint32_t empty_space) { + const size_t output_length = empty_space + num_pairs_to_get; + vector_u32 pairs(output_length); + size_t pair_index = empty_space; + for (unsigned row_index = 0; row_index < k; row_index++) { + uint8_t byte = window[row_index]; + while (byte != 0) { + const uint8_t col_index = byte_trailing_zeros_table[byte]; + byte = byte ^ (1 << col_index); // erase the 1 + pairs[pair_index++] = (row_index << 6) | col_index; + } + } + if (pair_index != output_length) throw std::logic_error("pair_index != output_length"); + return pairs; +} + +// returns an integer that is between +// zero and ceiling(log_2(k)) - 1, inclusive +template +uint64_t cpc_compressor::golomb_choose_number_of_base_bits(uint64_t k, uint64_t count) { + if (k < 1) throw std::invalid_argument("golomb_choose_number_of_base_bits: k < 1"); + if (count < 1) throw std::invalid_argument("golomb_choose_number_of_base_bits: count < 1"); + const uint64_t quotient = (k - count) / count; // integer division + if (quotient == 0) return 0; + else return long_floor_log2_of_long(quotient); +} + +} /* namespace datasketches */ + +#endif diff --git a/3rd/datasketches/datasketches/cpc/cpc_confidence.hpp b/3rd/datasketches/datasketches/cpc/cpc_confidence.hpp new file mode 100644 index 000000000..e5670575f --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/cpc_confidence.hpp @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// author Kevin Lang, Oath Research + +#ifndef CPC_CONFIDENCE_HPP_ +#define CPC_CONFIDENCE_HPP_ + +#include + +#include "cpc_sketch.hpp" + +namespace datasketches { + +// ln 2.0 +static const double ICON_ERROT_CONSTANT = 0.693147180559945286; + +// 1, 2, 3, // kappa +static const int16_t ICON_LOW_SIDE_DATA [33] = { // Empirically measured at N = 1000 * K. + 6037, 5720, 5328, // 4 1000000 + 6411, 6262, 5682, // 5 1000000 + 6724, 6403, 6127, // 6 1000000 + 6665, 6411, 6208, // 7 1000000 + 6959, 6525, 6427, // 8 1000000 + 6892, 6665, 6619, // 9 1000000 + 6792, 6752, 6690, // 10 1000000 + 6899, 6818, 6708, // 11 1000000 + 6871, 6845, 6812, // 12 1046369 + 6909, 6861, 6828, // 13 1043411 + 6919, 6897, 6842, // 14 1000297 +}; // lgK numtrials + +// 1, 2, 3, // kappa +static const int16_t ICON_HIGH_SIDE_DATA [33] = { // Empirically measured at N = 1000 * K. + 8031, 8559, 9309, // 4 1000000 + 7084, 7959, 8660, // 5 1000000 + 7141, 7514, 7876, // 6 1000000 + 7458, 7430, 7572, // 7 1000000 + 6892, 7141, 7497, // 8 1000000 + 6889, 7132, 7290, // 9 1000000 + 7075, 7118, 7185, // 10 1000000 + 7040, 7047, 7085, // 11 1000000 + 6993, 7019, 7053, // 12 1046369 + 6953, 7001, 6983, // 13 1043411 + 6944, 6966, 7004, // 14 1000297 +}; // lgK numtrials + +// sqrt((ln 2.0) / 2.0) +static const double HIP_ERROR_CONSTANT = 0.588705011257737332; + +// 1, 2, 3, // kappa +static const int16_t HIP_LOW_SIDE_DATA [33] = { // Empirically measured at N = 1000 * K. + 5871, 5247, 4826, // 4 1000000 + 5877, 5403, 5070, // 5 1000000 + 5873, 5533, 5304, // 6 1000000 + 5878, 5632, 5464, // 7 1000000 + 5874, 5690, 5564, // 8 1000000 + 5880, 5745, 5619, // 9 1000000 + 5875, 5784, 5701, // 10 1000000 + 5866, 5789, 5742, // 11 1000000 + 5869, 5827, 5784, // 12 1046369 + 5876, 5860, 5827, // 13 1043411 + 5881, 5853, 5842, // 14 1000297 +}; // lgK numtrials + +// 1, 2, 3, // kappa +static const int16_t HIP_HIGH_SIDE_DATA [33] = { // Empirically measured at N = 1000 * K. + 5855, 6688, 7391, // 4 1000000 + 5886, 6444, 6923, // 5 1000000 + 5885, 6254, 6594, // 6 1000000 + 5889, 6134, 6326, // 7 1000000 + 5900, 6072, 6203, // 8 1000000 + 5875, 6005, 6089, // 9 1000000 + 5871, 5980, 6040, // 10 1000000 + 5889, 5941, 6015, // 11 1000000 + 5871, 5926, 5973, // 12 1046369 + 5866, 5901, 5915, // 13 1043411 + 5880, 5914, 5953, // 14 1000297 +}; // lgK numtrials + +template +double get_icon_confidence_lb(const cpc_sketch_alloc& sketch, int kappa) { + if (sketch.get_num_coupons() == 0) return 0.0; + const int lg_k = sketch.get_lg_k(); + const long k = 1 << lg_k; + if (lg_k < 4) throw std::logic_error("lgk < 4"); + if (kappa < 1 || kappa > 3) throw std::invalid_argument("kappa must be between 1 and 3"); + double x = ICON_ERROT_CONSTANT; + if (lg_k <= 14) x = ((double) ICON_HIGH_SIDE_DATA[3 * (lg_k - 4) + (kappa - 1)]) / 10000.0; + const double rel = x / sqrt(k); + const double eps = kappa * rel; + const double est = sketch.get_icon_estimate(); + double result = est / (1.0 + eps); + const double check = sketch.get_num_coupons(); + if (result < check) result = check; + return result; +} + +template +double get_icon_confidence_ub(const cpc_sketch_alloc& sketch, int kappa) { + if (sketch.get_num_coupons() == 0) return 0.0; + const int lg_k = sketch.get_lg_k(); + const long k = 1 << lg_k; + if (lg_k < 4) throw std::logic_error("lgk < 4"); + if (kappa < 1 || kappa > 3) throw std::invalid_argument("kappa must be between 1 and 3"); + double x = ICON_ERROT_CONSTANT; + if (lg_k <= 14) x = ((double) ICON_LOW_SIDE_DATA[3 * (lg_k - 4) + (kappa - 1)]) / 10000.0; + const double rel = x / sqrt(k); + const double eps = kappa * rel; + const double est = sketch.get_icon_estimate(); + const double result = est / (1.0 - eps); + return ceil(result); // widening for coverage +} + +template +double get_hip_confidence_lb(const cpc_sketch_alloc& sketch, int kappa) { + if (sketch.get_num_coupons() == 0) return 0.0; + const int lg_k = sketch.get_lg_k(); + const long k = 1 << lg_k; + if (lg_k < 4) throw std::logic_error("lgk < 4"); + if (kappa < 1 || kappa > 3) throw std::invalid_argument("kappa must be between 1 and 3"); + double x = HIP_ERROR_CONSTANT; + if (lg_k <= 14) x = ((double) HIP_HIGH_SIDE_DATA[3 * (lg_k - 4) + (kappa - 1)]) / 10000.0; + const double rel = x / (sqrt((double) k)); + const double eps = ((double) kappa) * rel; + const double est = sketch.get_hip_estimate(); + double result = est / (1.0 + eps); + const double check = (double) sketch.get_num_coupons(); + if (result < check) result = check; + return result; +} + +template +double get_hip_confidence_ub(const cpc_sketch_alloc& sketch, int kappa) { + if (sketch.get_num_coupons() == 0) return 0.0; + const int lg_k = sketch.get_lg_k(); + const long k = 1 << lg_k; + if (lg_k < 4) throw std::logic_error("lgk < 4"); + if (kappa < 1 || kappa > 3) throw std::invalid_argument("kappa must be between 1 and 3"); + double x = HIP_ERROR_CONSTANT; + if (lg_k <= 14) x = ((double) HIP_LOW_SIDE_DATA[3 * (lg_k - 4) + (kappa - 1)]) / 10000.0; + const double rel = x / sqrt(k); + const double eps = kappa * rel; + const double est = sketch.get_hip_estimate(); + const double result = est / (1.0 - eps); + return ceil(result); // widening for coverage +} + +} /* namespace datasketches */ + +#endif diff --git a/3rd/datasketches/datasketches/cpc/cpc_sketch.hpp b/3rd/datasketches/datasketches/cpc/cpc_sketch.hpp new file mode 100644 index 000000000..85855de57 --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/cpc_sketch.hpp @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef CPC_SKETCH_HPP_ +#define CPC_SKETCH_HPP_ + +#include +#include +#include +#include + +#if defined(_MSC_VER) +#include // for and/or keywords +#endif // _MSC_VER + +#include "u32_table.hpp" +#include "cpc_common.hpp" +#include "cpc_compressor.hpp" +#include "cpc_confidence.hpp" + +namespace datasketches { + +/* + * High performance C++ implementation of Compressed Probabilistic Counting (CPC) Sketch + * + * This is a very compact (in serialized form) distinct counting sketch. + * The theory is described in the following paper: + * https://arxiv.org/abs/1708.06839 + * + * author Kevin Lang + * author Alexander Saydakov + */ + +// forward-declarations +template class cpc_sketch_alloc; +template class cpc_union_alloc; + +// alias with default allocator for convenience +typedef cpc_sketch_alloc> cpc_sketch; + +// allocation and initialization of global decompression (decoding) tables +// call this before anything else if you want to control the initialization time +// for instance, to have this happen outside of a transaction context +// otherwise initialization happens on the first use (serialization or deserialization) +// it is safe to call more than once assuming no race conditions +// this is not thread safe! neither is the rest of the library +template void cpc_init(); + +template +class cpc_sketch_alloc { + public: + + explicit cpc_sketch_alloc(uint8_t lg_k = CPC_DEFAULT_LG_K, uint64_t seed = DEFAULT_SEED); + uint8_t get_lg_k() const; + bool is_empty() const; + double get_estimate() const; + double get_lower_bound(unsigned kappa) const; + double get_upper_bound(unsigned kappa) const; + + void update(const std::string& value); + void update(uint64_t value); + void update(int64_t value); + + // for compatibility with Java implementation + void update(uint32_t value); + void update(int32_t value); + void update(uint16_t value); + void update(int16_t value); + void update(uint8_t value); + void update(int8_t value); + void update(double value); + void update(float value); + + // This is a "universal" update that covers all cases above, but may produce different hashes + // Be very careful to hash input values consistently using the same approach over time, + // on different platforms and while passing sketches from or to Java environment + // Otherwise two sketches that should represent overlapping sets will be disjoint + // For instance, for signed 32-bit values call update(int32_t) method above, + // which does widening conversion to int64_t, if compatibility with Java is expected + void update(const void* value, int size); + + // prints a sketch summary to a given stream + void to_stream(std::ostream& os) const; + + void serialize(std::ostream& os) const; + typedef vector_u8 vector_bytes; // alias for users + vector_bytes serialize(unsigned header_size_bytes = 0) const; + + static cpc_sketch_alloc deserialize(std::istream& is, uint64_t seed = DEFAULT_SEED); + static cpc_sketch_alloc deserialize(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED); + + // for internal use + uint32_t get_num_coupons() const; + + // for debugging + // this should catch some forms of corruption during serialization-deserialization + bool validate() const; + + private: + static const uint8_t SERIAL_VERSION = 1; + static const uint8_t FAMILY = 16; + + enum flags { IS_BIG_ENDIAN, IS_COMPRESSED, HAS_HIP, HAS_TABLE, HAS_WINDOW }; + + // Note: except for brief transitional moments, these sketches always obey + // the following strict mapping between the flavor of a sketch and the + // number of coupons that it has collected + enum flavor { + EMPTY, // 0 == C < 1 + SPARSE, // 1 <= C < 3K/32 + HYBRID, // 3K/32 <= C < K/2 + PINNED, // K/2 <= C < 27K/8 [NB: 27/8 = 3 + 3/8] + SLIDING // 27K/8 <= C + }; + + uint8_t lg_k; + uint64_t seed; + bool was_merged; // is the sketch the result of merging? + uint32_t num_coupons; // the number of coupons collected so far + + u32_table surprising_value_table; + vector_u8 sliding_window; + uint8_t window_offset; // derivable from num_coupons, but made explicit for speed + uint8_t first_interesting_column; // This is part of a speed optimization + + double kxp; + double hip_est_accum; + + // for deserialization and cpc_union::get_result() + cpc_sketch_alloc(uint8_t lg_k, uint32_t num_coupons, uint8_t first_interesting_column, u32_table&& table, + vector_u8&& window, bool has_hip, double kxp, double hip_est_accum, uint64_t seed); + + inline void row_col_update(uint32_t row_col); + inline void update_sparse(uint32_t row_col); + inline void update_windowed(uint32_t row_col); + inline void update_hip(uint32_t row_col); + void promote_sparse_to_windowed(); + void move_window(); + void refresh_kxp(const uint64_t* bit_matrix); + + friend double get_hip_confidence_lb(const cpc_sketch_alloc& sketch, int kappa); + friend double get_hip_confidence_ub(const cpc_sketch_alloc& sketch, int kappa); + friend double get_icon_confidence_lb(const cpc_sketch_alloc& sketch, int kappa); + friend double get_icon_confidence_ub(const cpc_sketch_alloc& sketch, int kappa); + double get_hip_estimate() const; + double get_icon_estimate() const; + + inline flavor determine_flavor() const; + static inline flavor determine_flavor(uint8_t lg_k, uint64_t c); + + static inline uint8_t determine_correct_offset(uint8_t lg_k, uint64_t c); + + // this produces a full-size k-by-64 bit matrix + vector_u64 build_bit_matrix() const; + + static uint8_t get_preamble_ints(uint32_t num_coupons, bool has_hip, bool has_table, bool has_window); + inline void write_hip(std::ostream& os) const; + inline size_t copy_hip_to_mem(void* dst) const; + + friend cpc_compressor; + friend cpc_union_alloc; +}; + +} /* namespace datasketches */ + +#include "cpc_sketch_impl.hpp" + +#endif diff --git a/3rd/datasketches/datasketches/cpc/cpc_sketch_impl.hpp b/3rd/datasketches/datasketches/cpc/cpc_sketch_impl.hpp new file mode 100644 index 000000000..58eb1080f --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/cpc_sketch_impl.hpp @@ -0,0 +1,793 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef CPC_SKETCH_IMPL_HPP_ +#define CPC_SKETCH_IMPL_HPP_ + +#include +#include +#include + +#include "cpc_confidence.hpp" +#include "kxp_byte_lookup.hpp" +#include "inv_pow_2_tab.hpp" +#include "cpc_util.hpp" +#include "icon_estimator.hpp" +#include "serde.hpp" + +namespace datasketches { + +template +void cpc_init() { + get_compressor(); // this initializes a global static instance of the compressor on the first use +} + +template +cpc_sketch_alloc::cpc_sketch_alloc(uint8_t lg_k, uint64_t seed): +lg_k(lg_k), +seed(seed), +was_merged(false), +num_coupons(0), +surprising_value_table(2, 6 + lg_k), +sliding_window(), +window_offset(0), +first_interesting_column(0), +kxp(1 << lg_k), +hip_est_accum(0) +{ + if (lg_k < CPC_MIN_LG_K or lg_k > CPC_MAX_LG_K) { + throw std::invalid_argument("lg_k must be >= " + std::to_string(CPC_MIN_LG_K) + " and <= " + std::to_string(CPC_MAX_LG_K) + ": " + std::to_string(lg_k)); + } +} + +template +uint8_t cpc_sketch_alloc::get_lg_k() const { + return lg_k; +} + +template +bool cpc_sketch_alloc::is_empty() const { + return num_coupons == 0; +} + +template +double cpc_sketch_alloc::get_estimate() const { + if (!was_merged) return get_hip_estimate(); + return get_icon_estimate(); +} + +template +double cpc_sketch_alloc::get_hip_estimate() const { + return hip_est_accum; +} + +template +double cpc_sketch_alloc::get_icon_estimate() const { + return compute_icon_estimate(lg_k, num_coupons); +} + +template +double cpc_sketch_alloc::get_lower_bound(unsigned kappa) const { + if (kappa < 1 or kappa > 3) { + throw std::invalid_argument("kappa must be 1, 2 or 3"); + } + if (!was_merged) return get_hip_confidence_lb(*this, kappa); + return get_icon_confidence_lb(*this, kappa); +} + +template +double cpc_sketch_alloc::get_upper_bound(unsigned kappa) const { + if (kappa < 1 or kappa > 3) { + throw std::invalid_argument("kappa must be 1, 2 or 3"); + } + if (!was_merged) return get_hip_confidence_ub(*this, kappa); + return get_icon_confidence_ub(*this, kappa); +} + +template +void cpc_sketch_alloc::update(const std::string& value) { + if (value.empty()) return; + update(value.c_str(), value.length()); +} + +template +void cpc_sketch_alloc::update(uint64_t value) { + update(&value, sizeof(value)); +} + +template +void cpc_sketch_alloc::update(int64_t value) { + update(&value, sizeof(value)); +} + +template +void cpc_sketch_alloc::update(uint32_t value) { + update(static_cast(value)); +} + +template +void cpc_sketch_alloc::update(int32_t value) { + update(static_cast(value)); +} + +template +void cpc_sketch_alloc::update(uint16_t value) { + update(static_cast(value)); +} + +template +void cpc_sketch_alloc::update(int16_t value) { + update(static_cast(value)); +} + +template +void cpc_sketch_alloc::update(uint8_t value) { + update(static_cast(value)); +} + +template +void cpc_sketch_alloc::update(int8_t value) { + update(static_cast(value)); +} + +template +void cpc_sketch_alloc::update(double value) { + union { + int64_t long_value; + double double_value; + } ldu; + if (value == 0.0) { + ldu.double_value = 0.0; // canonicalize -0.0 to 0.0 + } else if (std::isnan(value)) { + ldu.long_value = 0x7ff8000000000000L; // canonicalize NaN using value from Java's Double.doubleToLongBits() + } else { + ldu.double_value = value; + } + update(&ldu, sizeof(ldu)); +} + +template +void cpc_sketch_alloc::update(float value) { + update(static_cast(value)); +} + +static inline uint32_t row_col_from_two_hashes(uint64_t hash0, uint64_t hash1, uint8_t lg_k) { + if (lg_k > 26) throw std::logic_error("lg_k > 26"); + const uint64_t k = 1 << lg_k; + uint8_t col = count_leading_zeros_in_u64(hash1); // 0 <= col <= 64 + if (col > 63) col = 63; // clip so that 0 <= col <= 63 + const uint32_t row = hash0 & (k - 1); + uint32_t row_col = (row << 6) | col; + // To avoid the hash table's "empty" value, we change the row of the following pair. + // This case is extremely unlikely, but we might as well handle it. + if (row_col == UINT32_MAX) row_col ^= 1 << 6; + return row_col; +} + +template +void cpc_sketch_alloc::update(const void* value, int size) { + HashState hashes; + MurmurHash3_x64_128(value, size, seed, hashes); + row_col_update(row_col_from_two_hashes(hashes.h1, hashes.h2, lg_k)); +} + +template +void cpc_sketch_alloc::row_col_update(uint32_t row_col) { + const uint8_t col = row_col & 63; + if (col < first_interesting_column) return; // important speed optimization + // window size is 0 until sketch is promoted from sparse to windowed + if (sliding_window.size() == 0) { + update_sparse(row_col); + } else { + update_windowed(row_col); + } +} + +template +void cpc_sketch_alloc::update_sparse(uint32_t row_col) { + const uint64_t k = 1 << lg_k; + const uint64_t c32pre = static_cast(num_coupons) << 5; + if (c32pre >= 3 * k) throw std::logic_error("c32pre >= 3 * k"); // C < 3K/32, in other words flavor == SPARSE + bool is_novel = surprising_value_table.maybe_insert(row_col); + if (is_novel) { + num_coupons++; + update_hip(row_col); + const uint64_t c32post = static_cast(num_coupons) << 5; + if (c32post >= 3 * k) promote_sparse_to_windowed(); // C >= 3K/32 + } +} + +// the flavor is HYBRID, PINNED, or SLIDING +template +void cpc_sketch_alloc::update_windowed(uint32_t row_col) { + if (window_offset > 56) throw std::logic_error("wrong window offset"); + const uint64_t k = 1 << lg_k; + const uint64_t c32pre = static_cast(num_coupons) << 5; + if (c32pre < 3 * k) throw std::logic_error("c32pre < 3 * k"); // C < 3K/32, in other words flavor >= HYBRID + const uint64_t c8pre = static_cast(num_coupons) << 3; + const uint64_t w8pre = static_cast(window_offset) << 3; + if (c8pre >= (27 + w8pre) * k) throw std::logic_error("c8pre is wrong"); // C < (K * 27/8) + (K * window_offset) + + bool is_novel = false; + const uint8_t col = row_col & 63; + + if (col < window_offset) { // track the surprising 0's "before" the window + is_novel = surprising_value_table.maybe_delete(row_col); // inverted logic + } else if (col < window_offset + 8) { // track the 8 bits inside the window + if (col < window_offset) throw std::logic_error("col < window_offset"); + const uint32_t row = row_col >> 6; + const uint8_t old_bits = sliding_window[row]; + const uint8_t new_bits = old_bits | (1 << (col - window_offset)); + if (new_bits != old_bits) { + sliding_window[row] = new_bits; + is_novel = true; + } + } else { // track the surprising 1's "after" the window + if (col < window_offset + 8) throw std::logic_error("col < window_offset + 8"); + is_novel = surprising_value_table.maybe_insert(row_col); // normal logic + } + + if (is_novel) { + num_coupons++; + update_hip(row_col); + const uint64_t c8post = static_cast(num_coupons) << 3; + if (c8post >= (27 + w8pre) * k) { + move_window(); + if (window_offset < 1 or window_offset > 56) throw std::logic_error("wrong window offset"); + const uint64_t w8post = static_cast(window_offset) << 3; + if (c8post >= (27 + w8post) * k) throw std::logic_error("c8pre is wrong"); // C < (K * 27/8) + (K * window_offset) + } + } +} + +// Call this whenever a new coupon has been collected. +template +void cpc_sketch_alloc::update_hip(uint32_t row_col) { + const uint64_t k = 1 << lg_k; + const uint8_t col = row_col & 63; + const double one_over_p = static_cast(k) / kxp; + hip_est_accum += one_over_p; + kxp -= INVERSE_POWERS_OF_2[col + 1]; // notice the "+1" +} + +// In terms of flavor, this promotes SPARSE to HYBRID +template +void cpc_sketch_alloc::promote_sparse_to_windowed() { + const uint64_t k = 1 << lg_k; + const uint64_t c32 = static_cast(num_coupons) << 5; + if (!(c32 == 3 * k or (lg_k == 4 and c32 > 3 * k))) throw std::logic_error("wrong c32"); + + sliding_window.resize(k, 0); // zero the memory (because we will be OR'ing into it) + + u32_table new_table(2, 6 + lg_k); + + const uint32_t* old_slots = surprising_value_table.get_slots(); + const size_t old_num_slots = 1 << surprising_value_table.get_lg_size(); + + if (window_offset != 0) throw std::logic_error("window_offset != 0"); + + for (size_t i = 0; i < old_num_slots; i++) { + const uint32_t row_col = old_slots[i]; + if (row_col != UINT32_MAX) { + const uint8_t col = row_col & 63; + if (col < 8) { + const size_t row = row_col >> 6; + sliding_window[row] |= 1 << col; + } else { + // cannot use u32_table::must_insert(), because it doesn't provide for growth + const bool is_novel = new_table.maybe_insert(row_col); + if (!is_novel) throw std::logic_error("is_novel != true"); + } + } + } + + surprising_value_table = std::move(new_table); +} + +template +void cpc_sketch_alloc::move_window() { + const uint8_t new_offset = window_offset + 1; + if (new_offset > 56) throw std::logic_error("new_offset > 56"); + if (new_offset != determine_correct_offset(lg_k, num_coupons)) throw std::logic_error("new_offset is wrong"); + + if (sliding_window.size() == 0) throw std::logic_error("no sliding window"); + const uint64_t k = 1 << lg_k; + + // Construct the full-sized bit matrix that corresponds to the sketch + vector_u64 bit_matrix = build_bit_matrix(); + + // refresh the KXP register on every 8th window shift. + if ((new_offset & 0x7) == 0) refresh_kxp(bit_matrix.data()); + + surprising_value_table.clear(); // the new number of surprises will be about the same + + const uint64_t mask_for_clearing_window = (static_cast(0xff) << new_offset) ^ UINT64_MAX; + const uint64_t mask_for_flipping_early_zone = (static_cast(1) << new_offset) - 1; + uint64_t all_surprises_ored = 0; + + for (size_t i = 0; i < k; i++) { + uint64_t pattern = bit_matrix[i]; + sliding_window[i] = (pattern >> new_offset) & 0xff; + pattern &= mask_for_clearing_window; + // The following line converts surprising 0's to 1's in the "early zone", + // (and vice versa, which is essential for this procedure's O(k) time cost). + pattern ^= mask_for_flipping_early_zone; + all_surprises_ored |= pattern; // a cheap way to recalculate first_interesting_column + while (pattern != 0) { + const uint8_t col = count_trailing_zeros_in_u64(pattern); + pattern = pattern ^ (static_cast(1) << col); // erase the 1 + const uint32_t row_col = (i << 6) | col; + const bool is_novel = surprising_value_table.maybe_insert(row_col); + if (!is_novel) throw std::logic_error("is_novel != true"); + } + } + + window_offset = new_offset; + + first_interesting_column = count_trailing_zeros_in_u64(all_surprises_ored); + if (first_interesting_column > new_offset) first_interesting_column = new_offset; // corner case +} + +// The KXP register is a double with roughly 50 bits of precision, but +// it might need roughly 90 bits to track the value with perfect accuracy. +// Therefore we recalculate KXP occasionally from the sketch's full bitmatrix +// so that it will reflect changes that were previously outside the mantissa. +template +void cpc_sketch_alloc::refresh_kxp(const uint64_t* bit_matrix) { + const uint64_t k = 1 << lg_k; + + // for improved numerical accuracy, we separately sum the bytes of the U64's + double byte_sums[8]; // allocating on the stack + std::fill(byte_sums, &byte_sums[8], 0); + + for (size_t i = 0; i < k; i++) { + uint64_t word = bit_matrix[i]; + for (unsigned j = 0; j < 8; j++) { + const uint8_t byte = word & 0xff; + byte_sums[j] += KXP_BYTE_TABLE[byte]; + word >>= 8; + } + } + + double total = 0.0; + for (int j = 7; j >= 0; j--) { // the reverse order is important + const double factor = INVERSE_POWERS_OF_2[8 * j]; // pow (256.0, (-1.0 * ((double) j))); + total += factor * byte_sums[j]; + } + + kxp = total; +} + +template +void cpc_sketch_alloc::to_stream(std::ostream& os) const { + os << "### CPC sketch summary:" << std::endl; + os << " lg_k : " << std::to_string(lg_k) << std::endl; + os << " seed hash : " << std::hex << compute_seed_hash(seed) << std::dec << std::endl; + os << " C : " << num_coupons << std::endl; + os << " flavor : " << determine_flavor() << std::endl; + os << " merged : " << (was_merged ? "true" : "false") << std::endl; + if (!was_merged) { + os << " HIP estimate : " << hip_est_accum << std::endl; + os << " kxp : " << kxp << std::endl; + } + os << " intresting col : " << std::to_string(first_interesting_column) << std::endl; + os << " table entries : " << surprising_value_table.get_num_items() << std::endl; + os << " window : " << (sliding_window.size() == 0 ? "not " : "") << "allocated" << std::endl; + if (sliding_window.size() > 0) { + os << " window offset : " << std::to_string(window_offset) << std::endl; + } + os << "### End sketch summary" << std::endl; +} + +template +void cpc_sketch_alloc::serialize(std::ostream& os) const { + compressed_state compressed; + compressed.table_data_words = 0; + compressed.table_num_entries = 0; + compressed.window_data_words = 0; + get_compressor().compress(*this, compressed); + const bool has_hip = !was_merged; + const bool has_table = compressed.table_data.size() > 0; + const bool has_window = compressed.window_data.size() > 0; + const uint8_t preamble_ints = get_preamble_ints(num_coupons, has_hip, has_table, has_window); + os.write((char*)&preamble_ints, sizeof(preamble_ints)); + const uint8_t serial_version = SERIAL_VERSION; + os.write((char*)&serial_version, sizeof(serial_version)); + const uint8_t family = FAMILY; + os.write((char*)&family, sizeof(family)); + os.write((char*)&lg_k, sizeof(lg_k)); + os.write((char*)&first_interesting_column, sizeof(first_interesting_column)); + const uint8_t flags_byte( + (1 << flags::IS_COMPRESSED) + | (has_hip ? 1 << flags::HAS_HIP : 0) + | (has_table ? 1 << flags::HAS_TABLE : 0) + | (has_window ? 1 << flags::HAS_WINDOW : 0) + ); + os.write((char*)&flags_byte, sizeof(flags_byte)); + const uint16_t seed_hash(compute_seed_hash(seed)); + os.write((char*)&seed_hash, sizeof(seed_hash)); + if (!is_empty()) { + os.write((char*)&num_coupons, sizeof(num_coupons)); + if (has_table and has_window) { + // if there is no window it is the same as number of coupons + os.write((char*)&compressed.table_num_entries, sizeof(compressed.table_num_entries)); + // HIP values can be in two different places in the sequence of fields + // this is the first HIP decision point + if (has_hip) write_hip(os); + } + if (has_table) { + os.write((char*)&compressed.table_data_words, sizeof(compressed.table_data_words)); + } + if (has_window) { + os.write((char*)&compressed.window_data_words, sizeof(compressed.window_data_words)); + } + // this is the second HIP decision point + if (has_hip and !(has_table and has_window)) write_hip(os); + if (has_window) { + os.write((char*)compressed.window_data.data(), compressed.window_data_words * sizeof(uint32_t)); + } + if (has_table) { + os.write((char*)compressed.table_data.data(), compressed.table_data_words * sizeof(uint32_t)); + } + } +} + +template +vector_u8 cpc_sketch_alloc::serialize(unsigned header_size_bytes) const { + compressed_state compressed; + compressed.table_data_words = 0; + compressed.table_num_entries = 0; + compressed.window_data_words = 0; + get_compressor().compress(*this, compressed); + const bool has_hip = !was_merged; + const bool has_table = compressed.table_data.size() > 0; + const bool has_window = compressed.window_data.size() > 0; + const uint8_t preamble_ints = get_preamble_ints(num_coupons, has_hip, has_table, has_window); + const size_t size = header_size_bytes + (preamble_ints + compressed.table_data_words + compressed.window_data_words) * sizeof(uint32_t); + vector_u8 bytes(size); + uint8_t* ptr = bytes.data() + header_size_bytes; + ptr += copy_to_mem(&preamble_ints, ptr, sizeof(preamble_ints)); + const uint8_t serial_version = SERIAL_VERSION; + ptr += copy_to_mem(&serial_version, ptr, sizeof(serial_version)); + const uint8_t family = FAMILY; + ptr += copy_to_mem(&family, ptr, sizeof(family)); + ptr += copy_to_mem(&lg_k, ptr, sizeof(lg_k)); + ptr += copy_to_mem(&first_interesting_column, ptr, sizeof(first_interesting_column)); + const uint8_t flags_byte( + (1 << flags::IS_COMPRESSED) + | (has_hip ? 1 << flags::HAS_HIP : 0) + | (has_table ? 1 << flags::HAS_TABLE : 0) + | (has_window ? 1 << flags::HAS_WINDOW : 0) + ); + ptr += copy_to_mem(&flags_byte, ptr, sizeof(flags_byte)); + const uint16_t seed_hash = compute_seed_hash(seed); + ptr += copy_to_mem(&seed_hash, ptr, sizeof(seed_hash)); + if (!is_empty()) { + ptr += copy_to_mem(&num_coupons, ptr, sizeof(num_coupons)); + if (has_table and has_window) { + // if there is no window it is the same as number of coupons + ptr += copy_to_mem(&compressed.table_num_entries, ptr, sizeof(compressed.table_num_entries)); + // HIP values can be in two different places in the sequence of fields + // this is the first HIP decision point + if (has_hip) ptr += copy_hip_to_mem(ptr); + } + if (has_table) { + ptr += copy_to_mem(&compressed.table_data_words, ptr, sizeof(compressed.table_data_words)); + } + if (has_window) { + ptr += copy_to_mem(&compressed.window_data_words, ptr, sizeof(compressed.window_data_words)); + } + // this is the second HIP decision point + if (has_hip and !(has_table and has_window)) ptr += copy_hip_to_mem(ptr); + if (has_window) { + ptr += copy_to_mem(compressed.window_data.data(), ptr, compressed.window_data_words * sizeof(uint32_t)); + } + if (has_table) { + ptr += copy_to_mem(compressed.table_data.data(), ptr, compressed.table_data_words * sizeof(uint32_t)); + } + } + if (ptr != bytes.data() + size) throw std::logic_error("serialized size mismatch"); + return bytes; +} + +template +cpc_sketch_alloc cpc_sketch_alloc::deserialize(std::istream& is, uint64_t seed) { + uint8_t preamble_ints; + is.read((char*)&preamble_ints, sizeof(preamble_ints)); + uint8_t serial_version; + is.read((char*)&serial_version, sizeof(serial_version)); + uint8_t family_id; + is.read((char*)&family_id, sizeof(family_id)); + uint8_t lg_k; + is.read((char*)&lg_k, sizeof(lg_k)); + uint8_t first_interesting_column; + is.read((char*)&first_interesting_column, sizeof(first_interesting_column)); + uint8_t flags_byte; + is.read((char*)&flags_byte, sizeof(flags_byte)); + uint16_t seed_hash; + is.read((char*)&seed_hash, sizeof(seed_hash)); + const bool has_hip = flags_byte & (1 << flags::HAS_HIP); + const bool has_table = flags_byte & (1 << flags::HAS_TABLE); + const bool has_window = flags_byte & (1 << flags::HAS_WINDOW); + compressed_state compressed; + compressed.table_data_words = 0; + compressed.table_num_entries = 0; + compressed.window_data_words = 0; + uint32_t num_coupons = 0; + double kxp = 0; + double hip_est_accum = 0; + if (has_table or has_window) { + is.read((char*)&num_coupons, sizeof(num_coupons)); + if (has_table and has_window) { + is.read((char*)&compressed.table_num_entries, sizeof(compressed.table_num_entries)); + if (has_hip) { + is.read((char*)&kxp, sizeof(kxp)); + is.read((char*)&hip_est_accum, sizeof(hip_est_accum)); + } + } + if (has_table) { + is.read((char*)&compressed.table_data_words, sizeof(compressed.table_data_words)); + } + if (has_window) { + is.read((char*)&compressed.window_data_words, sizeof(compressed.window_data_words)); + } + if (has_hip and !(has_table and has_window)) { + is.read((char*)&kxp, sizeof(kxp)); + is.read((char*)&hip_est_accum, sizeof(hip_est_accum)); + } + if (has_window) { + compressed.window_data.resize(compressed.window_data_words); + is.read((char*)compressed.window_data.data(), compressed.window_data_words * sizeof(uint32_t)); + } + if (has_table) { + compressed.table_data.resize(compressed.table_data_words); + is.read((char*)compressed.table_data.data(), compressed.table_data_words * sizeof(uint32_t)); + } + if (!has_window) compressed.table_num_entries = num_coupons; + } + + uint8_t expected_preamble_ints = get_preamble_ints(num_coupons, has_hip, has_table, has_window); + if (preamble_ints != expected_preamble_ints) { + throw std::invalid_argument("Possible corruption: preamble ints: expected " + + std::to_string(expected_preamble_ints) + ", got " + std::to_string(preamble_ints)); + } + if (serial_version != SERIAL_VERSION) { + throw std::invalid_argument("Possible corruption: serial version: expected " + + std::to_string(SERIAL_VERSION) + ", got " + std::to_string(serial_version)); + } + if (family_id != FAMILY) { + throw std::invalid_argument("Possible corruption: family: expected " + + std::to_string(FAMILY) + ", got " + std::to_string(family_id)); + } + if (seed_hash != compute_seed_hash(seed)) { + throw std::invalid_argument("Incompatible seed hashes: " + std::to_string(seed_hash) + ", " + + std::to_string(compute_seed_hash(seed))); + } + uncompressed_state uncompressed; + get_compressor().uncompress(compressed, uncompressed, lg_k, num_coupons); + return cpc_sketch_alloc(lg_k, num_coupons, first_interesting_column, std::move(uncompressed.table), + std::move(uncompressed.window), has_hip, kxp, hip_est_accum, seed); +} + +template +cpc_sketch_alloc cpc_sketch_alloc::deserialize(const void* bytes, size_t size, uint64_t seed) { + const char* ptr = static_cast(bytes); + uint8_t preamble_ints; + ptr += copy_from_mem(ptr, &preamble_ints, sizeof(preamble_ints)); + uint8_t serial_version; + ptr += copy_from_mem(ptr, &serial_version, sizeof(serial_version)); + uint8_t family_id; + ptr += copy_from_mem(ptr, &family_id, sizeof(family_id)); + uint8_t lg_k; + ptr += copy_from_mem(ptr, &lg_k, sizeof(lg_k)); + uint8_t first_interesting_column; + ptr += copy_from_mem(ptr, &first_interesting_column, sizeof(first_interesting_column)); + uint8_t flags_byte; + ptr += copy_from_mem(ptr, &flags_byte, sizeof(flags_byte)); + uint16_t seed_hash; + ptr += copy_from_mem(ptr, &seed_hash, sizeof(seed_hash)); + const bool has_hip = flags_byte & (1 << flags::HAS_HIP); + const bool has_table = flags_byte & (1 << flags::HAS_TABLE); + const bool has_window = flags_byte & (1 << flags::HAS_WINDOW); + compressed_state compressed; + compressed.table_data_words = 0; + compressed.table_num_entries = 0; + compressed.window_data_words = 0; + uint32_t num_coupons = 0; + double kxp = 0; + double hip_est_accum = 0; + if (has_table or has_window) { + ptr += copy_from_mem(ptr, &num_coupons, sizeof(num_coupons)); + if (has_table and has_window) { + ptr += copy_from_mem(ptr, &compressed.table_num_entries, sizeof(compressed.table_num_entries)); + if (has_hip) { + ptr += copy_from_mem(ptr, &kxp, sizeof(kxp)); + ptr += copy_from_mem(ptr, &hip_est_accum, sizeof(hip_est_accum)); + } + } + if (has_table) { + ptr += copy_from_mem(ptr, &compressed.table_data_words, sizeof(compressed.table_data_words)); + } + if (has_window) { + ptr += copy_from_mem(ptr, &compressed.window_data_words, sizeof(compressed.window_data_words)); + } + if (has_hip and !(has_table and has_window)) { + ptr += copy_from_mem(ptr, &kxp, sizeof(kxp)); + ptr += copy_from_mem(ptr, &hip_est_accum, sizeof(hip_est_accum)); + } + if (has_window) { + compressed.window_data.resize(compressed.window_data_words); + ptr += copy_from_mem(ptr, compressed.window_data.data(), compressed.window_data_words * sizeof(uint32_t)); + } + if (has_table) { + compressed.table_data.resize(compressed.table_data_words); + ptr += copy_from_mem(ptr, compressed.table_data.data(), compressed.table_data_words * sizeof(uint32_t)); + } + if (!has_window) compressed.table_num_entries = num_coupons; + } + if (ptr != static_cast(bytes) + size) throw std::logic_error("deserialized size mismatch"); + + uint8_t expected_preamble_ints = get_preamble_ints(num_coupons, has_hip, has_table, has_window); + if (preamble_ints != expected_preamble_ints) { + throw std::invalid_argument("Possible corruption: preamble ints: expected " + + std::to_string(expected_preamble_ints) + ", got " + std::to_string(preamble_ints)); + } + if (serial_version != SERIAL_VERSION) { + throw std::invalid_argument("Possible corruption: serial version: expected " + + std::to_string(SERIAL_VERSION) + ", got " + std::to_string(serial_version)); + } + if (family_id != FAMILY) { + throw std::invalid_argument("Possible corruption: family: expected " + + std::to_string(FAMILY) + ", got " + std::to_string(family_id)); + } + if (seed_hash != compute_seed_hash(seed)) { + throw std::invalid_argument("Incompatible seed hashes: " + std::to_string(seed_hash) + ", " + + std::to_string(compute_seed_hash(seed))); + } + uncompressed_state uncompressed; + get_compressor().uncompress(compressed, uncompressed, lg_k, num_coupons); + return cpc_sketch_alloc(lg_k, num_coupons, first_interesting_column, std::move(uncompressed.table), + std::move(uncompressed.window), has_hip, kxp, hip_est_accum, seed); +} + +template +uint32_t cpc_sketch_alloc::get_num_coupons() const { + return num_coupons; +} + +template +bool cpc_sketch_alloc::validate() const { + vector_u64 bit_matrix = build_bit_matrix(); + const uint64_t num_bits_set = count_bits_set_in_matrix(bit_matrix.data(), 1 << lg_k); + return num_bits_set == num_coupons; +} + +template +cpc_sketch_alloc::cpc_sketch_alloc(uint8_t lg_k, uint32_t num_coupons, uint8_t first_interesting_column, + u32_table&& table, vector_u8&& window, bool has_hip, double kxp, double hip_est_accum, uint64_t seed): +lg_k(lg_k), +seed(seed), +was_merged(!has_hip), +num_coupons(num_coupons), +surprising_value_table(std::move(table)), +sliding_window(std::move(window)), +window_offset(determine_correct_offset(lg_k, num_coupons)), +first_interesting_column(first_interesting_column), +kxp(kxp), +hip_est_accum(hip_est_accum) +{} + +template +uint8_t cpc_sketch_alloc::get_preamble_ints(uint32_t num_coupons, bool has_hip, bool has_table, bool has_window) { + uint8_t preamble_ints = 2; + if (num_coupons > 0) { + preamble_ints += 1; // number of coupons + if (has_hip) { + preamble_ints += 4; // HIP + } + if (has_table) { + preamble_ints += 1; // table data length + // number of values (if there is no window it is the same as number of coupons) + if (has_window) { + preamble_ints += 1; + } + } + if (has_window) { + preamble_ints += 1; // window length + } + } + return preamble_ints; +} + +template +typename cpc_sketch_alloc::flavor cpc_sketch_alloc::determine_flavor() const { + return determine_flavor(lg_k, num_coupons); +} + +template +typename cpc_sketch_alloc::flavor cpc_sketch_alloc::determine_flavor(uint8_t lg_k, uint64_t c) { + const uint64_t k = 1 << lg_k; + const uint64_t c2 = c << 1; + const uint64_t c8 = c << 3; + const uint64_t c32 = c << 5; + if (c == 0) return EMPTY; // 0 == C < 1 + if (c32 < 3 * k) return SPARSE; // 1 <= C < 3K/32 + if (c2 < k) return HYBRID; // 3K/32 <= C < K/2 + if (c8 < 27 * k) return PINNED; // K/2 <= C < 27K/8 + else return SLIDING; // 27K/8 <= C +} + +template +uint8_t cpc_sketch_alloc::determine_correct_offset(uint8_t lg_k, uint64_t c) { + const uint64_t k = 1 << lg_k; + const int64_t tmp = static_cast(c << 3) - static_cast(19 * k); // 8C - 19K + if (tmp < 0) return 0; + return tmp >> (lg_k + 3); // tmp / 8K +} + +template +vector_u64 cpc_sketch_alloc::build_bit_matrix() const { + const size_t k = 1 << lg_k; + if (window_offset > 56) throw std::logic_error("offset > 56"); + + // Fill the matrix with default rows in which the "early zone" is filled with ones. + // This is essential for the routine's O(k) time cost (as opposed to O(C)). + const uint64_t default_row = (static_cast(1) << window_offset) - 1; + vector_u64 matrix(k, default_row); + + if (num_coupons == 0) return matrix; + + if (sliding_window.size() > 0) { // In other words, we are in window mode, not sparse mode + for (size_t i = 0; i < k; i++) { // set the window bits, trusting the sketch's current offset + matrix[i] |= static_cast(sliding_window[i]) << window_offset; + } + } + + const uint32_t* slots = surprising_value_table.get_slots(); + const size_t num_slots = 1 << surprising_value_table.get_lg_size(); + for (size_t i = 0; i < num_slots; i++) { + const uint32_t row_col = slots[i]; + if (row_col != UINT32_MAX) { + const uint8_t col = row_col & 63; + const size_t row = row_col >> 6; + // Flip the specified matrix bit from its default value. + // In the "early" zone the bit changes from 1 to 0. + // In the "late" zone the bit changes from 0 to 1. + matrix[row] ^= static_cast(1) << col; + } + } + return matrix; +} + +template +void cpc_sketch_alloc::write_hip(std::ostream& os) const { + os.write((char*)&kxp, sizeof(kxp)); + os.write((char*)&hip_est_accum, sizeof(hip_est_accum)); +} + +template +size_t cpc_sketch_alloc::copy_hip_to_mem(void* dst) const { + memcpy(dst, &kxp, sizeof(kxp)); + memcpy(static_cast(dst) + sizeof(kxp), &hip_est_accum, sizeof(hip_est_accum)); + return sizeof(kxp) + sizeof(hip_est_accum); +} + +} /* namespace datasketches */ + +#endif diff --git a/3rd/datasketches/datasketches/cpc/cpc_union.hpp b/3rd/datasketches/datasketches/cpc/cpc_union.hpp new file mode 100644 index 000000000..9601cc03e --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/cpc_union.hpp @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef CPC_UNION_HPP_ +#define CPC_UNION_HPP_ + +#include + +#if defined(_MSC_VER) +#include // for and/or keywords +#endif // _MSC_VER + +#include "cpc_sketch.hpp" + +namespace datasketches { + +/* + * High performance C++ implementation of Compressed Probabilistic Counting (CPC) Union + * + * author Kevin Lang + * author Alexander Saydakov + */ + +// alias with default allocator for convenience +typedef cpc_union_alloc> cpc_union; + +template +class cpc_union_alloc { + public: + + explicit cpc_union_alloc(uint8_t lg_k = CPC_DEFAULT_LG_K, uint64_t seed = DEFAULT_SEED); + cpc_union_alloc(const cpc_union_alloc& other); + cpc_union_alloc(cpc_union_alloc&& other) noexcept; + ~cpc_union_alloc(); + + cpc_union_alloc& operator=(const cpc_union_alloc& other); + cpc_union_alloc& operator=(cpc_union_alloc&& other) noexcept; + + void update(const cpc_sketch_alloc& sketch); + cpc_sketch_alloc get_result() const; + + private: + typedef typename std::allocator_traits::template rebind_alloc AllocU8; + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + typedef typename std::allocator_traits::template rebind_alloc> AllocCpc; + + uint8_t lg_k; + uint64_t seed; + cpc_sketch_alloc* accumulator; + vector_u64 bit_matrix; + + cpc_sketch_alloc get_result_from_accumulator() const; + cpc_sketch_alloc get_result_from_bit_matrix() const; + + void switch_to_bit_matrix(); + void walk_table_updating_sketch(const u32_table& table); + void or_table_into_matrix(const u32_table& table); + void or_window_into_matrix(const vector_u8& sliding_window, uint8_t offset, uint8_t src_lg_k); + void or_matrix_into_matrix(const vector_u64& src_matrix, uint8_t src_lg_k); + void reduce_k(uint8_t new_lg_k); +}; + +} /* namespace datasketches */ + +#include "cpc_union_impl.hpp" + +#endif diff --git a/3rd/datasketches/datasketches/cpc/cpc_union_impl.hpp b/3rd/datasketches/datasketches/cpc/cpc_union_impl.hpp new file mode 100644 index 000000000..3363b7720 --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/cpc_union_impl.hpp @@ -0,0 +1,332 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef CPC_UNION_IMPL_HPP_ +#define CPC_UNION_IMPL_HPP_ + +namespace datasketches { + +template +cpc_union_alloc::cpc_union_alloc(uint8_t lg_k, uint64_t seed): +lg_k(lg_k), +seed(seed), +accumulator(new (AllocCpc().allocate(1)) cpc_sketch_alloc(lg_k, seed)), +bit_matrix() +{ + if (lg_k < CPC_MIN_LG_K or lg_k > CPC_MAX_LG_K) { + throw std::invalid_argument("lg_k must be >= " + std::to_string(CPC_MIN_LG_K) + " and <= " + std::to_string(CPC_MAX_LG_K) + ": " + std::to_string(lg_k)); + } +} + +template +cpc_union_alloc::cpc_union_alloc(const cpc_union_alloc& other): +lg_k(other.lg_k), +seed(other.seed), +accumulator(other.accumulator), +bit_matrix(other.bit_matrix) +{ + if (accumulator != nullptr) { + accumulator = new (AllocCpc().allocate(1)) cpc_sketch_alloc(*other.accumulator); + } +} + +template +cpc_union_alloc::cpc_union_alloc(cpc_union_alloc&& other) noexcept: +lg_k(other.lg_k), +seed(other.seed), +accumulator(other.accumulator), +bit_matrix(std::move(other.bit_matrix)) +{ + other.accumulator = nullptr; +} + +template +cpc_union_alloc::~cpc_union_alloc() { + if (accumulator != nullptr) { + accumulator->~cpc_sketch_alloc(); + AllocCpc().deallocate(accumulator, 1); + } +} + +template +cpc_union_alloc& cpc_union_alloc::operator=(const cpc_union_alloc& other) { + cpc_union_alloc copy(other); + std::swap(lg_k, copy.lg_k); + seed = copy.seed; + std::swap(accumulator, copy.accumulator); + bit_matrix = std::move(copy.bit_matrix); + return *this; +} + +template +cpc_union_alloc& cpc_union_alloc::operator=(cpc_union_alloc&& other) noexcept { + std::swap(lg_k, other.lg_k); + seed = other.seed; + std::swap(accumulator, other.accumulator); + bit_matrix = std::move(other.bit_matrix); + return *this; +} + +template +void cpc_union_alloc::update(const cpc_sketch_alloc& sketch) { + const uint16_t seed_hash_union = compute_seed_hash(seed); + const uint16_t seed_hash_sketch = compute_seed_hash(sketch.seed); + if (seed_hash_union != seed_hash_sketch) { + throw std::invalid_argument("Incompatible seed hashes: " + std::to_string(seed_hash_union) + ", " + + std::to_string(seed_hash_sketch)); + } + const auto src_flavor = sketch.determine_flavor(); + if (cpc_sketch_alloc::flavor::EMPTY == src_flavor) return; + + if (sketch.get_lg_k() < lg_k) reduce_k(sketch.get_lg_k()); + if (sketch.get_lg_k() < lg_k) throw std::logic_error("sketch lg_k < union lg_k"); + + if (accumulator == nullptr and bit_matrix.size() == 0) throw std::logic_error("both accumulator and bit matrix are absent"); + + if (cpc_sketch_alloc::flavor::SPARSE == src_flavor and accumulator != nullptr) { // Case A + if (bit_matrix.size() > 0) throw std::logic_error("union bit_matrix is not expected"); + const auto initial_dest_flavor = accumulator->determine_flavor(); + if (cpc_sketch_alloc::flavor::EMPTY != initial_dest_flavor and + cpc_sketch_alloc::flavor::SPARSE != initial_dest_flavor) throw std::logic_error("wrong flavor"); + + // The following partially fixes the snowplow problem provided that the K's are equal. + // A complete fix is coming soon. + if (cpc_sketch_alloc::flavor::EMPTY == initial_dest_flavor and lg_k == sketch.get_lg_k()) { + *accumulator = sketch; + } + + walk_table_updating_sketch(sketch.surprising_value_table); + const auto final_dst_flavor = accumulator->determine_flavor(); + // if the accumulator has graduated beyond sparse, switch to a bit matrix representation + if (final_dst_flavor != cpc_sketch_alloc::flavor::EMPTY and final_dst_flavor != cpc_sketch_alloc::flavor::SPARSE) { + switch_to_bit_matrix(); + } + return; + } + + if (cpc_sketch_alloc::flavor::SPARSE == src_flavor and bit_matrix.size() > 0) { // Case B + if (accumulator != nullptr) throw std::logic_error("union accumulator != null"); + or_table_into_matrix(sketch.surprising_value_table); + return; + } + + if (cpc_sketch_alloc::flavor::HYBRID != src_flavor and cpc_sketch_alloc::flavor::PINNED != src_flavor + and cpc_sketch_alloc::flavor::SLIDING != src_flavor) throw std::logic_error("wrong flavor"); + + // source is past SPARSE mode, so make sure that dest is a bit matrix + if (accumulator != nullptr) { + if (bit_matrix.size() > 0) throw std::logic_error("union bit matrix is not expected"); + const auto dst_flavor = accumulator->determine_flavor(); + if (cpc_sketch_alloc::flavor::EMPTY != dst_flavor and cpc_sketch_alloc::flavor::SPARSE != dst_flavor) { + throw std::logic_error("wrong flavor"); + } + switch_to_bit_matrix(); + } + if (bit_matrix.size() == 0) throw std::logic_error("union bit_matrix is expected"); + + if (cpc_sketch_alloc::flavor::HYBRID == src_flavor or cpc_sketch_alloc::flavor::PINNED == src_flavor) { // Case C + or_window_into_matrix(sketch.sliding_window, sketch.window_offset, sketch.get_lg_k()); + or_table_into_matrix(sketch.surprising_value_table); + return; + } + + // SLIDING mode involves inverted logic, so we can't just walk the source sketch. + // Instead, we convert it to a bitMatrix that can be OR'ed into the destination. + if (cpc_sketch_alloc::flavor::SLIDING != src_flavor) throw std::logic_error("wrong flavor"); // Case D + vector_u64 src_matrix = sketch.build_bit_matrix(); + or_matrix_into_matrix(src_matrix, sketch.get_lg_k()); +} + +template +cpc_sketch_alloc cpc_union_alloc::get_result() const { + if (accumulator != nullptr) { + if (bit_matrix.size() > 0) throw std::logic_error("bit_matrix is not expected"); + return get_result_from_accumulator(); + } + if (bit_matrix.size() == 0) throw std::logic_error("bit_matrix is expected"); + return get_result_from_bit_matrix(); +} + +template +cpc_sketch_alloc cpc_union_alloc::get_result_from_accumulator() const { + if (lg_k != accumulator->get_lg_k()) throw std::logic_error("lg_k != accumulator->lg_k"); + if (accumulator->get_num_coupons() == 0) { + return cpc_sketch_alloc(lg_k, seed); + } + if (accumulator->determine_flavor() != cpc_sketch_alloc::flavor::SPARSE) throw std::logic_error("wrong flavor"); + cpc_sketch_alloc copy(*accumulator); + copy.was_merged = true; + return copy; +} + +template +cpc_sketch_alloc cpc_union_alloc::get_result_from_bit_matrix() const { + const uint64_t k = 1 << lg_k; + const uint64_t num_coupons = count_bits_set_in_matrix(bit_matrix.data(), k); + + const auto flavor = cpc_sketch_alloc::determine_flavor(lg_k, num_coupons); + if (flavor != cpc_sketch_alloc::flavor::HYBRID and flavor != cpc_sketch_alloc::flavor::PINNED + and flavor != cpc_sketch_alloc::flavor::SLIDING) throw std::logic_error("wrong flavor"); + + const uint8_t offset = cpc_sketch_alloc::determine_correct_offset(lg_k, num_coupons); + + vector_u8 sliding_window(k); + // don't need to zero the window's memory + + // dynamically growing caused snowplow effect + uint8_t table_lg_size = lg_k - 4; // K/16; in some cases this will end up being oversized + if (table_lg_size < 2) table_lg_size = 2; + u32_table table(table_lg_size, 6 + lg_k); + + // the following should work even when the offset is zero + const uint64_t mask_for_clearing_window = (static_cast(0xff) << offset) ^ UINT64_MAX; + const uint64_t mask_for_flipping_early_zone = (static_cast(1) << offset) - 1; + uint64_t all_surprises_ored = 0; + + // The snowplow effect was caused by processing the rows in order, + // but we have fixed it by using a sufficiently large hash table. + for (unsigned i = 0; i < k; i++) { + uint64_t pattern = bit_matrix[i]; + sliding_window[i] = (pattern >> offset) & 0xff; + pattern &= mask_for_clearing_window; + pattern ^= mask_for_flipping_early_zone; // this flipping converts surprising 0's to 1's + all_surprises_ored |= pattern; + while (pattern != 0) { + const uint8_t col = count_trailing_zeros_in_u64(pattern); + pattern = pattern ^ (static_cast(1) << col); // erase the 1 + const uint32_t row_col = (i << 6) | col; + bool is_novel = table.maybe_insert(row_col); + if (!is_novel) throw std::logic_error("is_novel != true"); + } + } + + // at this point we could shrink an oversized hash table, but the relative waste isn't very big + + uint8_t first_interesting_column = count_trailing_zeros_in_u64(all_surprises_ored); + if (first_interesting_column > offset) first_interesting_column = offset; // corner case + + // HIP-related fields will contain zeros, and that is okay + return cpc_sketch_alloc(lg_k, num_coupons, first_interesting_column, std::move(table), std::move(sliding_window), false, 0, 0, seed); +} + +template +void cpc_union_alloc::switch_to_bit_matrix() { + bit_matrix = accumulator->build_bit_matrix(); + accumulator->~cpc_sketch_alloc(); + AllocCpc().deallocate(accumulator, 1); + accumulator = nullptr; +} + +template +void cpc_union_alloc::walk_table_updating_sketch(const u32_table& table) { + const uint32_t* slots = table.get_slots(); + const size_t num_slots = 1 << table.get_lg_size(); + const uint64_t dst_mask = (((1 << accumulator->get_lg_k()) - 1) << 6) | 63; // downsamples when dst lgK < src LgK + + // Using a golden ratio stride fixes the snowplow effect. + const double golden = 0.6180339887498949025; + size_t stride = static_cast(golden * static_cast(num_slots)); + if (stride < 2) throw std::logic_error("stride < 2"); + if (stride == ((stride >> 1) << 1)) stride += 1; // force the stride to be odd + if (stride < 3 or stride >= num_slots) throw std::out_of_range("stride out of range"); + + for (size_t i = 0, j = 0; i < num_slots; i++, j += stride) { + j &= num_slots - 1; + const uint32_t row_col = slots[j]; + if (row_col != UINT32_MAX) { + accumulator->row_col_update(row_col & dst_mask); + } + } +} + +template +void cpc_union_alloc::or_table_into_matrix(const u32_table& table) { + const uint32_t* slots = table.get_slots(); + const size_t num_slots = 1 << table.get_lg_size(); + const uint64_t dest_mask = (1 << lg_k) - 1; // downsamples when dst lgK < sr LgK + for (size_t i = 0; i < num_slots; i++) { + const uint32_t row_col = slots[i]; + if (row_col != UINT32_MAX) { + const uint8_t col = row_col & 63; + const size_t row = row_col >> 6; + bit_matrix[row & dest_mask] |= static_cast(1) << col; // set the bit + } + } +} + +template +void cpc_union_alloc::or_window_into_matrix(const vector_u8& sliding_window, uint8_t offset, uint8_t src_lg_k) { + if (lg_k > src_lg_k) throw std::logic_error("dst LgK > src LgK"); + const uint64_t dst_mask = (1 << lg_k) - 1; // downsamples when dst lgK < src LgK + const size_t src_k = 1 << src_lg_k; + for (size_t src_row = 0; src_row < src_k; src_row++) { + bit_matrix[src_row & dst_mask] |= static_cast(sliding_window[src_row]) << offset; + } +} + +template +void cpc_union_alloc::or_matrix_into_matrix(const vector_u64& src_matrix, uint8_t src_lg_k) { + if (lg_k > src_lg_k) throw std::logic_error("dst LgK > src LgK"); + const uint64_t dst_mask = (1 << lg_k) - 1; // downsamples when dst lgK < src LgK + const size_t src_k = 1 << src_lg_k; + for (size_t src_row = 0; src_row < src_k; src_row++) { + bit_matrix[src_row & dst_mask] |= src_matrix[src_row]; + } +} + +template +void cpc_union_alloc::reduce_k(uint8_t new_lg_k) { + if (new_lg_k >= lg_k) throw std::logic_error("new LgK >= union lgK"); + if (accumulator == nullptr and bit_matrix.size() == 0) throw std::logic_error("both accumulator and bit_matrix are absent"); + + if (bit_matrix.size() > 0) { // downsample the unioner's bit matrix + if (accumulator != nullptr) throw std::logic_error("accumulator is not null"); + vector_u64 old_matrix = std::move(bit_matrix); + const uint8_t old_lg_k = lg_k; + const size_t new_k = 1 << new_lg_k; + bit_matrix = vector_u64(new_k, 0); + lg_k = new_lg_k; + or_matrix_into_matrix(old_matrix, old_lg_k); + return; + } + + if (accumulator != nullptr) { // downsample the unioner's sketch + if (bit_matrix.size() > 0) throw std::logic_error("bit_matrix is not expected"); + if (!accumulator->is_empty()) { + cpc_sketch_alloc old_accumulator(*accumulator); + *accumulator = cpc_sketch_alloc(new_lg_k, seed); + walk_table_updating_sketch(old_accumulator.surprising_value_table); + } + lg_k = new_lg_k; + + const auto final_new_flavor = accumulator->determine_flavor(); + // if the new sketch has graduated beyond sparse, convert to bit_matrix + if (final_new_flavor != cpc_sketch_alloc::flavor::EMPTY and + final_new_flavor != cpc_sketch_alloc::flavor::SPARSE) { + switch_to_bit_matrix(); + } + return; + } + + throw std::logic_error("invalid state"); +} + +} /* namespace datasketches */ + +#endif diff --git a/3rd/datasketches/datasketches/cpc/cpc_util.hpp b/3rd/datasketches/datasketches/cpc/cpc_util.hpp new file mode 100644 index 000000000..b63f26f24 --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/cpc_util.hpp @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef CPC_UTIL_HPP_ +#define CPC_UTIL_HPP_ + +#include + +namespace datasketches { + +static inline uint16_t compute_seed_hash(uint64_t seed) { + HashState hashes; + MurmurHash3_x64_128(&seed, sizeof(seed), 0, hashes); + return hashes.h1 & 0xffff; +} + +static inline uint64_t divide_longs_rounding_up(uint64_t x, uint64_t y) { + if (y == 0) throw std::invalid_argument("divide_longs_rounding_up: bad argument"); + const uint64_t quotient = x / y; + if (quotient * y == x) return (quotient); + else return quotient + 1; +} + +static inline uint64_t long_floor_log2_of_long(uint64_t x) { + if (x < 1) throw std::invalid_argument("long_floor_log2_of_long: bad argument"); + uint64_t p = 0; + uint64_t y = 1; + while (true) { + if (y == x) return p; + if (y > x) return p - 1; + p += 1; + y <<= 1; + } +} + +// This place-holder code was inadequate because it caused +// the cost of the post-merge get_result() operation to be O(C) +// instead of O(K). It did have the advantage of being +// very simple and trustworthy during initial testing. +static inline uint64_t wegner_count_bits_set_in_matrix(const uint64_t* array, size_t length) { + uint64_t pattern = 0; + uint64_t count = 0; + // clock_t t0, t1; + // t0 = clock(); + // Wegner's Bit-Counting Algorithm, CACM 3 (1960), p. 322. + for (uint64_t i = 0; i < length; i++) { + pattern = array[i]; + while (pattern != 0) { + pattern &= (pattern - 1); + count++; + } + } + // t1 = clock(); + // printf ("\n(Wegner CountBitsTime %.1f)\n", ((double) (t1 - t0)) / 1000.0); + // fflush (stdout); + return count; +} + +// Note: this is an adaptation of the Java code, +// which is apparently a variation of Figure 5-2 in "Hacker's Delight" +// by Henry S. Warren. +static inline uint64_t warren_bit_count(uint64_t i) { + i = i - ((i >> 1) & 0x5555555555555555ULL); + i = (i & 0x3333333333333333ULL) + ((i >> 2) & 0x3333333333333333ULL); + i = (i + (i >> 4)) & 0x0f0f0f0f0f0f0f0fULL; + i = i + (i >> 8); + i = i + (i >> 16); + i = i + (i >> 32); + return i & 0x7f; +} + +static inline uint64_t warren_count_bits_set_in_matrix(const uint64_t* array, size_t length) { + uint64_t count = 0; + for (size_t i = 0; i < length; i++) { + count += warren_bit_count(array[i]); + } + return count; +} + +// This code is Figure 5-9 in "Hacker's Delight" by Henry S. Warren. + +#define CSA(h,l,a,b,c) {uint64_t u = a ^ b; uint64_t v = c; h = (a & b) | (u & v); l = u ^ v;} + +static inline uint64_t count_bits_set_in_matrix(const uint64_t* a, size_t length) { + if ((length & 0x7) != 0) throw std::invalid_argument("the length of the array must be a multiple of 8"); + uint64_t total = 0; + uint64_t ones, twos, twos_a, twos_b, fours, fours_a, fours_b, eights; + fours = twos = ones = 0; + + for (size_t i = 0; i <= length - 8; i = i + 8) { + CSA(twos_a, ones, ones, a[i+0], a[i+1]); + CSA(twos_b, ones, ones, a[i+2], a[i+3]); + CSA(fours_a, twos, twos, twos_a, twos_b); + + CSA(twos_a, ones, ones, a[i+4], a[i+5]); + CSA(twos_b, ones, ones, a[i+6], a[i+7]); + CSA(fours_b, twos, twos, twos_a, twos_b); + + CSA(eights, fours, fours, fours_a, fours_b); + + total += warren_bit_count(eights); + } + total = 8 * total + 4 * warren_bit_count(fours) + 2 * warren_bit_count(twos) + warren_bit_count(ones); + + // Because I still don't fully trust this fancy version + // assert(total == wegner_count_bits_set_in_matrix(A, length)); + //if (total != wegner_count_bits_set_in_matrix(a, length)) throw std::logic_error("count_bits_set_in_matrix error"); + + return total; +} + +// Here are some timings made with quickTestMerge.c +// for the "5 5" case: + +// Wegner CountBitsTime 29.3 +// Warren CountBitsTime 5.3 +// CSA CountBitsTime 4.3 + +} /* namespace datasketches */ + +#endif diff --git a/3rd/datasketches/datasketches/cpc/icon_estimator.hpp b/3rd/datasketches/datasketches/cpc/icon_estimator.hpp new file mode 100644 index 000000000..27d76cae9 --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/icon_estimator.hpp @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// author Kevin Lang, Oath Research + +#ifndef ICON_ESTIMATOR_HPP_ +#define ICON_ESTIMATOR_HPP_ + +#include +#include +#include + +namespace datasketches { + +// The ICON estimator for FM85 sketches is defined by the arXiv paper. + +// The current file provides exact and approximate implementations of this estimator. + +// The exact version works for any value of K, but is quite slow. + +// The much faster approximate version works for K values that are powers of two +// ranging from 2^4 to 2^32. + +// At a high-level, this approximation can be described as using an +// exponential approximation when C > K * (5.6 or 5.7), while smaller +// values of C are handled by a degree-19 polynomial approximation of +// a pre-conditioned version of the true ICON mapping from C to N_hat. + +// This file also provides a validation procedure that compares its approximate +// and exact implementations of the FM85 ICON estimator. + +static const int ICON_MIN_LOG_K = 4; +static const int ICON_MAX_LOG_K = 26; +static const int ICON_POLYNOMIAL_DEGREE = 19; +static const int ICON_POLYNOMIAL_NUM_COEFFICIENTS = 1 + ICON_POLYNOMIAL_DEGREE; +static const int ICON_TABLE_SIZE = ICON_POLYNOMIAL_NUM_COEFFICIENTS * (1 + (ICON_MAX_LOG_K - ICON_MIN_LOG_K)); + +static const double ICON_POLYNOMIAL_COEFFICIENTS[ICON_TABLE_SIZE] = { + + // log K = 4 + 0.9895027971889700513, 0.3319496644645180128, 0.1242818722715769986, -0.03324149686026930256, -0.2985637298081619817, + 1.366555923595830002, -4.705499366260569971, 11.61506432505530029, -21.11254986175579873, 28.89421695078809904, + -30.1383659011730991, 24.11946778830730054, -14.83391445199539938, 6.983088767267210173, -2.48964120264876998, + 0.6593243603602499947, -0.125493534558034997, 0.01620971672896159843, -0.001271267679036929953, 4.567178653294529745e-05, + + // log K = 5 + 0.9947713741300230339, 0.3326559581620939787, 0.1250050661634889981, -0.04130073804472530336, -0.2584095537451129854, + 1.218050389433120051, -4.319106696095399656, 10.87175052045090062, -20.0184979022142997, 27.63210188163320069, + -28.97950009664030091, 23.26740804691930009, -14.33375703270860058, 6.751281271241110105, -2.406363094133439962, + 0.6367414734718820357, -0.1210468076141379967, 0.01561196698118279963, -0.001222335432128580056, 4.383502970318410206e-05, + + // log K = 6 + 0.9973904854982870161, 0.3330148852217920119, 0.125251536589509993, -0.04434075124043219962, -0.2436238890691720116, + 1.163293254754570016, -4.177758779777369647, 10.60301981340099964, -19.6274507428828997, 27.18420839597660077, + -28.56827214174580121, 22.96268674086600114, -14.15234202220280046, 6.665700662642549901, -2.375043356720739851, + 0.6280993991240929608, -0.119319019358031006, 0.01537674055733759954, -0.001202881695730769916, 4.309894633186929849e-05, + + // log K = 7 + 0.9986963310058679655, 0.3331956705633329907, 0.125337696770523005, -0.04546817338088020299, -0.2386752211125199863, + 1.145927328111949972, -4.135694445582720036, 10.52805060502839929, -19.52408322548339825, 27.06921653903929936, + -28.46207532143190022, 22.88083524357429965, -14.10057147392659971, 6.63958754983273991, -2.364865219283200037, + 0.6251341806425250169, -0.1186991327450530043, 0.0152892726403408008, -0.001195439764873199896, 4.281098416794090072e-05, + + // log K = 8 + 0.999348600452531044, 0.3332480372393080148, 0.126666900963325002, -0.06495714694254159371, -0.08376282050638980681, + 0.3760158094643630267, -1.568204791601850001, 4.483117719555970382, -9.119180124379150598, 13.65799293358900002, + -15.3100211234349004, 12.97546344654869976, -8.351661538536939489, 4.075022612435580172, -1.49387015887069996, + 0.4040976870253379927, -0.07813232681879349328, 0.01020545649538820085, -0.0008063279210812720381, 2.909334976414100078e-05, + + // log K = 9 + 0.9996743787297059924, 0.3332925779481850093, 0.1267124599259649986, -0.06550452970936600228, -0.08191738117533520214, + 0.3773034458363569987, -1.604679509609959975, 4.636761898691969641, -9.487348609558699408, 14.25164235443030059, + -15.99674955529870068, 13.56353219046370029, -8.730194904342459594, 4.259010067932120336, -1.56106689792022002, + 0.4222540912786589828, -0.08165296504921559784, 0.01066878484925220041, -0.0008433887618256910015, 3.045339724886519912e-05, + + // log K = 10 + 0.999837191783945034, 0.3333142252339619804, 0.1267759538087240012, -0.06631005632753710077, -0.07692759158286699428, + 0.3568943956395980166, -1.546598721379510044, 4.51595019978557044, -9.298431968763770428, 14.02586858080080034, + -15.78858959520439953, 13.41484931677589998, -8.647958125130809748, 4.22398017468472009, -1.549708891200570093, + 0.419507410264540026, -0.08117411611046250475, 0.01061202286184199928, -0.000839300527596772007, 3.03185874520205985e-05, + + // log K = 11 + 0.9999186020796150265, 0.3333249054574359826, 0.126791713589799987, -0.06662487271699729652, -0.07335552427910230211, + 0.3316370184815959909, -1.434143797561290068, 4.180260309967409604, -8.593906870708760692, 12.95088874800289958, + -14.56876092520539956, 12.37074367531410068, -7.969152075707960137, 3.888774396648960074, -1.424923326506990051, + 0.385084561785229984, -0.07435541911616409816, 0.009695363567476529554, -0.0007644375960047160388, 2.75156194717188011e-05, + + // log K = 12 + 0.9999592955649559967, 0.3333310560725140093, 0.1267379744020450116, -0.06524495415766619344, -0.08854031542298740343, + 0.4244320628874230228, -1.794077789033230008, 5.133875262768450298, -10.40149374917120007, 15.47808115629240078, + -17.2272296137545986, 14.5002173676463002, -9.274819801602760094, 4.500782540026570189, -1.642359389030050076, + 0.442596113445525019, -0.0853226219238850947, 0.01111969379054169975, -0.0008771614088006969611, 3.161668519459719752e-05, + + // log K = 13 + 0.9999796468102559732, 0.3333336602394039727, 0.126728089053198989, -0.06503798598282370391, -0.09050261023823169548, + 0.4350609244189960201, -1.831274835815670077, 5.223387516985289913, -10.55574395269979959, 15.67359470222429962, + -17.41263416341029924, 14.63297400889229927, -9.346752431221359458, 4.530124905188380069, -1.651245566462089975, + 0.444542549250713015, -0.08561720963336499901, 0.01114805146185449992, -0.0008786251203363140043, 3.16416341644572998e-05, + + // log K = 14 + 0.9999898187060970445, 0.3333362579300819806, 0.1266984078369459976, -0.06464561179765909715, -0.09343280886228019777, + 0.4490702549264070087, -1.878087608052450008, 5.338004322057390283, -10.76690603590630069, 15.97069195083200022, + -17.73440379943459888, 14.90212518309260048, -9.520506013770420495, 4.616238931978830173, -1.68364817877918993, + 0.4536194960681350086, -0.087448605434800597, 0.01139929991331390009, -0.0008995891451622229631, 3.244407259782900338e-05, + + // log K = 15 + 0.9999949072549390028, 0.3333376334705290267, 0.126665364358402005, -0.06411790034705669439, -0.09776009134670660128, + 0.4704691112248470253, -1.948021675295769972, 5.497760972696490001, -11.03165645315390009, 16.29703330781000048, + -18.03851029448010124, 15.11836776139680083, -9.638205179917429533, 4.665122328753120051, -1.698980686525759953, + 0.4571799506245269873, -0.08804011353783609828, 0.01146553155965330043, -0.0009040455800659569869, 3.257931866957050274e-05, + + // log K = 16 + 0.9999974544793589493, 0.3333381337614599871, 0.1266524862971120102, -0.06391676499117690535, -0.09929616211306059592, + 0.4771390820378790254, -1.965762451227349938, 5.526802350376460282, -11.05703067024660058, 16.29535848023060041, + -18.00114005075790047, 15.06214012231560062, -9.58874727382628933, 4.63537541652793017, -1.686222848555620102, + 0.4532602373715179933, -0.08719448925964939923, 0.01134365425717459921, -0.0008934965241274289835, 3.216436244471380105e-05, + + // log K = 17 + 0.9999987278278800185, 0.3333383411464330148, 0.126642761751724009, -0.06371042959073920653, -0.1013564516034080043, + 0.4891311195679299839, -2.010971712051409899, 5.644390807952309963, -11.27697253921500042, 16.59957157207080058, + -18.31808338317799922, 15.31363518393730061, -9.741451446816620674, 4.706207545519429658, -1.711102469010010063, + 0.4597587341089349744, -0.08841670767182820134, 0.01149999225097850068, -0.0009056651366963050422, 3.259910736274500059e-05, + + // log K = 18 + 0.9999993637727100371, 0.3333385511608860097, 0.1266341580529160016, -0.06353272828164230335, -0.103139962850642003, + 0.4996216017206500104, -2.05099128585287982, 5.749874086531799655, -11.47727638570349917, 16.88141587810320132, + -18.61744656177490143, 15.55634230427719977, -9.892350736128680211, 4.778033520984200422, -1.737045483861280104, + 0.4667410882683730167, -0.08977256212421590165, 0.01167940146667079994, -0.0009201381242396030127, 3.313600701586759867e-05, + + // log K = 19 + 0.9999996805376010212, 0.3333372324328989778, 0.1267104737214659882, -0.06504749929326139601, -0.0882341962464350954, + 0.4131871162041140244, -1.725190703567099915, 4.900817515593920426, -9.883452720776510603, 14.6657081190816001, + -16.29398295135089825, 13.69805011761319946, -8.753475239465899449, 4.244072374564439976, -1.547202527706629915, + 0.4164770109614310267, -0.08017596922092029565, 0.01043146101701039954, -0.00082124200571200305, 2.953319493719429935e-05, + + // log K = 20 + 0.9999998390037539986, 0.3333365859956040067, 0.1267460211029839967, -0.06569456024647769843, -0.0823070353477164951, + 0.3810826463303410017, -1.611983580241109992, 4.624520077758210057, -9.397308335633589138, 14.03184981378050011, + -15.6703191315401007, 13.22992718704790072, -8.484216393184780713, 4.125607133488029987, -1.507690650697159906, + 0.4066678517577320129, -0.07842110121777939868, 0.01021780862225150042, -0.0008054065857047439754, 2.899431830426989844e-05, + + // log K = 21 + 0.9999999207001479817, 0.3333384953015239849, 0.1266331480396669928, -0.06345750166298599892, -0.1042341210992499961, + 0.5077112908497130039, -2.087398133609810191, 5.858842546192500222, -11.70620319777190055, 17.23103975433669888, + -19.01462552846669851, 15.89674059836560005, -10.11395134034419918, 4.88760796465891989, -1.777886770904629987, + 0.4780200178339499839, -0.09200895321782050218, 0.01198029553244219989, -0.0009447283875782100165, 3.405716775824710232e-05, + + // log K = 22 + 0.9999999606908690497, 0.3333383929524300071, 0.1266456445096819927, -0.06373504294081690225, -0.1012834291081849969, + 0.4893810690172959998, -2.01391428223606983, 5.656430437473649597, -11.3067201537791, 16.64980594135310099, + -18.3792355790383013, 15.36879753115040081, -9.778831246425049528, 4.725308061988969577, -1.718423596500280093, + 0.4618308177809870019, -0.08883675060799739454, 0.01155766944804260087, -0.0009104695617243750358, 3.278237729674439666e-05, + + // log K = 23 + 0.9999999794683379628, 0.3333386441751680085, 0.1266463995182049995, -0.06376031920455070556, -0.1010799540803130059, + 0.488540137426137, -2.012048323537570127, 5.654949475342659682, -11.31023240892979942, 16.66334675284959843, + -18.40241452866079896, 15.39443572867130072, -9.798844412838670692, 4.736683907539640082, -1.723168363744929987, + 0.463270349018644001, -0.08914619066708899531, 0.01160235936257320022, -0.0009143600818183229709, 3.293669304679140117e-05, + + // log K = 24 + 0.9999999911469820146, 0.3333376076934529975, 0.1266944349940530012, -0.06470524278387919381, -0.09189342220283110152, + 0.4359182372694809793, -1.815980282951169977, 5.149474056470340066, -10.37086570678100017, 15.36962686758569951, + -17.05756384717849983, 14.32755177515199918, -9.149944050025640152, 4.434601894497260055, -1.616478926806520056, + 0.4351979157055039793, -0.08381768225272340223, 0.01091321820476520016, -0.0008600264403629039739, 3.09667800347144002e-05, + + // log K = 25 + 0.9999999968592140354, 0.3333379164881000167, 0.1266782495827009913, -0.06434163088961859789, -0.09575258124988890451, + 0.4597843575354370049, -1.911374431241559924, 5.411856661251520428, -10.88850084646090011, 16.12298941380269923, + -17.88172178487259956, 15.01301780636859995, -9.585542896142529301, 4.645811872761620442, -1.693952293156189892, + 0.4563143308861309921, -0.08795976148455289523, 0.01146560428011200033, -0.0009048442931930629528, 3.26358391497329992e-05, + + // log K = 26 + 0.9999999970700530483, 0.333338329556315982, 0.126644753076394001, -0.06372365346512399997, -0.1012760856945769949, + 0.4886852278576360176, -2.009005418394389952, 5.638119224137019714, -11.26276715335160006, 16.57640024218650154, + -18.29035093605569884, 15.28892246224570073, -9.724916375991760731, 4.6978877652334603, -1.707974125916829955, + 0.4588937864564729963, -0.08824617586088029375, 0.01147732114826570046, -0.00090384524860747295, 3.253252703695579795e-05, + +#ifdef LARGER_K_VALUES + // log K = 27 + 1.000000000639100106, 0.3333378987508219815, 0.126670943746902992, -0.06418811974745139426, -0.0972951198506895043, + 0.4687977077401049852, -1.945290489888900076, 5.499494964974400268, -11.05078190574979935, 16.3446428009706004, + -18.10936908931320133, 15.19089294103859977, -9.691829972777059155, 4.694320543263319934, -1.710719212277360013, + 0.4606257962161550146, -0.08875858006645380438, 0.01156634964444109952, -0.0009125838337464230437, 3.290907977404550287e-05, + + // log K = 28 + 0.9999999993590269476, 0.3333385660745579737, 0.1266394134278630013, -0.0636305053404186971, -0.1022354305220320031, + 0.4945787360853979853, -2.032468917547570086, 5.702461924065530319, -11.38943406618639997, 16.76052144140630062, + -18.49169753114890113, 15.4564578116809006, -9.831507534599410292, 4.749667961030789698, -1.72701519749717991, + 0.4640997252013580043, -0.08927103511252110213, 0.01161455495023329919, -0.000915030036039231982, 3.295110296010450275e-05, + + // log K = 29 + 0.9999999998441060356, 0.3333383341194189886, 0.1266687338487519909, -0.06416245828383730643, -0.09764561286937140094, + 0.4715274747139350242, -1.958172229464169911, 5.539587632966780362, -11.13784217611559946, 16.48149277721759987, + -18.26888916646990069, 15.33085193018819936, -9.78493991484172021, 4.741302923579859829, -1.728568959451310061, + 0.4656457646521020011, -0.08977142058582450457, 0.01170492245846839995, -0.0009240931538567209464, 3.334703207098030245e-05, + + // log K = 30 + 0.9999999992599339915, 0.3333384538468979752, 0.1266452025739940035, -0.06374775920488300052, -0.1009917742909720029, + 0.4867931642504759737, -2.000981224888669807, 5.614968747087539569, -11.21527907219130071, 16.50500949673639894, + -18.21007853829650003, 15.22056128176249956, -9.680565515478869898, 4.675983737170599674, -1.69980511941418011, + 0.4566332138743600111, -0.08779650251621799739, 0.01141656381272189956, -0.0008988545845624889468, 3.234448025291899689e-05, + + // log K = 31 + 0.9999999973204000137, 0.333337762450663988, 0.1266965469104399944, -0.06475154253624139378, -0.09133098208494490333, + 0.4320356889637699815, -1.799236887220760028, 5.100971076171499696, -10.27175516606700079, 15.22198757843720074, + -16.89368636262300072, 14.19016571851859965, -9.062390133299189188, 4.39220025249522994, -1.600994848692480099, + 0.4310075283759189912, -0.08300339267288289746, 0.01080584419810979961, -0.0008514267355136160122, 3.065110087496039805e-05, + + // log K = 32 + 0.9999999987706390536, 0.3333387038350890119, 0.1266354589419070031, -0.06355195838981600454, -0.102952771506954005, + 0.4983589546197609854, -2.045281215270029929, 5.732181222451769642, -11.43849817800069957, 16.81961198331340057, + -18.54433120118400069, 15.49126422718470053, -9.84846998787154071, 4.755615082534379923, -1.728430514092559989, + 0.4642927653670489985, -0.08927380119154580684, 0.01161055316485629964, -0.0009143724787632470305, 3.291492066818770055e-05, + +#endif +}; + +static double evaluate_polynomial(const double* coefficients, int start, int num, double x) { + const int final = start + num - 1; + double total = coefficients[final]; + for (int j = final - 1; j >= start; j--) { + total *= x; + total += coefficients[j]; + } + return total; +} + +static double icon_exponential_approximation(double k, double c) { + return (0.7940236163830469 * k * pow(2.0, c / k)); +} + +static double compute_icon_estimate(uint8_t lg_k, uint64_t c) { + if (lg_k < ICON_MIN_LOG_K || lg_k > ICON_MAX_LOG_K) throw std::out_of_range("lg_k out of range"); + if (c < 2) return ((c == 0) ? 0.0 : 1.0); + const size_t k = 1 << lg_k; + const double double_k = k; + const double double_c = c; + // Differing thresholds ensure that the approximated estimator is monotonically increasing. + const double threshold_factor = ((lg_k < 14) ? 5.7 : 5.6); + if (double_c > (threshold_factor * double_k)) return icon_exponential_approximation(double_k, double_c); + const double factor = evaluate_polynomial( + ICON_POLYNOMIAL_COEFFICIENTS, + ICON_POLYNOMIAL_NUM_COEFFICIENTS * (lg_k - ICON_MIN_LOG_K), + ICON_POLYNOMIAL_NUM_COEFFICIENTS, + // The somewhat arbitrary constant 2.0 is baked into the table ICON_POLYNOMIAL_COEFFICIENTS + double_c / (2.0 * double_k) + ); + const double ratio = double_c / double_k; + // The somewhat arbitrary constant 66.774757 is baked into the table ICON_POLYNOMIAL_COEFFICIENTS + const double term = 1.0 + (ratio * ratio * ratio / 66.774757); + const double result = double_c * factor * term; + if (result >= double_c) return result; + else return double_c; +} + +} /* namespace datasketches */ + +#endif diff --git a/3rd/datasketches/datasketches/cpc/inv_pow_2_tab.hpp b/3rd/datasketches/datasketches/cpc/inv_pow_2_tab.hpp new file mode 100644 index 000000000..c904bfed7 --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/inv_pow_2_tab.hpp @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef INV_POW_2_TAB_HPP_ +#define INV_POW_2_TAB_HPP_ + +#include + +namespace datasketches { + +// the table was created by the following procedure: + +//void fill_inverse_power_of_2_table() { +// for (int i = 0; i < 256; i++) { +// inverse_powers_of_2_table[i] = pow(2.0, (-1.0 * ((double) i))); +// printf("%.17g", inverse_powers_of_2_table[i]); +// if (i != 255) printf(", "); +// if ((i + 1) % 4 == 0) printf("\n"); +// } +//} + +static const double INVERSE_POWERS_OF_2[256] = { + 1, 0.5, 0.25, 0.125, + 0.0625, 0.03125, 0.015625, 0.0078125, + 0.00390625, 0.001953125, 0.0009765625, 0.00048828125, + 0.000244140625, 0.0001220703125, 6.103515625e-05, 3.0517578125e-05, + 1.52587890625e-05, 7.62939453125e-06, 3.814697265625e-06, 1.9073486328125e-06, + 9.5367431640625e-07, 4.76837158203125e-07, 2.384185791015625e-07, 1.1920928955078125e-07, + 5.9604644775390625e-08, 2.9802322387695312e-08, 1.4901161193847656e-08, 7.4505805969238281e-09, + 3.7252902984619141e-09, 1.862645149230957e-09, 9.3132257461547852e-10, 4.6566128730773926e-10, + 2.3283064365386963e-10, 1.1641532182693481e-10, 5.8207660913467407e-11, 2.9103830456733704e-11, + 1.4551915228366852e-11, 7.2759576141834259e-12, 3.637978807091713e-12, 1.8189894035458565e-12, + 9.0949470177292824e-13, 4.5474735088646412e-13, 2.2737367544323206e-13, 1.1368683772161603e-13, + 5.6843418860808015e-14, 2.8421709430404007e-14, 1.4210854715202004e-14, 7.1054273576010019e-15, + 3.5527136788005009e-15, 1.7763568394002505e-15, 8.8817841970012523e-16, 4.4408920985006262e-16, + 2.2204460492503131e-16, 1.1102230246251565e-16, 5.5511151231257827e-17, 2.7755575615628914e-17, + 1.3877787807814457e-17, 6.9388939039072284e-18, 3.4694469519536142e-18, 1.7347234759768071e-18, + 8.6736173798840355e-19, 4.3368086899420177e-19, 2.1684043449710089e-19, 1.0842021724855044e-19, + 5.4210108624275222e-20, 2.7105054312137611e-20, 1.3552527156068805e-20, 6.7762635780344027e-21, + 3.3881317890172014e-21, 1.6940658945086007e-21, 8.4703294725430034e-22, 4.2351647362715017e-22, + 2.1175823681357508e-22, 1.0587911840678754e-22, 5.2939559203393771e-23, 2.6469779601696886e-23, + 1.3234889800848443e-23, 6.6174449004242214e-24, 3.3087224502121107e-24, 1.6543612251060553e-24, + 8.2718061255302767e-25, 4.1359030627651384e-25, 2.0679515313825692e-25, 1.0339757656912846e-25, + 5.169878828456423e-26, 2.5849394142282115e-26, 1.2924697071141057e-26, 6.4623485355705287e-27, + 3.2311742677852644e-27, 1.6155871338926322e-27, 8.0779356694631609e-28, 4.0389678347315804e-28, + 2.0194839173657902e-28, 1.0097419586828951e-28, 5.0487097934144756e-29, 2.5243548967072378e-29, + 1.2621774483536189e-29, 6.3108872417680944e-30, 3.1554436208840472e-30, 1.5777218104420236e-30, + 7.8886090522101181e-31, 3.944304526105059e-31, 1.9721522630525295e-31, 9.8607613152626476e-32, + 4.9303806576313238e-32, 2.4651903288156619e-32, 1.2325951644078309e-32, 6.1629758220391547e-33, + 3.0814879110195774e-33, 1.5407439555097887e-33, 7.7037197775489434e-34, 3.8518598887744717e-34, + 1.9259299443872359e-34, 9.6296497219361793e-35, 4.8148248609680896e-35, 2.4074124304840448e-35, + 1.2037062152420224e-35, 6.018531076210112e-36, 3.009265538105056e-36, 1.504632769052528e-36, + 7.5231638452626401e-37, 3.76158192263132e-37, 1.88079096131566e-37, 9.4039548065783001e-38, + 4.70197740328915e-38, 2.350988701644575e-38, 1.1754943508222875e-38, 5.8774717541114375e-39, + 2.9387358770557188e-39, 1.4693679385278594e-39, 7.3468396926392969e-40, 3.6734198463196485e-40, + 1.8367099231598242e-40, 9.1835496157991212e-41, 4.5917748078995606e-41, 2.2958874039497803e-41, + 1.1479437019748901e-41, 5.7397185098744507e-42, 2.8698592549372254e-42, 1.4349296274686127e-42, + 7.1746481373430634e-43, 3.5873240686715317e-43, 1.7936620343357659e-43, 8.9683101716788293e-44, + 4.4841550858394146e-44, 2.2420775429197073e-44, 1.1210387714598537e-44, 5.6051938572992683e-45, + 2.8025969286496341e-45, 1.4012984643248171e-45, 7.0064923216240854e-46, 3.5032461608120427e-46, + 1.7516230804060213e-46, 8.7581154020301067e-47, 4.3790577010150533e-47, 2.1895288505075267e-47, + 1.0947644252537633e-47, 5.4738221262688167e-48, 2.7369110631344083e-48, 1.3684555315672042e-48, + 6.8422776578360209e-49, 3.4211388289180104e-49, 1.7105694144590052e-49, 8.5528470722950261e-50, + 4.276423536147513e-50, 2.1382117680737565e-50, 1.0691058840368783e-50, 5.3455294201843913e-51, + 2.6727647100921956e-51, 1.3363823550460978e-51, 6.6819117752304891e-52, 3.3409558876152446e-52, + 1.6704779438076223e-52, 8.3523897190381114e-53, 4.1761948595190557e-53, 2.0880974297595278e-53, + 1.0440487148797639e-53, 5.2202435743988196e-54, 2.6101217871994098e-54, 1.3050608935997049e-54, + 6.5253044679985245e-55, 3.2626522339992623e-55, 1.6313261169996311e-55, 8.1566305849981557e-56, + 4.0783152924990778e-56, 2.0391576462495389e-56, 1.0195788231247695e-56, 5.0978941156238473e-57, + 2.5489470578119236e-57, 1.2744735289059618e-57, 6.3723676445298091e-58, 3.1861838222649046e-58, + 1.5930919111324523e-58, 7.9654595556622614e-59, 3.9827297778311307e-59, 1.9913648889155653e-59, + 9.9568244445778267e-60, 4.9784122222889134e-60, 2.4892061111444567e-60, 1.2446030555722283e-60, + 6.2230152778611417e-61, 3.1115076389305709e-61, 1.5557538194652854e-61, 7.7787690973264271e-62, + 3.8893845486632136e-62, 1.9446922743316068e-62, 9.7234613716580339e-63, 4.861730685829017e-63, + 2.4308653429145085e-63, 1.2154326714572542e-63, 6.0771633572862712e-64, 3.0385816786431356e-64, + 1.5192908393215678e-64, 7.596454196607839e-65, 3.7982270983039195e-65, 1.8991135491519597e-65, + 9.4955677457597987e-66, 4.7477838728798994e-66, 2.3738919364399497e-66, 1.1869459682199748e-66, + 5.9347298410998742e-67, 2.9673649205499371e-67, 1.4836824602749686e-67, 7.4184123013748428e-68, + 3.7092061506874214e-68, 1.8546030753437107e-68, 9.2730153767185535e-69, 4.6365076883592767e-69, + 2.3182538441796384e-69, 1.1591269220898192e-69, 5.7956346104490959e-70, 2.897817305224548e-70, + 1.448908652612274e-70, 7.2445432630613699e-71, 3.6222716315306849e-71, 1.8111358157653425e-71, + 9.0556790788267124e-72, 4.5278395394133562e-72, 2.2639197697066781e-72, 1.131959884853339e-72, + 5.6597994242666952e-73, 2.8298997121333476e-73, 1.4149498560666738e-73, 7.074749280333369e-74, + 3.5373746401666845e-74, 1.7686873200833423e-74, 8.8434366004167113e-75, 4.4217183002083556e-75, + 2.2108591501041778e-75, 1.1054295750520889e-75, 5.5271478752604446e-76, 2.7635739376302223e-76, + 1.3817869688151111e-76, 6.9089348440755557e-77, 3.4544674220377779e-77, 1.7272337110188889e-77 +}; + +} /* namespace datasketches */ + +#endif diff --git a/3rd/datasketches/datasketches/cpc/kxp_byte_lookup.hpp b/3rd/datasketches/datasketches/cpc/kxp_byte_lookup.hpp new file mode 100644 index 000000000..4df785d48 --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/kxp_byte_lookup.hpp @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef KXP_BYTE_LOOKUP_HPP_ +#define KXP_BYTE_LOOKUP_HPP_ + +namespace datasketches { + +// the table was created by the following procedure: + +//void fill_kxp_byte_table() { +// for (int byte = 0; byte < 256; byte++) { +// double sum = 0.0; +// for (int col = 0; col < 8; col++) { +// const uint8_t bit = (byte >> col) & 1; +// if (bit == 0) { // note the inverted logic +// sum += INVERSE_POWERS_OF_2[col + 1]; // note the "+1" +// } +// } +// kxp_byte_table[byte] = sum; +// +// printf("%.17g", kxp_byte_table[byte]); +// if (byte != 255) printf(", "); +// if ((byte + 1) % 8 == 0) printf("\n"); +// } +//} + +static const double KXP_BYTE_TABLE[256] = { + 0.99609375, 0.49609375, 0.74609375, 0.24609375, 0.87109375, 0.37109375, 0.62109375, 0.12109375, + 0.93359375, 0.43359375, 0.68359375, 0.18359375, 0.80859375, 0.30859375, 0.55859375, 0.05859375, + 0.96484375, 0.46484375, 0.71484375, 0.21484375, 0.83984375, 0.33984375, 0.58984375, 0.08984375, + 0.90234375, 0.40234375, 0.65234375, 0.15234375, 0.77734375, 0.27734375, 0.52734375, 0.02734375, + 0.98046875, 0.48046875, 0.73046875, 0.23046875, 0.85546875, 0.35546875, 0.60546875, 0.10546875, + 0.91796875, 0.41796875, 0.66796875, 0.16796875, 0.79296875, 0.29296875, 0.54296875, 0.04296875, + 0.94921875, 0.44921875, 0.69921875, 0.19921875, 0.82421875, 0.32421875, 0.57421875, 0.07421875, + 0.88671875, 0.38671875, 0.63671875, 0.13671875, 0.76171875, 0.26171875, 0.51171875, 0.01171875, + 0.98828125, 0.48828125, 0.73828125, 0.23828125, 0.86328125, 0.36328125, 0.61328125, 0.11328125, + 0.92578125, 0.42578125, 0.67578125, 0.17578125, 0.80078125, 0.30078125, 0.55078125, 0.05078125, + 0.95703125, 0.45703125, 0.70703125, 0.20703125, 0.83203125, 0.33203125, 0.58203125, 0.08203125, + 0.89453125, 0.39453125, 0.64453125, 0.14453125, 0.76953125, 0.26953125, 0.51953125, 0.01953125, + 0.97265625, 0.47265625, 0.72265625, 0.22265625, 0.84765625, 0.34765625, 0.59765625, 0.09765625, + 0.91015625, 0.41015625, 0.66015625, 0.16015625, 0.78515625, 0.28515625, 0.53515625, 0.03515625, + 0.94140625, 0.44140625, 0.69140625, 0.19140625, 0.81640625, 0.31640625, 0.56640625, 0.06640625, + 0.87890625, 0.37890625, 0.62890625, 0.12890625, 0.75390625, 0.25390625, 0.50390625, 0.00390625, + 0.9921875, 0.4921875, 0.7421875, 0.2421875, 0.8671875, 0.3671875, 0.6171875, 0.1171875, + 0.9296875, 0.4296875, 0.6796875, 0.1796875, 0.8046875, 0.3046875, 0.5546875, 0.0546875, + 0.9609375, 0.4609375, 0.7109375, 0.2109375, 0.8359375, 0.3359375, 0.5859375, 0.0859375, + 0.8984375, 0.3984375, 0.6484375, 0.1484375, 0.7734375, 0.2734375, 0.5234375, 0.0234375, + 0.9765625, 0.4765625, 0.7265625, 0.2265625, 0.8515625, 0.3515625, 0.6015625, 0.1015625, + 0.9140625, 0.4140625, 0.6640625, 0.1640625, 0.7890625, 0.2890625, 0.5390625, 0.0390625, + 0.9453125, 0.4453125, 0.6953125, 0.1953125, 0.8203125, 0.3203125, 0.5703125, 0.0703125, + 0.8828125, 0.3828125, 0.6328125, 0.1328125, 0.7578125, 0.2578125, 0.5078125, 0.0078125, + 0.984375, 0.484375, 0.734375, 0.234375, 0.859375, 0.359375, 0.609375, 0.109375, + 0.921875, 0.421875, 0.671875, 0.171875, 0.796875, 0.296875, 0.546875, 0.046875, + 0.953125, 0.453125, 0.703125, 0.203125, 0.828125, 0.328125, 0.578125, 0.078125, + 0.890625, 0.390625, 0.640625, 0.140625, 0.765625, 0.265625, 0.515625, 0.015625, + 0.96875, 0.46875, 0.71875, 0.21875, 0.84375, 0.34375, 0.59375, 0.09375, + 0.90625, 0.40625, 0.65625, 0.15625, 0.78125, 0.28125, 0.53125, 0.03125, + 0.9375, 0.4375, 0.6875, 0.1875, 0.8125, 0.3125, 0.5625, 0.0625, + 0.875, 0.375, 0.625, 0.125, 0.75, 0.25, 0.5, 0 +}; + +} /* namespace datasketches */ + +#endif diff --git a/3rd/datasketches/datasketches/cpc/u32_table.hpp b/3rd/datasketches/datasketches/cpc/u32_table.hpp new file mode 100644 index 000000000..2316fc177 --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/u32_table.hpp @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// author Kevin Lang, Oath Research + +#ifndef U32_TABLE_HPP_ +#define U32_TABLE_HPP_ + +// This is a highly specialized hash table that was designed +// to be a part of the library's CPC sketch implementation + +#include "cpc_common.hpp" + +namespace datasketches { + +static const uint64_t U32_TABLE_UPSIZE_NUMER = 3LL; +static const uint64_t U32_TABLE_UPSIZE_DENOM = 4LL; + +static const uint64_t U32_TABLE_DOWNSIZE_NUMER = 1LL; +static const uint64_t U32_TABLE_DOWNSIZE_DENOM = 4LL; + +template +class u32_table { +public: + + u32_table(); + u32_table(uint8_t lg_size, uint8_t num_valid_bits); + + inline size_t get_num_items() const; + inline const uint32_t* get_slots() const; + inline uint8_t get_lg_size() const; + inline void clear(); + + // returns true iff the item was new and was therefore added to the table + inline bool maybe_insert(uint32_t item); + // returns true iff the item was present and was therefore removed from the table + inline bool maybe_delete(uint32_t item); + + static u32_table make_from_pairs(const uint32_t* pairs, size_t num_pairs, uint8_t lg_k); + + vector_u32 unwrapping_get_items() const; + + static void merge( + const uint32_t* arr_a, size_t start_a, size_t length_a, // input + const uint32_t* arr_b, size_t start_b, size_t length_b, // input + uint32_t* arr_c, size_t start_c // output + ); + + static void introspective_insertion_sort(uint32_t* a, size_t l, size_t r); + static void knuth_shell_sort3(uint32_t* a, size_t l, size_t r); + +private: + + uint8_t lg_size; // log2 of number of slots + uint8_t num_valid_bits; + size_t num_items; + vector_u32 slots; + + inline size_t lookup(uint32_t item) const; + inline void must_insert(uint32_t item); + inline void rebuild(uint8_t new_lg_size); +}; + +} /* namespace datasketches */ + +#include "u32_table_impl.hpp" + +#endif diff --git a/3rd/datasketches/datasketches/cpc/u32_table_impl.hpp b/3rd/datasketches/datasketches/cpc/u32_table_impl.hpp new file mode 100644 index 000000000..3b49459f7 --- /dev/null +++ b/3rd/datasketches/datasketches/cpc/u32_table_impl.hpp @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// author Kevin Lang, Oath Research + +#ifndef U32_TABLE_IMPL_HPP_ +#define U32_TABLE_IMPL_HPP_ + +#include +#include +#include + +namespace datasketches { + +template +u32_table::u32_table(): +lg_size(0), +num_valid_bits(0), +num_items(0), +slots() +{} + +template +u32_table::u32_table(uint8_t lg_size, uint8_t num_valid_bits): +lg_size(lg_size), +num_valid_bits(num_valid_bits), +num_items(0), +slots(1 << lg_size, UINT32_MAX) +{ + if (lg_size < 2) throw std::invalid_argument("lg_size must be >= 2"); + if (num_valid_bits < 1 or num_valid_bits > 32) throw std::invalid_argument("num_valid_bits must be between 1 and 32"); +} + +template +size_t u32_table::get_num_items() const { + return num_items; +} + +template +const uint32_t* u32_table::get_slots() const { + return slots.data(); +} + +template +uint8_t u32_table::get_lg_size() const { + return lg_size; +} + +template +void u32_table::clear() { + std::fill(slots.begin(), slots.end(), UINT32_MAX); + num_items = 0; +} + +template +bool u32_table::maybe_insert(uint32_t item) { + const size_t index = lookup(item); + if (slots[index] == item) return false; + if (slots[index] != UINT32_MAX) throw std::logic_error("could not insert"); + slots[index] = item; + num_items++; + if (U32_TABLE_UPSIZE_DENOM * num_items > U32_TABLE_UPSIZE_NUMER * (1 << lg_size)) { + rebuild(lg_size + 1); + } + return true; +} + +template +bool u32_table::maybe_delete(uint32_t item) { + const size_t index = lookup(item); + if (slots[index] == UINT32_MAX) return false; + if (slots[index] != item) throw std::logic_error("item does not exist"); + if (num_items == 0) throw std::logic_error("delete error"); + // delete the item + slots[index] = UINT32_MAX; + num_items--; + + // re-insert all items between the freed slot and the next empty slot + const size_t mask = (1 << lg_size) - 1; + size_t probe = (index + 1) & mask; + uint32_t fetched = slots[probe]; + while (fetched != UINT32_MAX) { + slots[probe] = UINT32_MAX; + must_insert(fetched); + probe = (probe + 1) & mask; + fetched = slots[probe]; + } + // shrink if necessary + if (U32_TABLE_DOWNSIZE_DENOM * num_items < U32_TABLE_DOWNSIZE_NUMER * (1 << lg_size) and lg_size > 2) { + rebuild(lg_size - 1); + } + return true; +} + +// this one is specifically tailored to be a part of fm85 decompression scheme +template +u32_table u32_table::make_from_pairs(const uint32_t* pairs, size_t num_pairs, uint8_t lg_k) { + uint8_t lg_num_slots = 2; + while (U32_TABLE_UPSIZE_DENOM * num_pairs > U32_TABLE_UPSIZE_NUMER * (1 << lg_num_slots)) lg_num_slots++; + u32_table table(lg_num_slots, 6 + lg_k); + // Note: there is a possible "snowplow effect" here because the caller is passing in a sorted pairs array + // However, we are starting out with the correct final table size, so the problem might not occur + for (size_t i = 0; i < num_pairs; i++) { + table.must_insert(pairs[i]); + } + table.num_items = num_pairs; + return table; +} + +template +size_t u32_table::lookup(uint32_t item) const { + const size_t size = 1 << lg_size; + const size_t mask = size - 1; + const uint8_t shift = num_valid_bits - lg_size; + size_t probe = item >> shift; + if (probe > mask) throw std::logic_error("probe out of range"); + while (slots[probe] != item and slots[probe] != UINT32_MAX) { + probe = (probe + 1) & mask; + } + return probe; +} + +// counts and resizing must be handled by the caller +template +void u32_table::must_insert(uint32_t item) { + const size_t index = lookup(item); + if (slots[index] == item) throw std::logic_error("item exists"); + if (slots[index] != UINT32_MAX) throw std::logic_error("could not insert"); + slots[index] = item; +} + +template +void u32_table::rebuild(uint8_t new_lg_size) { + if (new_lg_size < 2) throw std::logic_error("lg_size must be >= 2"); + const size_t old_size = 1 << lg_size; + const size_t new_size = 1 << new_lg_size; + if (new_size <= num_items) throw std::logic_error("new_size <= num_items"); + vector_u32 old_slots = std::move(slots); + slots = vector_u32(new_size, UINT32_MAX); + lg_size = new_lg_size; + for (size_t i = 0; i < old_size; i++) { + if (old_slots[i] != UINT32_MAX) { + must_insert(old_slots[i]); + } + } +} + +// While extracting the items from a linear probing hashtable, +// this will usually undo the wrap-around provided that the table +// isn't too full. Experiments suggest that for sufficiently large tables +// the load factor would have to be over 90 percent before this would fail frequently, +// and even then the subsequent sort would fix things up. +// The result is nearly sorted, so make sure to use an efficient sort for that case +template +vector_u32 u32_table::unwrapping_get_items() const { + if (num_items == 0) return vector_u32(); + const size_t table_size = 1 << lg_size; + vector_u32 result(num_items); + size_t i = 0; + size_t l = 0; + size_t r = num_items - 1; + + // special rules for the region before the first empty slot + uint32_t hi_bit = 1 << (num_valid_bits - 1); + while (i < table_size and slots[i] != UINT32_MAX) { + const uint32_t item = slots[i++]; + if (item & hi_bit) { result[r--] = item; } // this item was probably wrapped, so move to end + else { result[l++] = item; } + } + + // the rest of the table is processed normally + while (i < table_size) { + const uint32_t item = slots[i++]; + if (item != UINT32_MAX) result[l++] = item; + } + if (l != r + 1) throw std::logic_error("unwrapping error"); + return result; +} + +// This merge is safe to use in carefully designed overlapping scenarios. +template +void u32_table::merge( + const uint32_t* arr_a, size_t start_a, size_t length_a, // input + const uint32_t* arr_b, size_t start_b, size_t length_b, // input + uint32_t* arr_c, size_t start_c // output +) { + const size_t length_c = length_a + length_b; + const size_t lim_a = start_a + length_a; + const size_t lim_b = start_b + length_b; + const size_t lim_c = start_c + length_c; + size_t a = start_a; + size_t b = start_b; + size_t c = start_c; + for ( ; c < lim_c ; c++) { + if (b >= lim_b) { arr_c[c] = arr_a[a++]; } + else if (a >= lim_a) { arr_c[c] = arr_b[b++]; } + else if (arr_a[a] < arr_b[b]) { arr_c[c] = arr_a[a++]; } + else { arr_c[c] = arr_b[b++]; } + } + if (a != lim_a or b != lim_b) throw std::logic_error("merging error"); +} + +// In applications where the input array is already nearly sorted, +// insertion sort runs in linear time with a very small constant. +// This introspective version of insertion sort protects against +// the quadratic cost of sorting bad input arrays. +// It keeps track of how much work has been done, and if that exceeds a +// constant times the array length, it switches to a different sorting algorithm. + +template +void u32_table::introspective_insertion_sort(uint32_t* a, size_t l, size_t r) { // r points past the rightmost element + const size_t length = r - l; + const size_t cost_limit = 8 * length; + size_t cost = 0; + for (size_t i = l + 1; i < r; i++) { + size_t j = i; + uint32_t v = a[i]; + while (j >= l + 1 and v < a[j - 1]) { + a[j] = a[j - 1]; + j--; + } + a[j] = v; + cost += i - j; // distance moved is a measure of work + if (cost > cost_limit) { + knuth_shell_sort3(a, l, r); + return; + } + } +} + +template +void u32_table::knuth_shell_sort3(uint32_t* a, size_t l, size_t r) { + size_t h; + for (h = 1; h < (r - l) / 9; h = 3 * h + 1); + for ( ; h > 0; h /= 3) { + for (size_t i = l + h; i < r; i++) { + size_t j = i; + const uint32_t v = a[i]; + while (j >= l + h and v < a[j - h]) { + a[j] = a[j - h]; + j -= h; + } + a[j] = v; + } + } +} + +} /* namespace datasketches */ + +#endif diff --git a/3rd/datasketches/datasketches/fi/frequent_items_sketch.hpp b/3rd/datasketches/datasketches/fi/frequent_items_sketch.hpp new file mode 100644 index 000000000..52fb4376b --- /dev/null +++ b/3rd/datasketches/datasketches/fi/frequent_items_sketch.hpp @@ -0,0 +1,500 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef FREQUENT_ITEMS_SKETCH_HPP_ +#define FREQUENT_ITEMS_SKETCH_HPP_ + +#include +#include +#include +#include +#include +#include + +#include "reverse_purge_hash_map.hpp" +#include "serde.hpp" + +namespace datasketches { + +/* + * Based on Java implementation here: + * https://github.com/DataSketches/sketches-core/blob/master/src/main/java/com/yahoo/sketches/frequencies/ItemsSketch.java + * author Alexander Saydakov + */ + +enum frequent_items_error_type { NO_FALSE_POSITIVES, NO_FALSE_NEGATIVES }; + +// for serialization as raw bytes +template using AllocU8 = typename std::allocator_traits::template rebind_alloc; +template using vector_u8 = std::vector>; + +template, typename E = std::equal_to, typename S = serde, typename A = std::allocator> +class frequent_items_sketch { +public: + static const uint64_t USE_MAX_ERROR = 0; // used in get_frequent_items + + explicit frequent_items_sketch(uint8_t lg_max_map_size); + frequent_items_sketch(uint8_t lg_start_map_size, uint8_t lg_max_map_size); + class row; + void update(const T& item, uint64_t weight = 1); + void update(T&& item, uint64_t weight = 1); + void merge(const frequent_items_sketch& other); + bool is_empty() const; + uint32_t get_num_active_items() const; + uint64_t get_total_weight() const; + uint64_t get_estimate(const T& item) const; + uint64_t get_lower_bound(const T& item) const; + uint64_t get_upper_bound(const T& item) const; + uint64_t get_maximum_error() const; + double get_epsilon() const; + static double get_epsilon(uint8_t lg_max_map_size); + static double get_apriori_error(uint8_t lg_max_map_size, uint64_t estimated_total_weight); + typedef typename std::allocator_traits::template rebind_alloc AllocRow; + std::vector get_frequent_items(frequent_items_error_type err_type, uint64_t threshold = USE_MAX_ERROR) const; + size_t get_serialized_size_bytes() const; + void serialize(std::ostream& os) const; + typedef vector_u8 vector_bytes; // alias for users + vector_bytes serialize(unsigned header_size_bytes = 0) const; + static frequent_items_sketch deserialize(std::istream& is); + static frequent_items_sketch deserialize(const void* bytes, size_t size); + void to_stream(std::ostream& os, bool print_items = false) const; +private: + static const uint8_t LG_MIN_MAP_SIZE = 3; + static const uint8_t SERIAL_VERSION = 1; + static const uint8_t FAMILY_ID = 10; + static const uint8_t PREAMBLE_LONGS_EMPTY = 1; + static const uint8_t PREAMBLE_LONGS_NONEMPTY = 4; + static constexpr double EPSILON_FACTOR = 3.5; + enum flags { IS_EMPTY }; + uint64_t total_weight; + uint64_t offset; + reverse_purge_hash_map map; + static void check_preamble_longs(uint8_t preamble_longs, bool is_empty); + static void check_serial_version(uint8_t serial_version); + static void check_family_id(uint8_t family_id); + static void check_size(uint8_t lg_cur_size, uint8_t lg_max_size); +}; + +// clang++ seems to require this declaration for CMAKE_BUILD_TYPE='Debug" +template +const uint8_t frequent_items_sketch::LG_MIN_MAP_SIZE; + +template +frequent_items_sketch::frequent_items_sketch(uint8_t lg_max_map_size): +total_weight(0), +offset(0), +map(frequent_items_sketch::LG_MIN_MAP_SIZE, std::max(lg_max_map_size, frequent_items_sketch::LG_MIN_MAP_SIZE)) +{ +} + +template +frequent_items_sketch::frequent_items_sketch(uint8_t lg_start_map_size, uint8_t lg_max_map_size): +total_weight(0), +offset(0), +map(std::max(lg_start_map_size, frequent_items_sketch::LG_MIN_MAP_SIZE), std::max(lg_max_map_size, frequent_items_sketch::LG_MIN_MAP_SIZE)) +{ +} + +template +void frequent_items_sketch::update(const T& item, uint64_t weight) { + if (weight == 0) return; + total_weight += weight; + offset += map.adjust_or_insert(item, weight); +} + +template +void frequent_items_sketch::update(T&& item, uint64_t weight) { + if (weight == 0) return; + total_weight += weight; + offset += map.adjust_or_insert(std::move(item), weight); +} + +template +void frequent_items_sketch::merge(const frequent_items_sketch& other) { + if (other.is_empty()) return; + const uint64_t merged_total_weight = total_weight + other.get_total_weight(); // for correction at the end + for (auto &it: other.map) { + update(it.first, it.second); + } + offset += other.offset; + total_weight = merged_total_weight; +} + +template +bool frequent_items_sketch::is_empty() const { + return map.get_num_active() == 0; +} + +template +uint32_t frequent_items_sketch::get_num_active_items() const { + return map.get_num_active(); +} + +template +uint64_t frequent_items_sketch::get_total_weight() const { + return total_weight; +} + +template +uint64_t frequent_items_sketch::get_estimate(const T& item) const { + // if item is tracked estimate = weight + offset, otherwise 0 + const uint64_t weight = map.get(item); + if (weight > 0) return weight + offset; + return 0; +} + +template +uint64_t frequent_items_sketch::get_lower_bound(const T& item) const { + return map.get(item); +} + +template +uint64_t frequent_items_sketch::get_upper_bound(const T& item) const { + return map.get(item) + offset; +} + +template +uint64_t frequent_items_sketch::get_maximum_error() const { + return offset; +} + +template +double frequent_items_sketch::get_epsilon() const { + return EPSILON_FACTOR / (1 << map.get_lg_max_size()); +} + +template +double frequent_items_sketch::get_epsilon(uint8_t lg_max_map_size) { + return EPSILON_FACTOR / (1 << lg_max_map_size); +} + +template +double frequent_items_sketch::get_apriori_error(uint8_t lg_max_map_size, uint64_t estimated_total_weight) { + return get_epsilon(lg_max_map_size) * estimated_total_weight; +} + +template +class frequent_items_sketch::row { +public: + row(const T* item, uint64_t weight, uint64_t offset): + item(item), weight(weight), offset(offset) {} + const T& get_item() const { return *item; } + uint64_t get_estimate() const { return weight + offset; } + uint64_t get_lower_bound() const { return weight; } + uint64_t get_upper_bound() const { return weight + offset; } +private: + const T* item; + uint64_t weight; + uint64_t offset; +}; + +template +std::vector::row, typename frequent_items_sketch::AllocRow> +frequent_items_sketch::get_frequent_items(frequent_items_error_type err_type, uint64_t threshold) const { + if (threshold == USE_MAX_ERROR) { + threshold = get_maximum_error(); + } + + std::vector items; + for (auto &it: map) { + const uint64_t lb = it.second; + const uint64_t ub = it.second + offset; + if ((err_type == NO_FALSE_NEGATIVES and ub > threshold) or (err_type == NO_FALSE_POSITIVES and lb > threshold)) { + items.push_back(row(&it.first, it.second, offset)); + } + } + // sort by estimate in descending order + std::sort(items.begin(), items.end(), [](row a, row b){ return a.get_estimate() > b.get_estimate(); }); + return items; +} + +template +void frequent_items_sketch::serialize(std::ostream& os) const { + const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_EMPTY : PREAMBLE_LONGS_NONEMPTY; + os.write((char*)&preamble_longs, sizeof(preamble_longs)); + const uint8_t serial_version = SERIAL_VERSION; + os.write((char*)&serial_version, sizeof(serial_version)); + const uint8_t family = FAMILY_ID; + os.write((char*)&family, sizeof(family)); + const uint8_t lg_max_size = map.get_lg_max_size(); + os.write((char*)&lg_max_size, sizeof(lg_max_size)); + const uint8_t lg_cur_size = map.get_lg_cur_size(); + os.write((char*)&lg_cur_size, sizeof(lg_cur_size)); + const uint8_t flags_byte( + (is_empty() ? 1 << flags::IS_EMPTY : 0) + ); + os.write((char*)&flags_byte, sizeof(flags_byte)); + const uint16_t unused16 = 0; + os.write((char*)&unused16, sizeof(unused16)); + if (!is_empty()) { + const uint32_t num_items = map.get_num_active(); + os.write((char*)&num_items, sizeof(num_items)); + const uint32_t unused32 = 0; + os.write((char*)&unused32, sizeof(unused32)); + os.write((char*)&total_weight, sizeof(total_weight)); + os.write((char*)&offset, sizeof(offset)); + + // copy active items and their weights to use batch serialization + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + uint64_t* weights = AllocU64().allocate(num_items); + T* items = A().allocate(num_items); + uint32_t i = 0; + for (auto &it: map) { + new (&items[i]) T(it.first); + weights[i++] = it.second; + } + os.write((char*)weights, sizeof(uint64_t) * num_items); + AllocU64().deallocate(weights, num_items); + S().serialize(os, items, num_items); + for (unsigned i = 0; i < num_items; i++) items[i].~T(); + A().deallocate(items, num_items); + } +} + +template +size_t frequent_items_sketch::get_serialized_size_bytes() const { + if (is_empty()) return PREAMBLE_LONGS_EMPTY * sizeof(uint64_t); + size_t size = (PREAMBLE_LONGS_NONEMPTY + map.get_num_active()) * sizeof(uint64_t); + for (auto &it: map) size += S().size_of_item(it.first); + return size; +} + +template +vector_u8 frequent_items_sketch::serialize(unsigned header_size_bytes) const { + const size_t size = header_size_bytes + get_serialized_size_bytes(); + vector_u8 bytes(size); + uint8_t* ptr = bytes.data() + header_size_bytes; + + const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_EMPTY : PREAMBLE_LONGS_NONEMPTY; + ptr += copy_to_mem(&preamble_longs, ptr, sizeof(uint8_t)); + const uint8_t serial_version = SERIAL_VERSION; + ptr += copy_to_mem(&serial_version, ptr, sizeof(uint8_t)); + const uint8_t family = FAMILY_ID; + ptr += copy_to_mem(&family, ptr, sizeof(uint8_t)); + const uint8_t lg_max_size = map.get_lg_max_size(); + ptr += copy_to_mem(&lg_max_size, ptr, sizeof(uint8_t)); + const uint8_t lg_cur_size = map.get_lg_cur_size(); + ptr += copy_to_mem(&lg_cur_size, ptr, sizeof(uint8_t)); + const uint8_t flags_byte( + (is_empty() ? 1 << flags::IS_EMPTY : 0) + ); + ptr += copy_to_mem(&flags_byte, ptr, sizeof(uint8_t)); + const uint16_t unused16 = 0; + ptr += copy_to_mem(&unused16, ptr, sizeof(uint16_t)); + if (!is_empty()) { + const uint32_t num_items = map.get_num_active(); + ptr += copy_to_mem(&num_items, ptr, sizeof(uint32_t)); + const uint32_t unused32 = 0; + ptr += copy_to_mem(&unused32, ptr, sizeof(uint32_t)); + ptr += copy_to_mem(&total_weight, ptr, sizeof(uint64_t)); + ptr += copy_to_mem(&offset, ptr, sizeof(uint64_t)); + + // copy active items and their weights to use batch serialization + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + uint64_t* weights = AllocU64().allocate(num_items); + T* items = A().allocate(num_items); + uint32_t i = 0; + for (auto &it: map) { + new (&items[i]) T(it.first); + weights[i++] = it.second; + } + ptr += copy_to_mem(weights, ptr, sizeof(uint64_t) * num_items); + AllocU64().deallocate(weights, num_items); + ptr += S().serialize(ptr, items, num_items); + for (unsigned i = 0; i < num_items; i++) items[i].~T(); + A().deallocate(items, num_items); + } + return bytes; +} + +template +frequent_items_sketch frequent_items_sketch::deserialize(std::istream& is) { + uint8_t preamble_longs; + is.read((char*)&preamble_longs, sizeof(preamble_longs)); + uint8_t serial_version; + is.read((char*)&serial_version, sizeof(serial_version)); + uint8_t family_id; + is.read((char*)&family_id, sizeof(family_id)); + uint8_t lg_max_size; + is.read((char*)&lg_max_size, sizeof(lg_max_size)); + uint8_t lg_cur_size; + is.read((char*)&lg_cur_size, sizeof(lg_cur_size)); + uint8_t flags_byte; + is.read((char*)&flags_byte, sizeof(flags_byte)); + uint16_t unused16; + is.read((char*)&unused16, sizeof(unused16)); + + const bool is_empty = flags_byte & (1 << flags::IS_EMPTY); + + check_preamble_longs(preamble_longs, is_empty); + check_serial_version(serial_version); + check_family_id(family_id); + check_size(lg_cur_size, lg_max_size); + + frequent_items_sketch sketch(lg_cur_size, lg_max_size); + if (!is_empty) { + uint32_t num_items; + is.read((char*)&num_items, sizeof(num_items)); + uint32_t unused32; + is.read((char*)&unused32, sizeof(unused32)); + uint64_t total_weight; + is.read((char*)&total_weight, sizeof(total_weight)); + uint64_t offset; + is.read((char*)&offset, sizeof(offset)); + + // batch deserialization with intermediate array of items and weights + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + uint64_t* weights = AllocU64().allocate(num_items); + is.read((char*)weights, sizeof(uint64_t) * num_items); + T* items = A().allocate(num_items); // rely on serde to construct items + S().deserialize(is, items, num_items); + for (uint32_t i = 0; i < num_items; i++) { + sketch.update(std::move(items[i]), weights[i]); + items[i].~T(); + } + AllocU64().deallocate(weights, num_items); + A().deallocate(items, num_items); + + sketch.total_weight = total_weight; + sketch.offset = offset; + } + return sketch; +} + +template +frequent_items_sketch frequent_items_sketch::deserialize(const void* bytes, size_t size) { + const char* ptr = static_cast(bytes); + uint8_t preamble_longs; + ptr += copy_from_mem(ptr, &preamble_longs, sizeof(uint8_t)); + uint8_t serial_version; + ptr += copy_from_mem(ptr, &serial_version, sizeof(uint8_t)); + uint8_t family_id; + ptr += copy_from_mem(ptr, &family_id, sizeof(uint8_t)); + uint8_t lg_max_size; + ptr += copy_from_mem(ptr, &lg_max_size, sizeof(uint8_t)); + uint8_t lg_cur_size; + ptr += copy_from_mem(ptr, &lg_cur_size, sizeof(uint8_t)); + uint8_t flags_byte; + ptr += copy_from_mem(ptr, &flags_byte, sizeof(uint8_t)); + uint16_t unused16; + ptr += copy_from_mem(ptr, &unused16, sizeof(uint16_t)); + + const bool is_empty = flags_byte & (1 << flags::IS_EMPTY); + + check_preamble_longs(preamble_longs, is_empty); + check_serial_version(serial_version); + check_family_id(family_id); + check_size(lg_cur_size, lg_max_size); + + frequent_items_sketch sketch(lg_cur_size, lg_max_size); + if (!is_empty) { + uint32_t num_items; + ptr += copy_from_mem(ptr, &num_items, sizeof(uint32_t)); + uint32_t unused32; + ptr += copy_from_mem(ptr, &unused32, sizeof(uint32_t)); + uint64_t total_weight; + ptr += copy_from_mem(ptr, &total_weight, sizeof(uint64_t)); + uint64_t offset; + ptr += copy_from_mem(ptr, &offset, sizeof(uint64_t)); + + // batch deserialization with intermediate array of items and weights + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + uint64_t* weights = AllocU64().allocate(num_items); + ptr += copy_from_mem(ptr, weights, sizeof(uint64_t) * num_items); + T* items = A().allocate(num_items); + ptr += S().deserialize(ptr, items, num_items); + for (uint32_t i = 0; i < num_items; i++) { + sketch.update(std::move(items[i]), weights[i]); + items[i].~T(); + } + AllocU64().deallocate(weights, num_items); + A().deallocate(items, num_items); + + sketch.total_weight = total_weight; + sketch.offset = offset; + } + return sketch; +} + +template +void frequent_items_sketch::check_preamble_longs(uint8_t preamble_longs, bool is_empty) { + if (is_empty) { + if (preamble_longs != PREAMBLE_LONGS_EMPTY) { + throw std::invalid_argument("Possible corruption: preamble longs of an empty sketch must be " + std::to_string(PREAMBLE_LONGS_EMPTY) + ": " + std::to_string(preamble_longs)); + } + } else { + if (preamble_longs != PREAMBLE_LONGS_NONEMPTY) { + throw std::invalid_argument("Possible corruption: preamble longs of an non-empty sketch must be " + std::to_string(PREAMBLE_LONGS_NONEMPTY) + ": " + std::to_string(preamble_longs)); + } + } +} + +template +void frequent_items_sketch::check_serial_version(uint8_t serial_version) { + if (serial_version != SERIAL_VERSION) { + throw std::invalid_argument("Possible corruption: serial version must be " + std::to_string(SERIAL_VERSION) + ": " + std::to_string(serial_version)); + } +} + +template +void frequent_items_sketch::check_family_id(uint8_t family_id) { + if (family_id != FAMILY_ID) { + throw std::invalid_argument("Possible corruption: family ID must be " + std::to_string(FAMILY_ID) + ": " + std::to_string(family_id)); + } +} + +template +void frequent_items_sketch::check_size(uint8_t lg_cur_size, uint8_t lg_max_size) { + if (lg_cur_size > lg_max_size) { + throw std::invalid_argument("Possible corruption: expected lg_cur_size <= lg_max_size: " + std::to_string(lg_cur_size) + " <= " + std::to_string(lg_max_size)); + } + if (lg_cur_size < LG_MIN_MAP_SIZE) { + throw std::invalid_argument("Possible corruption: lg_cur_size must not be less than " + std::to_string(LG_MIN_MAP_SIZE) + ": " + std::to_string(lg_cur_size)); + } +} + +template +void frequent_items_sketch::to_stream(std::ostream& os, bool print_items) const { + os << "### Frequent items sketch summary:" << std::endl; + os << " lg cur map size : " << (int) map.get_lg_cur_size() << std::endl; + os << " lg max map size : " << (int) map.get_lg_max_size() << std::endl; + os << " num active items : " << get_num_active_items() << std::endl; + os << " total weight : " << get_total_weight() << std::endl; + os << " max error : " << get_maximum_error() << std::endl; + os << "### End sketch summary" << std::endl; + if (print_items) { + std::vector items; + for (auto &it: map) { + items.push_back(row(&it.first, it.second, offset)); + } + // sort by estimate in descending order + std::sort(items.begin(), items.end(), [](row a, row b){ return a.get_estimate() > b.get_estimate(); }); + os << "### Items in descending order by estimate" << std::endl; + os << " item, estimate, lower bound, upper bound" << std::endl; + for (auto &it: items) { + os << " " << it.get_item() << ", " << it.get_estimate() << ", " + << it.get_lower_bound() << ", " << it.get_upper_bound() << std::endl; + } + os << "### End items" << std::endl; + } +} + +} /* namespace datasketches */ + +# endif diff --git a/3rd/datasketches/datasketches/fi/reverse_purge_hash_map.hpp b/3rd/datasketches/datasketches/fi/reverse_purge_hash_map.hpp new file mode 100644 index 000000000..1b8a7b6c5 --- /dev/null +++ b/3rd/datasketches/datasketches/fi/reverse_purge_hash_map.hpp @@ -0,0 +1,425 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef REVERSE_PURGE_HASH_MAP_HPP_ +#define REVERSE_PURGE_HASH_MAP_HPP_ + +#include +#include +#include +#include + +#if defined(_MSC_VER) +#include // for and/or keywords +#endif // _MSC_VER + +namespace datasketches { + +/* + * Based on Java implementation here: + * https://github.com/DataSketches/sketches-core/blob/master/src/main/java/com/yahoo/sketches/frequencies/ReversePurgeItemHashMap.java + * author Alexander Saydakov + */ + +template, typename E = std::equal_to, typename A = std::allocator> +class reverse_purge_hash_map { + typedef typename std::allocator_traits::template rebind_alloc AllocU16; + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + +public: + reverse_purge_hash_map(uint8_t lg_size, uint8_t lg_max_size); + reverse_purge_hash_map(const reverse_purge_hash_map& other); + reverse_purge_hash_map(reverse_purge_hash_map&& other) noexcept; + ~reverse_purge_hash_map(); + reverse_purge_hash_map& operator=(reverse_purge_hash_map other); + reverse_purge_hash_map& operator=(reverse_purge_hash_map&& other); + uint64_t adjust_or_insert(const T& key, uint64_t value); + uint64_t adjust_or_insert(T&& key, uint64_t value); + uint64_t get(const T& key) const; + uint8_t get_lg_cur_size() const; + uint8_t get_lg_max_size() const; + uint32_t get_capacity() const; + uint32_t get_num_active() const; + class const_iterator; + const_iterator begin() const; + const_iterator end() const; +private: + static constexpr double LOAD_FACTOR = 0.75; + static constexpr uint16_t DRIFT_LIMIT = 1024; // used only for stress testing + static constexpr uint32_t MAX_SAMPLE_SIZE = 1024; // number of samples to compute approximate median during purge + + uint8_t lg_cur_size; + uint8_t lg_max_size; + uint32_t num_active; + T* keys; + uint64_t* values; + uint16_t* states; + + inline bool is_active(uint32_t probe) const; + void subtract_and_keep_positive_only(uint64_t amount); + void hash_delete(uint32_t probe); + uint32_t internal_adjust_or_insert(const T& key, uint64_t value); + uint64_t resize_or_purge_if_needed(); + void resize(uint8_t lg_new_size); + uint64_t purge(); +}; + +// clang++ seems to require this declaration for CMAKE_BUILD_TYPE='Debug" +template +constexpr uint32_t reverse_purge_hash_map::MAX_SAMPLE_SIZE; + +// This iterator uses strides based on golden ratio to avoid clustering during merge +template +class reverse_purge_hash_map::const_iterator: public std::iterator { +public: + friend class reverse_purge_hash_map; + const_iterator(const const_iterator& other) : map(other.map), index(other.index), count(other.count), stride(other.stride) {} + const_iterator& operator++() { + ++count; + if (count < map->num_active) { + const uint32_t mask = (1 << map->lg_cur_size) - 1; + do { + index = (index + stride) & mask; + } while (!map->is_active(index)); + } + return *this; + } + const_iterator operator++(int) { const_iterator tmp(*this); operator++(); return tmp; } + bool operator==(const const_iterator& rhs) const { return count == rhs.count; } + bool operator!=(const const_iterator& rhs) const { return count != rhs.count; } + const std::pair operator*() const { + return std::pair(map->keys[index], map->values[index]); + } +private: + static constexpr double GOLDEN_RATIO_RECIPROCAL = 0.6180339887498949; // = (sqrt(5) - 1) / 2 + const reverse_purge_hash_map* map; + uint32_t index; + uint32_t count; + uint32_t stride; + const_iterator(const reverse_purge_hash_map* map, uint32_t index, uint32_t count): + map(map), index(index), count(count), stride(static_cast((1 << map->lg_cur_size) * GOLDEN_RATIO_RECIPROCAL) | 1) {} +}; + +template +reverse_purge_hash_map::reverse_purge_hash_map(uint8_t lg_cur_size, uint8_t lg_max_size): +lg_cur_size(lg_cur_size), +lg_max_size(lg_max_size), +num_active(0), +keys(A().allocate(1 << lg_cur_size)), +values(AllocU64().allocate(1 << lg_cur_size)), +states(AllocU16().allocate(1 << lg_cur_size)) +{ + std::fill(states, &states[1 << lg_cur_size], 0); +} + +template +reverse_purge_hash_map::reverse_purge_hash_map(const reverse_purge_hash_map& other): +lg_cur_size(other.lg_cur_size), +lg_max_size(other.lg_max_size), +num_active(other.num_active), +keys(A().allocate(1 << lg_cur_size)), +values(AllocU64().allocate(1 << lg_cur_size)), +states(AllocU16().allocate(1 << lg_cur_size)) +{ + const uint32_t size = 1 << lg_cur_size; + if (num_active > 0) { + auto num = num_active; + for (uint32_t i = 0; i < size; i++) { + if (other.states[i] > 0) { + new (&keys[i]) T(other.keys[i]); + values[i] = other.values[i]; + } + if (--num == 0) break; + } + } + std::copy(&other.states[0], &other.states[size], states); +} + +template +reverse_purge_hash_map::reverse_purge_hash_map(reverse_purge_hash_map&& other) noexcept: +lg_cur_size(other.lg_cur_size), +lg_max_size(other.lg_max_size), +num_active(other.num_active), +keys(nullptr), +values(nullptr), +states(nullptr) +{ + std::swap(keys, other.keys); + std::swap(values, other.values); + std::swap(states, other.states); + other.num_active = 0; +} + +template +reverse_purge_hash_map::~reverse_purge_hash_map() { + const uint32_t size = 1 << lg_cur_size; + if (num_active > 0) { + for (uint32_t i = 0; i < size; i++) { + if (is_active(i)) { + keys[i].~T(); + if (--num_active == 0) break; + } + } + } + A().deallocate(keys, size); + AllocU64().deallocate(values, size); + AllocU16().deallocate(states, size); +} + +template +reverse_purge_hash_map& reverse_purge_hash_map::operator=(reverse_purge_hash_map other) { + std::swap(lg_cur_size, other.lg_cur_size); + std::swap(lg_max_size, other.lg_max_size); + std::swap(num_active, other.num_active); + std::swap(keys, other.keys); + std::swap(values, other.values); + std::swap(states, other.states); + return *this; +} + +template +reverse_purge_hash_map& reverse_purge_hash_map::operator=(reverse_purge_hash_map&& other) { + std::swap(lg_cur_size, other.lg_cur_size); + std::swap(lg_max_size, other.lg_max_size); + std::swap(num_active, other.num_active); + std::swap(keys, other.keys); + std::swap(values, other.values); + std::swap(states, other.states); + return *this; +} + +template +uint64_t reverse_purge_hash_map::adjust_or_insert(const T& key, uint64_t value) { + const uint32_t num_active_before = num_active; + const uint32_t index = internal_adjust_or_insert(key, value); + if (num_active > num_active_before) { + new (&keys[index]) T(key); + return resize_or_purge_if_needed(); + } + return 0; +} + +template +uint64_t reverse_purge_hash_map::adjust_or_insert(T&& key, uint64_t value) { + const uint32_t num_active_before = num_active; + const uint32_t index = internal_adjust_or_insert(key, value); + if (num_active > num_active_before) { + new (&keys[index]) T(std::move(key)); + return resize_or_purge_if_needed(); + } + return 0; +} + +template +uint64_t reverse_purge_hash_map::get(const T& key) const { + const uint32_t mask = (1 << lg_cur_size) - 1; + uint32_t probe = H()(key) & mask; + while (is_active(probe)) { + if (E()(keys[probe], key)) return values[probe]; + probe = (probe + 1) & mask; + } + return 0; +} + +template +uint8_t reverse_purge_hash_map::get_lg_cur_size() const { + return lg_cur_size; +} + +template +uint8_t reverse_purge_hash_map::get_lg_max_size() const { + return lg_max_size; +} + +template +uint32_t reverse_purge_hash_map::get_capacity() const { + return (1 << lg_cur_size) * LOAD_FACTOR; +} + +template +uint32_t reverse_purge_hash_map::get_num_active() const { + return num_active; +} + +template +typename reverse_purge_hash_map::const_iterator reverse_purge_hash_map::begin() const { + const uint32_t size = 1 << lg_cur_size; + uint32_t i = 0; + while (i < size and !is_active(i)) i++; + return reverse_purge_hash_map::const_iterator(this, i, 0); +} + +template +typename reverse_purge_hash_map::const_iterator reverse_purge_hash_map::end() const { + return reverse_purge_hash_map::const_iterator(this, 1 << lg_cur_size, num_active); +} + +template +bool reverse_purge_hash_map::is_active(uint32_t index) const { + return states[index] > 0; +} + +template +void reverse_purge_hash_map::subtract_and_keep_positive_only(uint64_t amount) { + // starting from the back, find the first empty cell, + // which establishes the high end of a cluster. + uint32_t first_probe = (1 << lg_cur_size) - 1; + while (is_active(first_probe)) first_probe--; + // when we find the next non-empty cell, we know we are at the high end of a cluster + // work towards the front, delete any non-positive entries. + for (uint32_t probe = first_probe; probe-- > 0;) { + if (is_active(probe)) { + if (values[probe] <= amount) { + hash_delete(probe); // does the work of deletion and moving higher items towards the front + num_active--; + } else { + values[probe] -= amount; + } + } + } + // now work on the first cluster that was skipped + for (uint32_t probe = (1 << lg_cur_size); probe-- > first_probe;) { + if (is_active(probe)) { + if (values[probe] <= amount) { + hash_delete(probe); + num_active--; + } else { + values[probe] -= amount; + } + } + } +} + +template +void reverse_purge_hash_map::hash_delete(uint32_t delete_index) { + // Looks ahead in the table to search for another + // item to move to this location + // if none are found, the status is changed + states[delete_index] = 0; // mark as empty + keys[delete_index].~T(); + uint32_t drift = 1; + const uint32_t mask = (1 << lg_cur_size) - 1; + uint32_t probe = (delete_index + drift) & mask; // map length must be a power of 2 + // advance until we find a free location replacing locations as needed + while (is_active(probe)) { + if (states[probe] > drift) { + // move current element + new (&keys[delete_index]) T(std::move(keys[probe])); + values[delete_index] = values[probe]; + states[delete_index] = states[probe] - drift; + states[probe] = 0; // mark as empty + keys[probe].~T(); + drift = 0; + delete_index = probe; + } + probe = (probe + 1) & mask; + drift++; + // only used for theoretical analysis + if (drift >= DRIFT_LIMIT) throw std::logic_error("drift: " + std::to_string(drift) + " >= DRIFT_LIMIT"); + } +} + +template +uint32_t reverse_purge_hash_map::internal_adjust_or_insert(const T& key, uint64_t value) { + const uint32_t mask = (1 << lg_cur_size) - 1; + uint32_t index = H()(key) & mask; + uint16_t drift = 1; + while (is_active(index)) { + if (E()(keys[index], key)) { + // adjusting the value of an existing key + values[index] += value; + return index; + } + index = (index + 1) & mask; + drift++; + // only used for theoretical analysis + if (drift >= DRIFT_LIMIT) throw std::logic_error("drift limit reached"); + } + // adding the key and value to the table + if (num_active > get_capacity()) { + throw std::logic_error("num_active " + std::to_string(num_active) + " > capacity " + std::to_string(get_capacity())); + } + values[index] = value; + states[index] = drift; + num_active++; + return index; +} + +template +uint64_t reverse_purge_hash_map::resize_or_purge_if_needed() { + if (num_active > get_capacity()) { + if (lg_cur_size < lg_max_size) { // can grow + resize(lg_cur_size + 1); + } else { // at target size, must purge + const uint64_t offset = purge(); + if (num_active > get_capacity()) { + throw std::logic_error("purge did not reduce number of active items"); + } + return offset; + } + } + return 0; +} + +template +void reverse_purge_hash_map::resize(uint8_t lg_new_size) { + const uint32_t old_size = 1 << lg_cur_size; + T* old_keys = keys; + uint64_t* old_values = values; + uint16_t* old_states = states; + const uint32_t new_size = 1 << lg_new_size; + keys = A().allocate(new_size); + values = AllocU64().allocate(new_size); + states = AllocU16().allocate(new_size); + std::fill(states, &states[new_size], 0); + num_active = 0; + lg_cur_size = lg_new_size; + for (uint32_t i = 0; i < old_size; i++) { + if (old_states[i] > 0) { + adjust_or_insert(std::move(old_keys[i]), old_values[i]); + old_keys[i].~T(); + } + } + A().deallocate(old_keys, old_size); + AllocU64().deallocate(old_values, old_size); + AllocU16().deallocate(old_states, old_size); +} + +template +uint64_t reverse_purge_hash_map::purge() { + const uint32_t limit = std::min(MAX_SAMPLE_SIZE, num_active); + uint32_t num_samples = 0; + uint32_t i = 0; + uint64_t* samples = AllocU64().allocate(limit); + while (num_samples < limit) { + if (is_active(i)) { + samples[num_samples++] = values[i]; + } + i++; + } + std::nth_element(&samples[0], &samples[num_samples / 2], &samples[num_samples - 1]); + const uint64_t median = samples[num_samples / 2]; + AllocU64().deallocate(samples, limit); + subtract_and_keep_positive_only(median); + return median; +} + +} /* namespace datasketches */ + +# endif diff --git a/3rd/datasketches/datasketches/kll/kll_helper.hpp b/3rd/datasketches/datasketches/kll/kll_helper.hpp new file mode 100644 index 000000000..a13e49be2 --- /dev/null +++ b/3rd/datasketches/datasketches/kll/kll_helper.hpp @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef KLL_HELPER_HPP_ +#define KLL_HELPER_HPP_ + +#include +#include +#include + +namespace datasketches { + +static std::independent_bits_engine random_bit(std::chrono::system_clock::now().time_since_epoch().count()); + +#ifdef KLL_VALIDATION +extern uint32_t kll_next_offset; +#endif + +// 0 <= power <= 30 +static const uint64_t powers_of_three[] = {1, 3, 9, 27, 81, 243, 729, 2187, 6561, 19683, 59049, 177147, 531441, +1594323, 4782969, 14348907, 43046721, 129140163, 387420489, 1162261467, +3486784401, 10460353203, 31381059609, 94143178827, 282429536481, +847288609443, 2541865828329, 7625597484987, 22876792454961, 68630377364883, +205891132094649}; + +class kll_helper { + public: + static inline bool is_even(uint32_t value); + static inline bool is_odd(uint32_t value); + static inline uint8_t floor_of_log2_of_fraction(uint64_t numer, uint64_t denom); + static inline uint8_t ub_on_num_levels(uint64_t n); + static inline uint32_t compute_total_capacity(uint16_t k, uint8_t m, uint8_t num_levels); + static inline uint32_t level_capacity(uint16_t k, uint8_t numLevels, uint8_t height, uint8_t min_wid); + static inline uint32_t int_cap_aux(uint16_t k, uint8_t depth); + static inline uint32_t int_cap_aux_aux(uint16_t k, uint8_t depth); + static inline uint64_t sum_the_sample_weights(uint8_t num_levels, const uint32_t* levels); + + /* + * This version is for floating point types + * Checks the sequential validity of the given array of values. + * They must be unique, monotonically increasing and not NaN. + */ + template + static typename std::enable_if::value, void>::type + validate_values(const T* values, uint32_t size) { + for (uint32_t i = 0; i < size ; i++) { + if (std::isnan(values[i])) { + throw std::invalid_argument("Values must not be NaN"); + } + if ((i < (size - 1)) and !(C()(values[i], values[i + 1]))) { + throw std::invalid_argument("Values must be unique and monotonically increasing"); + } + } + } + /* + * This version is for non-floating point types + * Checks the sequential validity of the given array of values. + * They must be unique and monotonically increasing. + */ + template + static typename std::enable_if::value, void>::type + validate_values(const T* values, uint32_t size) { + for (uint32_t i = 0; i < size ; i++) { + if ((i < (size - 1)) and !(C()(values[i], values[i + 1]))) { + throw std::invalid_argument("Values must be unique and monotonically increasing"); + } + } + } + + template + static void randomly_halve_down(T* buf, uint32_t start, uint32_t length); + + template + static void randomly_halve_up(T* buf, uint32_t start, uint32_t length); + + // this version moves objects within the same buffer + // assumes that destination has initialized objects + // does not destroy the originals after the move + template + static void merge_sorted_arrays(T* buf, uint32_t start_a, uint32_t len_a, uint32_t start_b, uint32_t len_b, uint32_t start_c); + + // this version is to merge from two different buffers into a third buffer + // initializes objects is the destination buffer + // moves objects from buf_a and destroys the originals + // copies objects from buf_b + template + static void merge_sorted_arrays(const T* buf_a, uint32_t start_a, uint32_t len_a, const T* buf_b, uint32_t start_b, uint32_t len_b, T* buf_c, uint32_t start_c); + + struct compress_result { + uint8_t final_num_levels; + uint32_t final_capacity; + uint32_t final_num_items; + }; + + /* + * Here is what we do for each level: + * If it does not need to be compacted, then simply copy it over. + * + * Otherwise, it does need to be compacted, so... + * Copy zero or one guy over. + * If the level above is empty, halve up. + * Else the level above is nonempty, so... + * halve down, then merge up. + * Adjust the boundaries of the level above. + * + * It can be proved that general_compress returns a sketch that satisfies the space constraints + * no matter how much data is passed in. + * All levels except for level zero must be sorted before calling this, and will still be + * sorted afterwards. + * Level zero is not required to be sorted before, and may not be sorted afterwards. + */ + template + static compress_result general_compress(uint16_t k, uint8_t m, uint8_t num_levels_in, T* items, + uint32_t* in_levels, uint32_t* out_levels, bool is_level_zero_sorted); + + template + static void copy_construct(const T* src, size_t src_first, size_t src_last, T* dst, size_t dst_first); + + template + static void move_construct(T* src, size_t src_first, size_t src_last, T* dst, size_t dst_first, bool destroy); + +#ifdef KLL_VALIDATION + private: + + static inline uint32_t deterministic_offset(); +#endif + +}; + +} /* namespace datasketches */ + +#include "kll_helper_impl.hpp" + +#endif // KLL_HELPER_HPP_ diff --git a/3rd/datasketches/datasketches/kll/kll_helper_impl.hpp b/3rd/datasketches/datasketches/kll/kll_helper_impl.hpp new file mode 100644 index 000000000..bc8435fce --- /dev/null +++ b/3rd/datasketches/datasketches/kll/kll_helper_impl.hpp @@ -0,0 +1,319 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef KLL_HELPER_IMPL_HPP_ +#define KLL_HELPER_IMPL_HPP_ + +#include + +namespace datasketches { + +bool kll_helper::is_even(uint32_t value) { + return (value & 1) == 0; +} + +bool kll_helper::is_odd(uint32_t value) { + return (value & 1) > 0; +} + +uint8_t kll_helper::floor_of_log2_of_fraction(uint64_t numer, uint64_t denom) { + if (denom > numer) return 0; + uint8_t count = 0; + while (true) { + denom <<= 1; + if (denom > numer) return count; + count++; + } +} + +uint8_t kll_helper::ub_on_num_levels(uint64_t n) { + if (n == 0) return 1; + return 1 + floor_of_log2_of_fraction(n, 1); +} + +uint32_t kll_helper::compute_total_capacity(uint16_t k, uint8_t m, uint8_t num_levels) { + uint32_t total = 0; + for (uint8_t h = 0; h < num_levels; h++) { + total += level_capacity(k, num_levels, h, m); + } + return total; +} + +uint32_t kll_helper::level_capacity(uint16_t k, uint8_t numLevels, uint8_t height, uint8_t min_wid) { + if (height >= numLevels) throw std::invalid_argument("height >= numLevels"); + const uint8_t depth = numLevels - height - 1; + return std::max((uint32_t) min_wid, int_cap_aux(k, depth)); +} + +uint32_t kll_helper::int_cap_aux(uint16_t k, uint8_t depth) { + if (depth > 60) throw std::invalid_argument("depth > 60"); + if (depth <= 30) return int_cap_aux_aux(k, depth); + const uint8_t half = depth / 2; + const uint8_t rest = depth - half; + const uint32_t tmp = int_cap_aux_aux(k, half); + return int_cap_aux_aux(tmp, rest); +} + +uint32_t kll_helper::int_cap_aux_aux(uint16_t k, uint8_t depth) { + if (depth > 30) throw std::invalid_argument("depth > 30"); + const uint64_t twok = k << 1; // for rounding, we pre-multiply by 2 + const uint64_t tmp = (uint64_t) (((uint64_t) twok << depth) / powers_of_three[depth]); + const uint64_t result = (tmp + 1) >> 1; // then here we add 1 and divide by 2 + if (result > k) throw std::logic_error("result > k"); + return result; +} + +uint64_t kll_helper::sum_the_sample_weights(uint8_t num_levels, const uint32_t* levels) { + uint64_t total = 0; + uint64_t weight = 1; + for (uint8_t lvl = 0; lvl < num_levels; lvl++) { + total += weight * (levels[lvl + 1] - levels[lvl]); + weight *= 2; + } + return total; +} + +template +void kll_helper::randomly_halve_down(T* buf, uint32_t start, uint32_t length) { + if (!is_even(length)) throw std::invalid_argument("length must be even"); + const uint32_t half_length = length / 2; +#ifdef KLL_VALIDATION + const uint32_t offset = deterministic_offset(); +#else + const uint32_t offset = random_bit(); +#endif + uint32_t j = start + offset; + for (uint32_t i = start; i < (start + half_length); i++) { + if (i != j) buf[i] = std::move(buf[j]); + j += 2; + } +} + +template +void kll_helper::randomly_halve_up(T* buf, uint32_t start, uint32_t length) { + if (!is_even(length)) throw std::invalid_argument("length must be even"); + const uint32_t half_length = length / 2; +#ifdef KLL_VALIDATION + const uint32_t offset = deterministic_offset(); +#else + const uint32_t offset = random_bit(); +#endif + uint32_t j = (start + length) - 1 - offset; + for (uint32_t i = (start + length) - 1; i >= (start + half_length); i--) { + if (i != j) buf[i] = std::move(buf[j]); + j -= 2; + } +} + +// this version moves objects within the same buffer +// assumes that destination has initialized objects +// does not destroy the originals after the move +template +void kll_helper::merge_sorted_arrays(T* buf, uint32_t start_a, uint32_t len_a, uint32_t start_b, uint32_t len_b, uint32_t start_c) { + const uint32_t len_c = len_a + len_b; + const uint32_t lim_a = start_a + len_a; + const uint32_t lim_b = start_b + len_b; + const uint32_t lim_c = start_c + len_c; + + uint32_t a = start_a; + uint32_t b = start_b; + + for (uint32_t c = start_c; c < lim_c; c++) { + if (a == lim_a) { + if (b != c) buf[c] = std::move(buf[b]); + b++; + } else if (b == lim_b) { + if (a != c) buf[c] = std::move(buf[a]); + a++; + } else if (C()(buf[a], buf[b])) { + if (a != c) buf[c] = std::move(buf[a]); + a++; + } else { + if (b != c) buf[c] = std::move(buf[b]); + b++; + } + } + if (a != lim_a || b != lim_b) throw std::logic_error("inconsistent state"); +} + +// this version is to merge from two different buffers into a third buffer +// initializes objects is the destination buffer +// moves objects from buf_a and destroys the originals +// copies objects from buf_b +template +void kll_helper::merge_sorted_arrays(const T* buf_a, uint32_t start_a, uint32_t len_a, const T* buf_b, uint32_t start_b, uint32_t len_b, T* buf_c, uint32_t start_c) { + const uint32_t len_c = len_a + len_b; + const uint32_t lim_a = start_a + len_a; + const uint32_t lim_b = start_b + len_b; + const uint32_t lim_c = start_c + len_c; + + uint32_t a = start_a; + uint32_t b = start_b; + + for (uint32_t c = start_c; c < lim_c; c++) { + if (a == lim_a) { + new (&buf_c[c]) T(buf_b[b++]); + } else if (b == lim_b) { + new (&buf_c[c]) T(std::move(buf_a[a])); + buf_a[a++].~T(); + } else if (C()(buf_a[a], buf_b[b])) { + new (&buf_c[c]) T(std::move(buf_a[a])); + buf_a[a++].~T(); + } else { + new (&buf_c[c]) T(buf_b[b++]); + } + } + if (a != lim_a || b != lim_b) throw std::logic_error("inconsistent state"); +} + +/* + * Here is what we do for each level: + * If it does not need to be compacted, then simply copy it over. + * + * Otherwise, it does need to be compacted, so... + * Copy zero or one guy over. + * If the level above is empty, halve up. + * Else the level above is nonempty, so... + * halve down, then merge up. + * Adjust the boundaries of the level above. + * + * It can be proved that general_compress returns a sketch that satisfies the space constraints + * no matter how much data is passed in. + * All levels except for level zero must be sorted before calling this, and will still be + * sorted afterwards. + * Level zero is not required to be sorted before, and may not be sorted afterwards. + */ +template +kll_helper::compress_result kll_helper::general_compress(uint16_t k, uint8_t m, uint8_t num_levels_in, T* items, + uint32_t* in_levels, uint32_t* out_levels, bool is_level_zero_sorted) +{ + if (num_levels_in == 0) throw std::invalid_argument("num_levels_in == 0"); // things are too weird if zero levels are allowed + const uint32_t starting_item_count = in_levels[num_levels_in] - in_levels[0]; + uint8_t current_num_levels = num_levels_in; + uint32_t current_item_count = starting_item_count; // decreases with each compaction + uint32_t target_item_count = compute_total_capacity(k, m, current_num_levels); // increases if we add levels + bool done_yet = false; + out_levels[0] = 0; + uint8_t current_level = 0; + while (!done_yet) { + + // If we are at the current top level, add an empty level above it for convenience, + // but do not increment num_levels until later + if (current_level == (current_num_levels - 1)) { + in_levels[current_level + 2] = in_levels[current_level + 1]; + } + + const auto raw_beg = in_levels[current_level]; + const auto raw_lim = in_levels[current_level + 1]; + const auto raw_pop = raw_lim - raw_beg; + + if ((current_item_count < target_item_count) or (raw_pop < level_capacity(k, current_num_levels, current_level, m))) { + // move level over as is + // make sure we are not moving data upwards + if (raw_beg < out_levels[current_level]) throw std::logic_error("wrong move"); + std::move(&items[raw_beg], &items[raw_lim], &items[out_levels[current_level]]); + out_levels[current_level + 1] = out_levels[current_level] + raw_pop; + } else { + // The sketch is too full AND this level is too full, so we compact it + // Note: this can add a level and thus change the sketches capacities + + const auto pop_above = in_levels[current_level + 2] - raw_lim; + const bool odd_pop = is_odd(raw_pop); + const auto adj_beg = odd_pop ? 1 + raw_beg : raw_beg; + const auto adj_pop = odd_pop ? raw_pop - 1 : raw_pop; + const auto half_adj_pop = adj_pop / 2; + + if (odd_pop) { // move one guy over + items[out_levels[current_level]] = std::move(items[raw_beg]); + out_levels[current_level + 1] = out_levels[current_level] + 1; + } else { // even number of items + out_levels[current_level + 1] = out_levels[current_level]; + } + + // level zero might not be sorted, so we must sort it if we wish to compact it + if ((current_level == 0) and !is_level_zero_sorted) { + std::sort(&items[adj_beg], &items[adj_beg + adj_pop], C()); + } + + if (pop_above == 0) { // Level above is empty, so halve up + randomly_halve_up(items, adj_beg, adj_pop); + } else { // Level above is nonempty, so halve down, then merge up + randomly_halve_down(items, adj_beg, adj_pop); + merge_sorted_arrays(items, adj_beg, half_adj_pop, raw_lim, pop_above, adj_beg + half_adj_pop); + } + + // track the fact that we just eliminated some data + current_item_count -= half_adj_pop; + + // adjust the boundaries of the level above + in_levels[current_level + 1] = in_levels[current_level + 1] - half_adj_pop; + + // increment num_levels if we just compacted the old top level + // this creates some more capacity (the size of the new bottom level) + if (current_level == (current_num_levels - 1)) { + current_num_levels++; + target_item_count += level_capacity(k, current_num_levels, 0, m); + } + + } // end of code for compacting a level + + // determine whether we have processed all levels yet (including any new levels that we created) + + if (current_level == (current_num_levels - 1)) done_yet = true; + current_level++; + } // end of loop over levels + + if ((out_levels[current_num_levels] - out_levels[0]) != current_item_count) throw std::logic_error("inconsistent state"); + + for (uint32_t i = current_item_count; i < starting_item_count; i++) items[i].~T(); + + compress_result result; + result.final_num_levels = current_num_levels; + result.final_capacity = target_item_count; + result.final_num_items = current_item_count; + return result; +} + +template +void kll_helper::copy_construct(const T* src, size_t src_first, size_t src_last, T* dst, size_t dst_first) { + while (src_first != src_last) { + new (&dst[dst_first++]) T(src[src_first++]); + } +} + +template +void kll_helper::move_construct(T* src, size_t src_first, size_t src_last, T* dst, size_t dst_first, bool destroy) { + while (src_first != src_last) { + new (&dst[dst_first++]) T(std::move(src[src_first])); + if (destroy) src[src_first].~T(); + src_first++; + } +} + +#ifdef KLL_VALIDATION +uint32_t kll_helper::deterministic_offset() { + const uint32_t result(kll_next_offset); + kll_next_offset = 1 - kll_next_offset; + return result; +} +#endif + +} /* namespace datasketches */ + +#endif // KLL_HELPER_IMPL_HPP_ diff --git a/3rd/datasketches/datasketches/kll/kll_quantile_calculator.hpp b/3rd/datasketches/datasketches/kll/kll_quantile_calculator.hpp new file mode 100644 index 000000000..f77071e9c --- /dev/null +++ b/3rd/datasketches/datasketches/kll/kll_quantile_calculator.hpp @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef KLL_QUANTILE_CALCULATOR_HPP_ +#define KLL_QUANTILE_CALCULATOR_HPP_ + +#include + +namespace datasketches { + +template +class kll_quantile_calculator { + typedef typename std::allocator_traits::template rebind_alloc AllocU32; + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + public: + // assumes that all levels are sorted including level 0 + kll_quantile_calculator(const T* items, const uint32_t* levels, uint8_t num_levels, uint64_t n); + ~kll_quantile_calculator(); + T get_quantile(double fraction) const; + + private: + uint64_t n_; + T* items_; + uint64_t* weights_; + uint32_t* levels_; + uint8_t levels_size_; + uint8_t num_levels_; + + void populate_from_sketch(const T* items, uint32_t num_items, const uint32_t* levels, uint8_t num_levels); + T approximately_answer_positional_query(uint64_t pos) const; + static void convert_to_preceding_cummulative(uint64_t* weights, uint32_t weights_size); + static uint64_t pos_of_phi(double phi, uint64_t n); + static uint32_t chunk_containing_pos(uint64_t* weights, uint32_t weights_size, uint64_t pos); + static uint32_t search_for_chunk_containing_pos(const uint64_t* arr, uint64_t pos, uint32_t l, uint32_t r); + static void blocky_tandem_merge_sort(T* items, uint64_t* weights, uint32_t num_items, const uint32_t* levels, uint8_t num_levels); + static void blocky_tandem_merge_sort_recursion(T* items_src, uint64_t* weights_src, T* items_dst, uint64_t* weights_dst, const uint32_t* levels, uint8_t starting_level, uint8_t num_levels); + static void tandem_merge(const T* items_src, const uint64_t* weights_src, T* items_dst, uint64_t* weights_dst, const uint32_t* levels, uint8_t starting_level_1, uint8_t num_levels_1, uint8_t starting_level_2, uint8_t num_levels_2); +}; + +} /* namespace datasketches */ + +#include "kll_quantile_calculator_impl.hpp" + +#endif // KLL_QUANTILE_CALCULATOR_HPP_ diff --git a/3rd/datasketches/datasketches/kll/kll_quantile_calculator_impl.hpp b/3rd/datasketches/datasketches/kll/kll_quantile_calculator_impl.hpp new file mode 100644 index 000000000..16dc06fb2 --- /dev/null +++ b/3rd/datasketches/datasketches/kll/kll_quantile_calculator_impl.hpp @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef KLL_QUANTILE_CALCULATOR_IMPL_HPP_ +#define KLL_QUANTILE_CALCULATOR_IMPL_HPP_ + +#include +#include +#include + +#include "kll_helper.hpp" + +namespace datasketches { + +template +kll_quantile_calculator::kll_quantile_calculator(const T* items, const uint32_t* levels, uint8_t num_levels, uint64_t n) { + n_ = n; + const uint32_t num_items = levels[num_levels] - levels[0]; + items_ = A().allocate(num_items); + weights_ = AllocU64().allocate(num_items + 1); // one more is intentional + levels_size_ = num_levels + 1; + levels_ = AllocU32().allocate(levels_size_); + populate_from_sketch(items, num_items, levels, num_levels); + blocky_tandem_merge_sort(items_, weights_, num_items, levels_, num_levels_); + convert_to_preceding_cummulative(weights_, num_items + 1); +} + +template +kll_quantile_calculator::~kll_quantile_calculator() { + const uint32_t num_items = levels_[num_levels_] - levels_[0]; + for (uint32_t i = 0; i < num_items; i++) items_[i].~T(); + A().deallocate(items_, num_items); + AllocU64().deallocate(weights_, num_items + 1); + AllocU32().deallocate(levels_, levels_size_); +} + +template +T kll_quantile_calculator::get_quantile(double fraction) const { + return approximately_answer_positional_query(pos_of_phi(fraction, n_)); +} + +template +void kll_quantile_calculator::populate_from_sketch(const T* items, uint32_t num_items, const uint32_t* levels, uint8_t num_levels) { + kll_helper::copy_construct(items, levels[0], levels[num_levels], items_, 0); + uint8_t src_level = 0; + uint8_t dst_level = 0; + uint64_t weight = 1; + uint32_t offset = levels[0]; + while (src_level < num_levels) { + const uint32_t from_index(levels[src_level] - offset); + const uint32_t to_index(levels[src_level + 1] - offset); // exclusive + if (from_index < to_index) { // skip empty levels + std::fill(&weights_[from_index], &weights_[to_index], weight); + levels_[dst_level] = from_index; + levels_[dst_level + 1] = to_index; + dst_level++; + } + src_level++; + weight *= 2; + } + weights_[num_items] = 0; + num_levels_ = dst_level; +} + +template +T kll_quantile_calculator::approximately_answer_positional_query(uint64_t pos) const { + assert (pos < n_); + const uint32_t weights_size(levels_[num_levels_] + 1); + const uint32_t index = chunk_containing_pos(weights_, weights_size, pos); + return items_[index]; +} + +template +void kll_quantile_calculator::convert_to_preceding_cummulative(uint64_t* weights, uint32_t weights_size) { + uint64_t subtotal(0); + for (uint32_t i = 0; i < weights_size; i++) { + const uint32_t new_subtotal = subtotal + weights[i]; + weights[i] = subtotal; + subtotal = new_subtotal; + } +} + +template +uint64_t kll_quantile_calculator::pos_of_phi(double phi, uint64_t n) { + const uint64_t pos = std::floor(phi * n); + return (pos == n) ? n - 1 : pos; +} + +template +uint32_t kll_quantile_calculator::chunk_containing_pos(uint64_t* weights, uint32_t weights_size, uint64_t pos) { + assert (weights_size > 1); // remember, weights_ contains an "extra" position + const uint32_t nominal_length(weights_size - 1); + assert (weights[0] <= pos); + assert (pos < weights[nominal_length]); + return search_for_chunk_containing_pos(weights, pos, 0, nominal_length); +} + +template +uint32_t kll_quantile_calculator::search_for_chunk_containing_pos(const uint64_t* arr, uint64_t pos, uint32_t l, uint32_t r) { + if (l + 1 == r) { + return l; + } + const uint32_t m(l + (r - l) / 2); + if (arr[m] <= pos) { + return search_for_chunk_containing_pos(arr, pos, m, r); + } + return search_for_chunk_containing_pos(arr, pos, l, m); +} + +template +void kll_quantile_calculator::blocky_tandem_merge_sort(T* items, uint64_t* weights, uint32_t num_items, const uint32_t* levels, uint8_t num_levels) { + if (num_levels == 1) return; + + // move the input in preparation for the "ping-pong" reduction strategy + auto tmp_items_deleter = [num_items](T* ptr) { + for (uint32_t i = 0; i < num_items; i++) ptr[i].~T(); + A().deallocate(ptr, num_items); + }; + std::unique_ptr tmp_items(A().allocate(num_items), tmp_items_deleter); + kll_helper::move_construct(items, 0, num_items, tmp_items.get(), 0, false); // do not destroy since the items will be moved back + auto tmp_weights_deleter = [num_items](uint64_t* ptr) { AllocU64().deallocate(ptr, num_items); }; + std::unique_ptr tmp_weights(AllocU64().allocate(num_items), tmp_weights_deleter); // don't need the extra one here + std::copy(weights, &weights[num_items], tmp_weights.get()); + blocky_tandem_merge_sort_recursion(tmp_items.get(), tmp_weights.get(), items, weights, levels, 0, num_levels); +} + +template +void kll_quantile_calculator::blocky_tandem_merge_sort_recursion(T* items_src, uint64_t* weights_src, T* items_dst, uint64_t* weights_dst, const uint32_t* levels, uint8_t starting_level, uint8_t num_levels) { + if (num_levels == 1) return; + const uint8_t num_levels_1 = num_levels / 2; + const uint8_t num_levels_2 = num_levels - num_levels_1; + assert (num_levels_1 >= 1); + assert (num_levels_2 >= num_levels_1); + const uint8_t starting_level_1 = starting_level; + const uint8_t starting_level_2 = starting_level + num_levels_1; + // swap roles of src and dst + blocky_tandem_merge_sort_recursion(items_dst, weights_dst, items_src, weights_src, levels, starting_level_1, num_levels_1); + blocky_tandem_merge_sort_recursion(items_dst, weights_dst, items_src, weights_src, levels, starting_level_2, num_levels_2); + tandem_merge(items_src, weights_src, items_dst, weights_dst, levels, starting_level_1, num_levels_1, starting_level_2, num_levels_2); +} + +template +void kll_quantile_calculator::tandem_merge(const T* items_src, const uint64_t* weights_src, T* items_dst, uint64_t* weights_dst, const uint32_t* levels, uint8_t starting_level_1, uint8_t num_levels_1, uint8_t starting_level_2, uint8_t num_levels_2) { + const auto from_index_1 = levels[starting_level_1]; + const auto to_index_1 = levels[starting_level_1 + num_levels_1]; // exclusive + const auto from_index_2 = levels[starting_level_2]; + const auto to_index_2 = levels[starting_level_2 + num_levels_2]; // exclusive + auto i_src_1 = from_index_1; + auto i_src_2 = from_index_2; + auto i_dst = from_index_1; + + while ((i_src_1 < to_index_1) and (i_src_2 < to_index_2)) { + if (C()(items_src[i_src_1], items_src[i_src_2])) { + items_dst[i_dst] = std::move(items_src[i_src_1]); + weights_dst[i_dst] = weights_src[i_src_1]; + i_src_1++; + } else { + items_dst[i_dst] = std::move(items_src[i_src_2]); + weights_dst[i_dst] = weights_src[i_src_2]; + i_src_2++; + } + i_dst++; + } + if (i_src_1 < to_index_1) { + std::move(&items_src[i_src_1], &items_src[to_index_1], &items_dst[i_dst]); + std::copy(&weights_src[i_src_1], &weights_src[to_index_1], &weights_dst[i_dst]); + } else if (i_src_2 < to_index_2) { + std::move(&items_src[i_src_2], &items_src[to_index_2], &items_dst[i_dst]); + std::copy(&weights_src[i_src_2], &weights_src[to_index_2], &weights_dst[i_dst]); + } +} + +} /* namespace datasketches */ + +#endif // KLL_QUANTILE_CALCULATOR_IMPL_HPP_ diff --git a/3rd/datasketches/datasketches/kll/kll_sketch.hpp b/3rd/datasketches/datasketches/kll/kll_sketch.hpp new file mode 100644 index 000000000..141f55cf4 --- /dev/null +++ b/3rd/datasketches/datasketches/kll/kll_sketch.hpp @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef KLL_SKETCH_HPP_ +#define KLL_SKETCH_HPP_ + +#include +#include +#include + +#if defined(_MSC_VER) +#include // for and/or keywords +#endif // _MSC_VER + +#include "kll_quantile_calculator.hpp" +#include "serde.hpp" + +namespace datasketches { + +/* + * Implementation of a very compact quantiles sketch with lazy compaction scheme + * and nearly optimal accuracy per retained item. + * See Optimal Quantile Approximation in Streams. + * + *

This is a stochastic streaming sketch that enables near-real time analysis of the + * approximate distribution of values from a very large stream in a single pass, requiring only + * that the values are comparable. + * The analysis is obtained using get_quantile() or get_quantiles() functions or the + * inverse functions get_rank(), get_PMF() (Probability Mass Function), and get_CDF() + * (Cumulative Distribution Function). + * + *

Given an input stream of N numeric values, the absolute rank of any specific + * value is defined as its index (0 to N-1) in the hypothetical sorted stream of all + * N input values. + * + *

The normalized rank (rank) of any specific value is defined as its + * absolute rank divided by N. + * Thus, the normalized rank is a value between zero and one. + * In the documentation for this sketch absolute rank is never used so any + * reference to just rank should be interpreted to mean normalized rank. + * + *

This sketch is configured with a parameter k, which affects the size of the sketch + * and its estimation error. + * + *

The estimation error is commonly called epsilon (or eps) and is a fraction + * between zero and one. Larger values of k result in smaller values of epsilon. + * Epsilon is always with respect to the rank and cannot be applied to the + * corresponding values. + * + *

The relationship between the normalized rank and the corresponding values can be viewed + * as a two dimensional monotonic plot with the normalized rank on one axis and the + * corresponding values on the other axis. If the y-axis is specified as the value-axis and + * the x-axis as the normalized rank, then y = get_quantile(x) is a monotonically + * increasing function. + * + *

The functions get_quantile(rank) and get_quantiles(...) translate ranks into + * corresponding values. The functions get_rank(value), + * get_CDF(...) (Cumulative Distribution Function), and get_PMF(...) + * (Probability Mass Function) perform the opposite operation and translate values into ranks. + * + *

The getPMF(...) function has about 13 to 47% worse rank error (depending + * on k) than the other queries because the mass of each "bin" of the PMF has + * "double-sided" error from the upper and lower edges of the bin as a result of a subtraction, + * as the errors from the two edges can sometimes add. + * + *

The default k of 200 yields a "single-sided" epsilon of about 1.33% and a + * "double-sided" (PMF) epsilon of about 1.65%. + * + *

A get_quantile(rank) query has the following guarantees: + *

    + *
  • Let v = get_quantile(r) where r is the rank between zero and one.
    • + *
  • The value v will be a value from the input stream.
    • + *
  • Let trueRank be the true rank of v derived from the hypothetical sorted + * stream of all N values.
    • + *
  • Let eps = get_normalized_rank_error(false).
    • + *
  • Then r - eps ≤ trueRank ≤ r + eps with a confidence of 99%. Note that the + * error is on the rank, not the value.
    • + *
+ * + *

A get_rank(value) query has the following guarantees: + *

    + *
  • Let r = get_rank(v) where v is a value between the min and max values of + * the input stream.
    • + *
  • Let true_rank be the true rank of v derived from the hypothetical sorted + * stream of all N values.
    • + *
  • Let eps = get_normalized_rank_error(false).
    • + *
  • Then r - eps ≤ trueRank ≤ r + eps with a confidence of 99%.
    • + *
+ * + *

A get_PMF() query has the following guarantees: + *

    + *
  • Let {r1, r2, ..., r(m+1)} = get_PMF(v1, v2, ..., vm) where v1, v2 are values + * between the min and max values of the input stream. + *
  • Let massi = estimated mass between vi and vi+1.
    • + *
  • Let trueMass be the true mass between the values of vi, + * vi+1 derived from the hypothetical sorted stream of all N values.
    • + *
  • Let eps = get_normalized_rank_error(true).
    • + *
  • then mass - eps ≤ trueMass ≤ mass + eps with a confidence of 99%.
    • + *
  • r(m+1) includes the mass of all points larger than vm.
    • + *
+ * + *

A get_CDF(...) query has the following guarantees; + *

    + *
  • Let {r1, r2, ..., r(m+1)} = get_CDF(v1, v2, ..., vm) where v1, v2 are values + * between the min and max values of the input stream. + *
  • Let massi = ri+1 - ri.
    • + *
  • Let trueMass be the true mass between the true ranks of vi, + * vi+1 derived from the hypothetical sorted stream of all N values.
    • + *
  • Let eps = get_normalized_rank_error(true).
    • + *
  • then mass - eps ≤ trueMass ≤ mass + eps with a confidence of 99%.
    • + *
  • 1 - r(m+1) includes the mass of all points larger than vm.
    • + *
+ * + *

From the above, it might seem like we could make some estimates to bound the + * value returned from a call to get_quantile(). The sketch, however, does not + * let us derive error bounds or confidences around values. Because errors are independent, we + * can approximately bracket a value as shown below, but there are no error estimates available. + * Additionally, the interval may be quite large for certain distributions. + *

    + *
  • Let v = get_quantile(r), the estimated quantile value of rank r.
    • + *
  • Let eps = get_normalized_rank_error(false).
    • + *
  • Let vlo = estimated quantile value of rank (r - eps).
    • + *
  • Let vhi = estimated quantile value of rank (r + eps).
    • + *
  • Then vlo ≤ v ≤ vhi, with 99% confidence.
    • + *
+ * + * author Kevin Lang + * author Alexander Saydakov + * author Lee Rhodes + */ + +template using AllocU8 = typename std::allocator_traits::template rebind_alloc; +template using vector_u8 = std::vector>; +template using AllocD = typename std::allocator_traits::template rebind_alloc; +template using vector_d = std::vector>; + +template , typename S = serde, typename A = std::allocator> +class kll_sketch { + typedef typename std::allocator_traits::template rebind_alloc AllocU32; + + public: + static const uint8_t DEFAULT_M = 8; + static const uint16_t DEFAULT_K = 200; + static const uint16_t MIN_K = DEFAULT_M; + static const uint16_t MAX_K = (1 << 16) - 1; + + explicit kll_sketch(uint16_t k = DEFAULT_K); + kll_sketch(const kll_sketch& other); + kll_sketch(kll_sketch&& other) noexcept; + ~kll_sketch(); + kll_sketch& operator=(const kll_sketch& other); + kll_sketch& operator=(kll_sketch&& other); + void update(const T& value); + void update(T&& value); + void merge(const kll_sketch& other); + bool is_empty() const; + uint64_t get_n() const; + uint32_t get_num_retained() const; + bool is_estimation_mode() const; + T get_min_value() const; + T get_max_value() const; + T get_quantile(double fraction) const; + std::vector get_quantiles(const double* fractions, uint32_t size) const; + double get_rank(const T& value) const; + vector_d get_PMF(const T* split_points, uint32_t size) const; + vector_d get_CDF(const T* split_points, uint32_t size) const; + double get_normalized_rank_error(bool pmf) const; + + // implementation for fixed-size arithmetic types (integral and floating point) + template::value, int>::type = 0> + size_t get_serialized_size_bytes() const { + if (is_empty()) { return EMPTY_SIZE_BYTES; } + if (num_levels_ == 1 and get_num_retained() == 1) { + return DATA_START_SINGLE_ITEM + sizeof(TT); + } + // the last integer in the levels_ array is not serialized because it can be derived + return DATA_START + num_levels_ * sizeof(uint32_t) + (get_num_retained() + 2) * sizeof(TT); + } + + // implementation for all other types + template::value, int>::type = 0> + size_t get_serialized_size_bytes() const { + if (is_empty()) { return EMPTY_SIZE_BYTES; } + if (num_levels_ == 1 and get_num_retained() == 1) { + return DATA_START_SINGLE_ITEM + S().size_of_item(items_[levels_[0]]); + } + // the last integer in the levels_ array is not serialized because it can be derived + size_t size = DATA_START + num_levels_ * sizeof(uint32_t); + size += S().size_of_item(*min_value_); + size += S().size_of_item(*max_value_); + for (auto& it: *this) size += S().size_of_item(it.first); + return size; + } + + // this may need to be specialized to return correct size if sizeof(T) does not match the actual serialized size of an item + // this method is for the user's convenience to predict the sketch size before serialization + // and is not used in the serialization and deserialization code + // predicting the size before serialization may not make sense if the item type is not of a fixed size (like string) + static size_t get_max_serialized_size_bytes(uint16_t k, uint64_t n); + + void serialize(std::ostream& os) const; + typedef vector_u8 vector_bytes; // alias for users + vector_bytes serialize(unsigned header_size_bytes = 0) const; + static kll_sketch deserialize(std::istream& is); + static kll_sketch deserialize(const void* bytes, size_t size); + + /* + * Gets the normalized rank error given k and pmf. + * k - the configuration parameter + * pmf - if true, returns the "double-sided" normalized rank error for the get_PMF() function. + * Otherwise, it is the "single-sided" normalized rank error for all the other queries. + * Constants were derived as the best fit to 99 percentile empirically measured max error in thousands of trials + */ + static double get_normalized_rank_error(uint16_t k, bool pmf); + + void to_stream(std::ostream& os, bool print_levels = false, bool print_items = false) const; + + class const_iterator; + const_iterator begin() const; + const_iterator end() const; + + #ifdef KLL_VALIDATION + uint8_t get_num_levels() { return num_levels_; } + uint32_t* get_levels() { return levels_; } + T* get_items() { return items_; } + #endif + + private: + /* Serialized sketch layout: + * Adr: + * || 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | + * 0 || unused | M |--------K--------| Flags | FamID | SerVer | PreambleInts | + * || 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 | + * 1 ||-----------------------------------N------------------------------------------| + * || 23 | 22 | 21 | 20 | 19 | 18 | 17 | 16 | + * 2 ||---------------data----------------|-unused-|numLevels|-------min K-----------| + */ + + static const size_t EMPTY_SIZE_BYTES = 8; + static const size_t DATA_START_SINGLE_ITEM = 8; + static const size_t DATA_START = 20; + + static const uint8_t SERIAL_VERSION_1 = 1; + static const uint8_t SERIAL_VERSION_2 = 2; + static const uint8_t FAMILY = 15; + + enum flags { IS_EMPTY, IS_LEVEL_ZERO_SORTED, IS_SINGLE_ITEM }; + + static const uint8_t PREAMBLE_INTS_SHORT = 2; // for empty and single item + static const uint8_t PREAMBLE_INTS_FULL = 5; + + uint16_t k_; + uint8_t m_; // minimum buffer "width" + uint16_t min_k_; // for error estimation after merging with different k + uint64_t n_; + uint8_t num_levels_; + uint32_t* levels_; + uint8_t levels_size_; + T* items_; + uint32_t items_size_; + T* min_value_; + T* max_value_; + bool is_level_zero_sorted_; + + // for deserialization + // the common part of the preamble was read and compatibility checks were done + kll_sketch(uint16_t k, uint8_t flags_byte, std::istream& is); + + // for deserialization + // the common part of the preamble was read and compatibility checks were done + kll_sketch(uint16_t k, uint8_t flags_byte, const void* bytes, size_t size); + + // common update code + inline uint32_t internal_update(const T& value); + + // The following code is only valid in the special case of exactly reaching capacity while updating. + // It cannot be used while merging, while reducing k, or anything else. + void compress_while_updating(void); + + uint8_t find_level_to_compact() const; + void add_empty_top_level_to_completely_full_sketch(); + void sort_level_zero(); + std::unique_ptr, std::function*)>> get_quantile_calculator(); + vector_d get_PMF_or_CDF(const T* split_points, uint32_t size, bool is_CDF) const; + void increment_buckets_unsorted_level(uint32_t from_index, uint32_t to_index, uint64_t weight, + const T* split_points, uint32_t size, double* buckets) const; + void increment_buckets_sorted_level(uint32_t from_index, uint32_t to_index, uint64_t weight, + const T* split_points, uint32_t size, double* buckets) const; + void merge_higher_levels(const kll_sketch& other, uint64_t final_n); + void populate_work_arrays(const kll_sketch& other, T* workbuf, uint32_t* worklevels, uint8_t provisional_num_levels); + void assert_correct_total_weight() const; + uint32_t safe_level_size(uint8_t level) const; + uint32_t get_num_retained_above_level_zero() const; + + static void check_m(uint8_t m); + static void check_preamble_ints(uint8_t preamble_ints, uint8_t flags_byte); + static void check_serial_version(uint8_t serial_version); + static void check_family_id(uint8_t family_id); + + // implementation for floating point types + template::value, int>::type = 0> + static TT get_invalid_value() { + return std::numeric_limits::quiet_NaN(); + } + + // implementation for all other types + template::value, int>::type = 0> + static TT get_invalid_value() { + throw std::runtime_error("getting quantiles from empty sketch is not supported for this type of values"); + } + +}; + +template +class kll_sketch::const_iterator: public std::iterator { +public: + friend class kll_sketch; + const_iterator(const const_iterator& other); + const_iterator& operator++(); + const_iterator& operator++(int); + bool operator==(const const_iterator& other) const; + bool operator!=(const const_iterator& other) const; + const std::pair operator*() const; +private: + const T* items; + const uint32_t* levels; + const uint8_t num_levels; + uint32_t index; + uint8_t level; + uint64_t weight; + const_iterator(const T* items, const uint32_t* levels, const uint8_t num_levels); +}; + +} /* namespace datasketches */ + +#include "kll_sketch_impl.hpp" + +#endif diff --git a/3rd/datasketches/datasketches/kll/kll_sketch_impl.hpp b/3rd/datasketches/datasketches/kll/kll_sketch_impl.hpp new file mode 100644 index 000000000..dcd2453ad --- /dev/null +++ b/3rd/datasketches/datasketches/kll/kll_sketch_impl.hpp @@ -0,0 +1,977 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef KLL_SKETCH_IMPL_HPP_ +#define KLL_SKETCH_IMPL_HPP_ + +#include +#include + +#if defined(_MSC_VER) +#include // for and/or keywords +#endif // _MSC_VER + +#include "kll_helper.hpp" + +namespace datasketches { + +template +kll_sketch::kll_sketch(uint16_t k): +k_(k), +m_(DEFAULT_M), +min_k_(k), +n_(0), +num_levels_(1), +levels_(nullptr), +levels_size_(2), +items_(nullptr), +items_size_(k_), +min_value_(nullptr), +max_value_(nullptr), +is_level_zero_sorted_(false) +{ + if (k < MIN_K or k > MAX_K) { + throw std::invalid_argument("K must be >= " + std::to_string(MIN_K) + " and <= " + std::to_string(MAX_K) + ": " + std::to_string(k)); + } + levels_ = new (AllocU32().allocate(2)) uint32_t[2] {k_, k_}; + items_ = A().allocate(items_size_); +} + +template +kll_sketch::kll_sketch(const kll_sketch& other): +k_(other.k_), +m_(other.m_), +min_k_(other.min_k_), +n_(other.n_), +num_levels_(other.num_levels_), +levels_(nullptr), +levels_size_(other.levels_size_), +items_(nullptr), +items_size_(other.items_size_), +min_value_(nullptr), +max_value_(nullptr), +is_level_zero_sorted_(other.is_level_zero_sorted_) +{ + levels_ = AllocU32().allocate(levels_size_); + std::copy(&other.levels_[0], &other.levels_[levels_size_], levels_); + items_ = A().allocate(items_size_); + std::copy(&other.items_[levels_[0]], &other.items_[levels_[num_levels_]], &items_[levels_[0]]); + if (other.min_value_ != nullptr) min_value_ = new (A().allocate(1)) T(*other.min_value_); + if (other.max_value_ != nullptr) max_value_ = new (A().allocate(1)) T(*other.max_value_); +} + +template +kll_sketch::kll_sketch(kll_sketch&& other) noexcept: +k_(other.k_), +m_(other.m_), +min_k_(other.min_k_), +n_(other.n_), +num_levels_(other.num_levels_), +levels_(other.levels_), +levels_size_(other.levels_size_), +items_(other.items_), +items_size_(other.items_size_), +min_value_(other.min_value_), +max_value_(other.max_value_), +is_level_zero_sorted_(other.is_level_zero_sorted_) +{ + other.levels_ = nullptr; + other.items_ = nullptr; + other.min_value_ = nullptr; + other.max_value_ = nullptr; +} + +template +kll_sketch& kll_sketch::operator=(const kll_sketch& other) { + kll_sketch copy(other); + std::swap(k_, copy.k_); + std::swap(m_, copy.m_); + std::swap(min_k_, copy.min_k_); + std::swap(n_, copy.n_); + std::swap(num_levels_, copy.num_levels_); + std::swap(levels_, copy.levels_); + std::swap(levels_size_, copy.levels_size_); + std::swap(items_, copy.items_); + std::swap(items_size_, copy.items_size_); + std::swap(min_value_, copy.min_value_); + std::swap(max_value_, copy.max_value_); + std::swap(is_level_zero_sorted_, copy.is_level_zero_sorted_); + return *this; +} + +template +kll_sketch& kll_sketch::operator=(kll_sketch&& other) { + std::swap(k_, other.k_); + std::swap(m_, other.m_); + std::swap(min_k_, other.min_k_); + std::swap(n_, other.n_); + std::swap(num_levels_, other.num_levels_); + std::swap(levels_, other.levels_); + std::swap(levels_size_, other.levels_size_); + std::swap(items_, other.items_); + std::swap(items_size_, other.items_size_); + std::swap(min_value_, other.min_value_); + std::swap(max_value_, other.max_value_); + std::swap(is_level_zero_sorted_, other.is_level_zero_sorted_); + return *this; +} + +template +kll_sketch::~kll_sketch() { + if (items_ != nullptr) { + const uint32_t begin = levels_[0]; + const uint32_t end = levels_[num_levels_]; + for (uint32_t i = begin; i < end; i++) items_[i].~T(); + A().deallocate(items_, items_size_); + } + if (levels_ != nullptr) AllocU32().deallocate(levels_, levels_size_); + if (min_value_ != nullptr) { + min_value_->~T(); + A().deallocate(min_value_, 1); + } + if (max_value_ != nullptr) { + max_value_->~T(); + A().deallocate(max_value_, 1); + } +} + +template +void kll_sketch::update(const T& value) { + const uint32_t index = internal_update(value); + new (&items_[index]) T(value); +} + +template +void kll_sketch::update(T&& value) { + const uint32_t index = internal_update(value); + new (&items_[index]) T(std::move(value)); +} + +template +uint32_t kll_sketch::internal_update(const T& value) { + if (is_empty()) { + min_value_ = new (A().allocate(1)) T(value); + max_value_ = new (A().allocate(1)) T(value); + } else { + if (C()(value, *min_value_)) *min_value_ = value; + if (C()(*max_value_, value)) *max_value_ = value; + } + if (levels_[0] == 0) compress_while_updating(); + n_++; + is_level_zero_sorted_ = false; + return --levels_[0]; +} + +template +void kll_sketch::merge(const kll_sketch& other) { + if (other.is_empty()) return; + if (m_ != other.m_) { + throw std::invalid_argument("incompatible M: " + std::to_string(m_) + " and " + std::to_string(other.m_)); + } + const uint64_t final_n = n_ + other.n_; + for (uint32_t i = other.levels_[0]; i < other.levels_[1]; i++) { + update(other.items_[i]); + } + if (is_empty()) { + min_value_ = new (A().allocate(1)) T(*other.min_value_); + max_value_ = new (A().allocate(1)) T(*other.max_value_); + } else { + if (C()(*other.min_value_, *min_value_)) *min_value_ = *other.min_value_; + if (C()(*max_value_, *other.max_value_)) *max_value_ = *other.max_value_; + } + if (other.num_levels_ >= 2) merge_higher_levels(other, final_n); + n_ = final_n; + if (other.is_estimation_mode()) min_k_ = std::min(min_k_, other.min_k_); + assert_correct_total_weight(); +} + +template +bool kll_sketch::is_empty() const { + return n_ == 0; +} + +template +uint64_t kll_sketch::get_n() const { + return n_; +} + +template +uint32_t kll_sketch::get_num_retained() const { + return levels_[num_levels_] - levels_[0]; +} + +template +bool kll_sketch::is_estimation_mode() const { + return num_levels_ > 1; +} + +template +T kll_sketch::get_min_value() const { + if (is_empty()) return get_invalid_value(); + return *min_value_; +} + +template +T kll_sketch::get_max_value() const { + if (is_empty()) return get_invalid_value(); + return *max_value_; +} + +template +T kll_sketch::get_quantile(double fraction) const { + if (is_empty()) return get_invalid_value(); + if (fraction == 0.0) return *min_value_; + if (fraction == 1.0) return *max_value_; + if ((fraction < 0.0) or (fraction > 1.0)) { + throw std::invalid_argument("Fraction cannot be less than zero or greater than 1.0"); + } + // has side effect of sorting level zero if needed + auto quantile_calculator(const_cast(this)->get_quantile_calculator()); + return quantile_calculator->get_quantile(fraction); +} + +template +std::vector kll_sketch::get_quantiles(const double* fractions, uint32_t size) const { + std::vector quantiles; + if (is_empty()) return quantiles; + std::unique_ptr, std::function*)>> quantile_calculator; + quantiles.reserve(size); + for (uint32_t i = 0; i < size; i++) { + const double fraction = fractions[i]; + if ((fraction < 0.0) or (fraction > 1.0)) { + throw std::invalid_argument("Fraction cannot be less than zero or greater than 1.0"); + } + if (fraction == 0.0) quantiles.push_back(*min_value_); + else if (fraction == 1.0) quantiles.push_back(*max_value_); + else { + if (!quantile_calculator) { + // has side effect of sorting level zero if needed + quantile_calculator = const_cast(this)->get_quantile_calculator(); + } + quantiles.push_back(quantile_calculator->get_quantile(fraction)); + } + } + return quantiles; +} + +template +double kll_sketch::get_rank(const T& value) const { + if (is_empty()) return std::numeric_limits::quiet_NaN(); + uint8_t level(0); + uint64_t weight(1); + uint64_t total(0); + while (level < num_levels_) { + const auto from_index(levels_[level]); + const auto to_index(levels_[level + 1]); // exclusive + for (uint32_t i = from_index; i < to_index; i++) { + if (C()(items_[i], value)) { + total += weight; + } else if ((level > 0) or is_level_zero_sorted_) { + break; // levels above 0 are sorted, no point comparing further + } + } + level++; + weight *= 2; + } + return (double) total / n_; +} + +template +vector_d kll_sketch::get_PMF(const T* split_points, uint32_t size) const { + return get_PMF_or_CDF(split_points, size, false); +} + +template +vector_d kll_sketch::get_CDF(const T* split_points, uint32_t size) const { + return get_PMF_or_CDF(split_points, size, true); +} + +template +double kll_sketch::get_normalized_rank_error(bool pmf) const { + return get_normalized_rank_error(min_k_, pmf); +} + +template +void kll_sketch::serialize(std::ostream& os) const { + const bool is_single_item = n_ == 1; + const uint8_t preamble_ints(is_empty() or is_single_item ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_FULL); + os.write((char*)&preamble_ints, sizeof(preamble_ints)); + const uint8_t serial_version(is_single_item ? SERIAL_VERSION_2 : SERIAL_VERSION_1); + os.write((char*)&serial_version, sizeof(serial_version)); + const uint8_t family(FAMILY); + os.write((char*)&family, sizeof(family)); + const uint8_t flags_byte( + (is_empty() ? 1 << flags::IS_EMPTY : 0) + | (is_level_zero_sorted_ ? 1 << flags::IS_LEVEL_ZERO_SORTED : 0) + | (is_single_item ? 1 << flags::IS_SINGLE_ITEM : 0) + ); + os.write((char*)&flags_byte, sizeof(flags_byte)); + os.write((char*)&k_, sizeof(k_)); + os.write((char*)&m_, sizeof(m_)); + const uint8_t unused(0); + os.write((char*)&unused, sizeof(unused)); + if (is_empty()) return; + if (!is_single_item) { + os.write((char*)&n_, sizeof(n_)); + os.write((char*)&min_k_, sizeof(min_k_)); + os.write((char*)&num_levels_, sizeof(num_levels_)); + os.write((char*)&unused, sizeof(unused)); + os.write((char*)levels_, sizeof(levels_[0]) * num_levels_); + S().serialize(os, min_value_, 1); + S().serialize(os, max_value_, 1); + } + S().serialize(os, &items_[levels_[0]], get_num_retained()); +} + +template +vector_u8 kll_sketch::serialize(unsigned header_size_bytes) const { + const bool is_single_item = n_ == 1; + const size_t size = header_size_bytes + get_serialized_size_bytes(); + vector_u8 bytes(size); + uint8_t* ptr = bytes.data() + header_size_bytes; + const uint8_t preamble_ints(is_empty() or is_single_item ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_FULL); + ptr += copy_to_mem(&preamble_ints, ptr, sizeof(preamble_ints)); + const uint8_t serial_version(is_single_item ? SERIAL_VERSION_2 : SERIAL_VERSION_1); + ptr += copy_to_mem(&serial_version, ptr, sizeof(serial_version)); + const uint8_t family(FAMILY); + ptr += copy_to_mem(&family, ptr, sizeof(family)); + const uint8_t flags_byte( + (is_empty() ? 1 << flags::IS_EMPTY : 0) + | (is_level_zero_sorted_ ? 1 << flags::IS_LEVEL_ZERO_SORTED : 0) + | (is_single_item ? 1 << flags::IS_SINGLE_ITEM : 0) + ); + ptr += copy_to_mem(&flags_byte, ptr, sizeof(flags_byte)); + ptr += copy_to_mem(&k_, ptr, sizeof(k_)); + ptr += copy_to_mem(&m_, ptr, sizeof(m_)); + const uint8_t unused(0); + ptr += copy_to_mem(&unused, ptr, sizeof(unused)); + if (!is_empty()) { + if (!is_single_item) { + ptr += copy_to_mem(&n_, ptr, sizeof(n_)); + ptr += copy_to_mem(&min_k_, ptr, sizeof(min_k_)); + ptr += copy_to_mem(&num_levels_, ptr, sizeof(num_levels_)); + ptr += copy_to_mem(&unused, ptr, sizeof(unused)); + ptr += copy_to_mem(levels_, ptr, sizeof(levels_[0]) * num_levels_); + ptr += S().serialize(ptr, min_value_, 1); + ptr += S().serialize(ptr, max_value_, 1); + } + ptr += S().serialize(ptr, &items_[levels_[0]], get_num_retained()); + } + const size_t delta = ptr - bytes.data(); + if (delta != size) throw std::logic_error("serialized size mismatch: " + std::to_string(delta) + " != " + std::to_string(size)); + return bytes; +} + +template +kll_sketch kll_sketch::deserialize(std::istream& is) { + uint8_t preamble_ints; + is.read((char*)&preamble_ints, sizeof(preamble_ints)); + uint8_t serial_version; + is.read((char*)&serial_version, sizeof(serial_version)); + uint8_t family_id; + is.read((char*)&family_id, sizeof(family_id)); + uint8_t flags_byte; + is.read((char*)&flags_byte, sizeof(flags_byte)); + uint16_t k; + is.read((char*)&k, sizeof(k)); + uint8_t m; + is.read((char*)&m, sizeof(m)); + uint8_t unused; + is.read((char*)&unused, sizeof(unused)); + + check_m(m); + check_preamble_ints(preamble_ints, flags_byte); + check_serial_version(serial_version); + check_family_id(family_id); + + const bool is_empty(flags_byte & (1 << flags::IS_EMPTY)); + return is_empty ? kll_sketch(k) : kll_sketch(k, flags_byte, is); +} + +template +kll_sketch kll_sketch::deserialize(const void* bytes, size_t size) { + const char* ptr = static_cast(bytes); + uint8_t preamble_ints; + ptr += copy_from_mem(ptr, &preamble_ints, sizeof(preamble_ints)); + uint8_t serial_version; + ptr += copy_from_mem(ptr, &serial_version, sizeof(serial_version)); + uint8_t family_id; + ptr += copy_from_mem(ptr, &family_id, sizeof(family_id)); + uint8_t flags_byte; + ptr += copy_from_mem(ptr, &flags_byte, sizeof(flags_byte)); + uint16_t k; + ptr += copy_from_mem(ptr, &k, sizeof(k)); + uint8_t m; + ptr += copy_from_mem(ptr, &m, sizeof(m)); + + check_m(m); + check_preamble_ints(preamble_ints, flags_byte); + check_serial_version(serial_version); + check_family_id(family_id); + + const bool is_empty(flags_byte & (1 << flags::IS_EMPTY)); + return is_empty ? kll_sketch(k) : kll_sketch(k, flags_byte, bytes, size); +} + +/* + * Gets the normalized rank error given k and pmf. + * k - the configuration parameter + * pmf - if true, returns the "double-sided" normalized rank error for the get_PMF() function. + * Otherwise, it is the "single-sided" normalized rank error for all the other queries. + * Constants were derived as the best fit to 99 percentile empirically measured max error in thousands of trials + */ +template +double kll_sketch::get_normalized_rank_error(uint16_t k, bool pmf) { + return pmf + ? 2.446 / pow(k, 0.9433) + : 2.296 / pow(k, 0.9723); +} + +// for deserialization +// the common part of the preamble was read and compatibility checks were done +template +kll_sketch::kll_sketch(uint16_t k, uint8_t flags_byte, std::istream& is) { + k_ = k; + m_ = DEFAULT_M; + const bool is_single_item(flags_byte & (1 << flags::IS_SINGLE_ITEM)); // used in serial version 2 + if (is_single_item) { + n_ = 1; + min_k_ = k_; + num_levels_ = 1; + } else { + is.read((char*)&n_, sizeof(n_)); + is.read((char*)&min_k_, sizeof(min_k_)); + is.read((char*)&num_levels_, sizeof(num_levels_)); + uint8_t unused; + is.read((char*)&unused, sizeof(unused)); + } + levels_ = AllocU32().allocate(num_levels_ + 1); + levels_size_ = num_levels_ + 1; + const uint32_t capacity(kll_helper::compute_total_capacity(k_, m_, num_levels_)); + if (is_single_item) { + levels_[0] = capacity - 1; + } else { + // the last integer in levels_ is not serialized because it can be derived + is.read((char*)levels_, sizeof(levels_[0]) * num_levels_); + } + levels_[num_levels_] = capacity; + min_value_ = A().allocate(1); + max_value_ = A().allocate(1); + if (!is_single_item) { + S().deserialize(is, min_value_, 1); + S().deserialize(is, max_value_, 1); + } + items_ = A().allocate(capacity); + items_size_ = capacity; + const auto num_items = levels_[num_levels_] - levels_[0]; + S().deserialize(is, &items_[levels_[0]], num_items); + if (is_single_item) { + new (min_value_) T(items_[levels_[0]]); + new (max_value_) T(items_[levels_[0]]); + } + is_level_zero_sorted_ = (flags_byte & (1 << flags::IS_LEVEL_ZERO_SORTED)) > 0; +} + +// for deserialization +// the common part of the preamble was read and compatibility checks were done +template +kll_sketch::kll_sketch(uint16_t k, uint8_t flags_byte, const void* bytes, size_t size) { + k_ = k; + m_ = DEFAULT_M; + const bool is_single_item(flags_byte & (1 << flags::IS_SINGLE_ITEM)); // used in serial version 2 + const char* ptr = static_cast(bytes) + DATA_START_SINGLE_ITEM; + if (is_single_item) { + n_ = 1; + min_k_ = k_; + num_levels_ = 1; + } else { + ptr += copy_from_mem(ptr, &n_, sizeof(n_)); + ptr += copy_from_mem(ptr, &min_k_, sizeof(min_k_)); + ptr += copy_from_mem(ptr, &num_levels_, sizeof(num_levels_)); + ptr++; // skip unused byte + } + levels_ = AllocU32().allocate(num_levels_ + 1); + levels_size_ = num_levels_ + 1; + const uint32_t capacity(kll_helper::compute_total_capacity(k_, m_, num_levels_)); + if (is_single_item) { + levels_[0] = capacity - 1; + } else { + // the last integer in levels_ is not serialized because it can be derived + ptr += copy_from_mem(ptr, levels_, sizeof(levels_[0]) * num_levels_); + } + levels_[num_levels_] = capacity; + min_value_ = A().allocate(1); + max_value_ = A().allocate(1); + if (!is_single_item) { + ptr += S().deserialize(ptr, min_value_, 1); + ptr += S().deserialize(ptr, max_value_, 1); + } + items_ = A().allocate(capacity); + items_size_ = capacity; + const auto num_items(levels_[num_levels_] - levels_[0]); + ptr += S().deserialize(ptr, &items_[levels_[0]], num_items); + if (is_single_item) { + new (min_value_) T(items_[levels_[0]]); + new (max_value_) T(items_[levels_[0]]); + } + is_level_zero_sorted_ = (flags_byte & (1 << flags::IS_LEVEL_ZERO_SORTED)) > 0; + const size_t delta = ptr - static_cast(bytes); + if (delta != size) throw std::logic_error("deserialized size mismatch: " + std::to_string(delta) + " != " + std::to_string(size)); +} + +// The following code is only valid in the special case of exactly reaching capacity while updating. +// It cannot be used while merging, while reducing k, or anything else. +template +void kll_sketch::compress_while_updating(void) { + const uint8_t level = find_level_to_compact(); + + // It is important to add the new top level right here. Be aware that this operation + // grows the buffer and shifts the data and also the boundaries of the data and grows the + // levels array and increments num_levels_ + if (level == (num_levels_ - 1)) { + add_empty_top_level_to_completely_full_sketch(); + } + + const uint32_t raw_beg = levels_[level]; + const uint32_t raw_lim = levels_[level + 1]; + // +2 is OK because we already added a new top level if necessary + const uint32_t pop_above = levels_[level + 2] - raw_lim; + const uint32_t raw_pop = raw_lim - raw_beg; + const bool odd_pop = kll_helper::is_odd(raw_pop); + const uint32_t adj_beg = odd_pop ? raw_beg + 1 : raw_beg; + const uint32_t adj_pop = odd_pop ? raw_pop - 1 : raw_pop; + const uint32_t half_adj_pop = adj_pop / 2; + const uint32_t destroy_beg = levels_[0]; + + // level zero might not be sorted, so we must sort it if we wish to compact it + // sort_level_zero() is not used here because of the adjustment for odd number of items + if ((level == 0) and !is_level_zero_sorted_) { + std::sort(&items_[adj_beg], &items_[adj_beg + adj_pop], C()); + } + if (pop_above == 0) { + kll_helper::randomly_halve_up(items_, adj_beg, adj_pop); + } else { + kll_helper::randomly_halve_down(items_, adj_beg, adj_pop); + kll_helper::merge_sorted_arrays(items_, adj_beg, half_adj_pop, raw_lim, pop_above, adj_beg + half_adj_pop); + } + levels_[level + 1] -= half_adj_pop; // adjust boundaries of the level above + if (odd_pop) { + levels_[level] = levels_[level + 1] - 1; // the current level now contains one item + if (levels_[level] != raw_beg) items_[levels_[level]] = std::move(items_[raw_beg]); // namely this leftover guy + } else { + levels_[level] = levels_[level + 1]; // the current level is now empty + } + + // verify that we freed up half_adj_pop array slots just below the current level + assert (levels_[level] == (raw_beg + half_adj_pop)); + + // finally, we need to shift up the data in the levels below + // so that the freed-up space can be used by level zero + if (level > 0) { + const uint32_t amount = raw_beg - levels_[0]; + std::move_backward(&items_[levels_[0]], &items_[levels_[0] + amount], &items_[levels_[0] + half_adj_pop + amount]); + for (uint8_t lvl = 0; lvl < level; lvl++) levels_[lvl] += half_adj_pop; + } + for (uint32_t i = 0; i < half_adj_pop; i++) items_[i + destroy_beg].~T(); +} + +template +uint8_t kll_sketch::find_level_to_compact() const { + uint8_t level = 0; + while (true) { + assert (level < num_levels_); + const uint32_t pop = levels_[level + 1] - levels_[level]; + const uint32_t cap = kll_helper::level_capacity(k_, num_levels_, level, m_); + if (pop >= cap) { + return level; + } + level++; + } +} + +template +void kll_sketch::add_empty_top_level_to_completely_full_sketch() { + const uint32_t cur_total_cap = levels_[num_levels_]; + + // make sure that we are following a certain growth scheme + assert (levels_[0] == 0); + assert (items_size_ == cur_total_cap); + + // note that merging MIGHT over-grow levels_, in which case we might not have to grow it here + const uint8_t new_levels_size = num_levels_ + 2; + if (levels_size_ < new_levels_size) { + uint32_t* new_levels = AllocU32().allocate(new_levels_size); + std::copy(&levels_[0], &levels_[levels_size_], new_levels); + AllocU32().deallocate(levels_, levels_size_); + levels_ = new_levels; + levels_size_ = new_levels_size; + } + + const uint32_t delta_cap = kll_helper::level_capacity(k_, num_levels_ + 1, 0, m_); + const uint32_t new_total_cap = cur_total_cap + delta_cap; + + // move (and shift) the current data into the new buffer + T* new_buf = A().allocate(new_total_cap); + kll_helper::move_construct(items_, 0, cur_total_cap, new_buf, delta_cap, true); + A().deallocate(items_, items_size_); + items_ = new_buf; + items_size_ = new_total_cap; + + // this loop includes the old "extra" index at the top + for (uint8_t i = 0; i <= num_levels_; i++) { + levels_[i] += delta_cap; + } + + assert (levels_[num_levels_] == new_total_cap); + + num_levels_++; + levels_[num_levels_] = new_total_cap; // initialize the new "extra" index at the top +} + +template +void kll_sketch::sort_level_zero() { + if (!is_level_zero_sorted_) { + std::sort(&items_[levels_[0]], &items_[levels_[1]], C()); + is_level_zero_sorted_ = true; + } +} + +template +std::unique_ptr, std::function*)>> kll_sketch::get_quantile_calculator() { + sort_level_zero(); + typedef typename std::allocator_traits::template rebind_alloc> AllocCalc; + std::unique_ptr, std::function*)>> quantile_calculator( + new (AllocCalc().allocate(1)) kll_quantile_calculator(items_, levels_, num_levels_, n_), + [](kll_quantile_calculator* ptr){ ptr->~kll_quantile_calculator(); AllocCalc().deallocate(ptr, 1); } + ); + return quantile_calculator; +} + +template +vector_d kll_sketch::get_PMF_or_CDF(const T* split_points, uint32_t size, bool is_CDF) const { + if (is_empty()) return vector_d(); + kll_helper::validate_values(split_points, size); + vector_d buckets(size + 1, 0); + uint8_t level(0); + uint64_t weight(1); + while (level < num_levels_) { + const auto from_index = levels_[level]; + const auto to_index = levels_[level + 1]; // exclusive + if ((level == 0) and !is_level_zero_sorted_) { + increment_buckets_unsorted_level(from_index, to_index, weight, split_points, size, buckets.data()); + } else { + increment_buckets_sorted_level(from_index, to_index, weight, split_points, size, buckets.data()); + } + level++; + weight *= 2; + } + // normalize and, if CDF, convert to cumulative + if (is_CDF) { + double subtotal = 0; + for (uint32_t i = 0; i <= size; i++) { + subtotal += buckets[i]; + buckets[i] = subtotal / n_; + } + } else { + for (uint32_t i = 0; i <= size; i++) { + buckets[i] /= n_; + } + } + return buckets; +} + +template +void kll_sketch::increment_buckets_unsorted_level(uint32_t from_index, uint32_t to_index, uint64_t weight, + const T* split_points, uint32_t size, double* buckets) const +{ + for (uint32_t i = from_index; i < to_index; i++) { + uint32_t j; + for (j = 0; j < size; j++) { + if (C()(items_[i], split_points[j])) { + break; + } + } + buckets[j] += weight; + } +} + +template +void kll_sketch::increment_buckets_sorted_level(uint32_t from_index, uint32_t to_index, uint64_t weight, + const T* split_points, uint32_t size, double* buckets) const +{ + uint32_t i = from_index; + uint32_t j = 0; + while ((i < to_index) and (j < size)) { + if (C()(items_[i], split_points[j])) { + buckets[j] += weight; // this sample goes into this bucket + i++; // move on to next sample and see whether it also goes into this bucket + } else { + j++; // no more samples for this bucket + } + } + // now either i == to_index (we are out of samples), or + // j == size (we are out of buckets, but there are more samples remaining) + // we only need to do something in the latter case + if (j == size) { + buckets[j] += weight * (to_index - i); + } +} + +template +void kll_sketch::merge_higher_levels(const kll_sketch& other, uint64_t final_n) { + const uint32_t tmp_num_items = get_num_retained() + other.get_num_retained_above_level_zero(); + auto tmp_items_deleter = [tmp_num_items](T* ptr) { A().deallocate(ptr, tmp_num_items); }; // no destructor needed + const std::unique_ptr workbuf(A().allocate(tmp_num_items), tmp_items_deleter); + const uint8_t ub = kll_helper::ub_on_num_levels(final_n); + const size_t work_levels_size = ub + 2; // ub+1 does not work + auto tmp_levels_deleter = [work_levels_size](uint32_t* ptr) { AllocU32().deallocate(ptr, work_levels_size); }; + const std::unique_ptr worklevels(AllocU32().allocate(work_levels_size), tmp_levels_deleter); + const std::unique_ptr outlevels(AllocU32().allocate(work_levels_size), tmp_levels_deleter); + + const uint8_t provisional_num_levels = std::max(num_levels_, other.num_levels_); + + populate_work_arrays(other, workbuf.get(), worklevels.get(), provisional_num_levels); + + const kll_helper::compress_result result = kll_helper::general_compress(k_, m_, provisional_num_levels, workbuf.get(), + worklevels.get(), outlevels.get(), is_level_zero_sorted_); + + // ub can sometimes be much bigger + if (result.final_num_levels > ub) throw std::logic_error("merge error"); + + // now we need to transfer the results back into "this" sketch + if (result.final_capacity != items_size_) { + A().deallocate(items_, items_size_); + items_size_ = result.final_capacity; + items_ = A().allocate(items_size_); + } + const uint32_t free_space_at_bottom = result.final_capacity - result.final_num_items; + kll_helper::move_construct(workbuf.get(), outlevels[0], outlevels[0] + result.final_num_items, items_, free_space_at_bottom, true); + + if (levels_size_ < (result.final_num_levels + 1)) { + AllocU32().deallocate(levels_, levels_size_); + levels_size_ = result.final_num_levels + 1; + levels_ = AllocU32().allocate(levels_size_); + } + const uint32_t offset = free_space_at_bottom - outlevels[0]; + for (uint8_t lvl = 0; lvl < levels_size_; lvl++) { // includes the "extra" index + levels_[lvl] = outlevels[lvl] + offset; + } + num_levels_ = result.final_num_levels; +} + +// this leaves items_ uninitialized (all objects moved out and destroyed) +template +void kll_sketch::populate_work_arrays(const kll_sketch& other, T* workbuf, uint32_t* worklevels, uint8_t provisional_num_levels) { + worklevels[0] = 0; + + // the level zero data from "other" was already inserted into "this" + kll_helper::move_construct(items_, levels_[0], levels_[1], workbuf, 0, true); + worklevels[1] = safe_level_size(0); + + for (uint8_t lvl = 1; lvl < provisional_num_levels; lvl++) { + const uint32_t self_pop = safe_level_size(lvl); + const uint32_t other_pop = other.safe_level_size(lvl); + worklevels[lvl + 1] = worklevels[lvl] + self_pop + other_pop; + + if ((self_pop > 0) and (other_pop == 0)) { + kll_helper::move_construct(items_, levels_[lvl], levels_[lvl] + self_pop, workbuf, worklevels[lvl], true); + } else if ((self_pop == 0) and (other_pop > 0)) { + kll_helper::copy_construct(other.items_, other.levels_[lvl], other.levels_[lvl] + other_pop, workbuf, worklevels[lvl]); + } else if ((self_pop > 0) and (other_pop > 0)) { + kll_helper::merge_sorted_arrays(items_, levels_[lvl], self_pop, other.items_, other.levels_[lvl], other_pop, workbuf, worklevels[lvl]); + } + } +} + +template +void kll_sketch::assert_correct_total_weight() const { + const uint64_t total(kll_helper::sum_the_sample_weights(num_levels_, levels_)); + if (total != n_) { + throw std::logic_error("Total weight does not match N"); + } +} + +template +uint32_t kll_sketch::safe_level_size(uint8_t level) const { + if (level >= num_levels_) return 0; + return levels_[level + 1] - levels_[level]; +} + +template +uint32_t kll_sketch::get_num_retained_above_level_zero() const { + if (num_levels_ == 1) return 0; + return levels_[num_levels_] - levels_[1]; +} + +template +void kll_sketch::check_m(uint8_t m) { + if (m != DEFAULT_M) { + throw std::invalid_argument("Possible corruption: M must be " + std::to_string(DEFAULT_M) + + ": " + std::to_string(m)); + } +} + +template +void kll_sketch::check_preamble_ints(uint8_t preamble_ints, uint8_t flags_byte) { + const bool is_empty(flags_byte & (1 << flags::IS_EMPTY)); + const bool is_single_item(flags_byte & (1 << flags::IS_SINGLE_ITEM)); + if (is_empty or is_single_item) { + if (preamble_ints != PREAMBLE_INTS_SHORT) { + throw std::invalid_argument("Possible corruption: preamble ints must be " + + std::to_string(PREAMBLE_INTS_SHORT) + " for an empty or single item sketch: " + std::to_string(preamble_ints)); + } + } else { + if (preamble_ints != PREAMBLE_INTS_FULL) { + throw std::invalid_argument("Possible corruption: preamble ints must be " + + std::to_string(PREAMBLE_INTS_FULL) + " for a sketch with more than one item: " + std::to_string(preamble_ints)); + } + } +} + +template +void kll_sketch::check_serial_version(uint8_t serial_version) { + if (serial_version != SERIAL_VERSION_1 and serial_version != SERIAL_VERSION_2) { + throw std::invalid_argument("Possible corruption: serial version mismatch: expected " + + std::to_string(SERIAL_VERSION_1) + " or " + std::to_string(SERIAL_VERSION_2) + + ", got " + std::to_string(serial_version)); + } +} + +template +void kll_sketch::check_family_id(uint8_t family_id) { + if (family_id != FAMILY) { + throw std::invalid_argument("Possible corruption: family mismatch: expected " + + std::to_string(FAMILY) + ", got " + std::to_string(family_id)); + } +} + +template +void kll_sketch::to_stream(std::ostream& os, bool print_levels, bool print_items) const { + os << "### KLL sketch summary:" << std::endl; + os << " K : " << k_ << std::endl; + os << " min K : " << min_k_ << std::endl; + os << " M : " << (unsigned int) m_ << std::endl; + os << " N : " << n_ << std::endl; + os << " Epsilon : " << std::setprecision(3) << get_normalized_rank_error(false) * 100 << "%" << std::endl; + os << " Epsilon PMF : " << get_normalized_rank_error(true) * 100 << "%" << std::endl; + os << " Empty : " << (is_empty() ? "true" : "false") << std::endl; + os << " Estimation mode: " << (is_estimation_mode() ? "true" : "false") << std::endl; + os << " Levels : " << (unsigned int) num_levels_ << std::endl; + os << " Sorted : " << (is_level_zero_sorted_ ? "true" : "false") << std::endl; + os << " Capacity items : " << items_size_ << std::endl; + os << " Retained items : " << get_num_retained() << std::endl; + os << " Storage bytes : " << get_serialized_size_bytes() << std::endl; + if (!is_empty()) { + os << " Min value : " << *min_value_ << std::endl; + os << " Max value : " << *max_value_ << std::endl; + } + os << "### End sketch summary" << std::endl; + + // for debugging + const bool with_levels(false); + const bool with_data(false); + + if (with_levels) { + os << "### KLL sketch levels:" << std::endl; + os << " index: nominal capacity, actual size" << std::endl; + for (uint8_t i = 0; i < num_levels_; i++) { + os << " " << (unsigned int) i << ": " << kll_helper::level_capacity(k_, num_levels_, i, m_) << ", " << safe_level_size(i) << std::endl; + } + os << "### End sketch levels" << std::endl; + } + + if (with_data) { + os << "### KLL sketch data:" << std::endl; + uint8_t level(0); + while (level < num_levels_) { + const uint32_t from_index = levels_[level]; + const uint32_t to_index = levels_[level + 1]; // exclusive + if (from_index < to_index) { + os << " level " << (unsigned int) level << ":" << std::endl; + } + for (uint32_t i = from_index; i < to_index; i++) { + os << " " << items_[i] << std::endl; + } + level++; + } + os << "### End sketch data" << std::endl; + } +} + +template +typename kll_sketch::const_iterator kll_sketch::begin() const { + return kll_sketch::const_iterator(items_, levels_, num_levels_); +} + +template +typename kll_sketch::const_iterator kll_sketch::end() const { + return kll_sketch::const_iterator(nullptr, nullptr, num_levels_); +} + +// kll_sketch::const_iterator implementation + +template +kll_sketch::const_iterator::const_iterator(const T* items, const uint32_t* levels, const uint8_t num_levels): +items(items), levels(levels), num_levels(num_levels), index(levels == nullptr ? 0 : levels[0]), level(levels == nullptr ? num_levels : 0), weight(1) +{} + +template +kll_sketch::const_iterator::const_iterator(const const_iterator& other): +items(other.items), levels(other.levels), num_levels(other.num_levels), index(other.index), level(other.level), weight(other.weight) +{} + +template +typename kll_sketch::const_iterator& kll_sketch::const_iterator::operator++() { + ++index; + if (index == levels[level + 1]) { // go to the next non-empty level + do { + ++level; + weight *= 2; + } while (level < num_levels and levels[level] == levels[level + 1]); + } + return *this; +} + +template +typename kll_sketch::const_iterator& kll_sketch::const_iterator::operator++(int) { + const_iterator tmp(*this); + operator++(); + return tmp; +} + +template +bool kll_sketch::const_iterator::operator==(const const_iterator& other) const { + if (level != other.level) return false; + if (level == num_levels) return true; // end + return index == other.index; +} + +template +bool kll_sketch::const_iterator::operator!=(const const_iterator& other) const { + return !operator==(other); +} + +template +const std::pair kll_sketch::const_iterator::operator*() const { + return std::pair(items[index], weight); +} + +} /* namespace datasketches */ + +#endif diff --git a/3rd/datasketches/datasketches/serde.hpp b/3rd/datasketches/datasketches/serde.hpp new file mode 100644 index 000000000..4ed6e71cc --- /dev/null +++ b/3rd/datasketches/datasketches/serde.hpp @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef DATASKETCHES_SERDE_HPP_ +#define DATASKETCHES_SERDE_HPP_ + +#include +#include +#include +#include + +namespace datasketches { + +// serialize and deserialize +template struct serde { + // stream serialization + void serialize(std::ostream& os, const T* items, unsigned num); + void deserialize(std::istream& is, T* items, unsigned num); // items are not initialized + + // raw bytes serialization + size_t size_of_item(const T& item); + size_t serialize(void* ptr, const T* items, unsigned num); + size_t deserialize(const void* ptr, T* items, unsigned num); // items are not initialized +}; + +// serde for all fixed-size arithmetic types (int and float of different sizes) +// in particular, kll_sketch should produce sketches binary-compatible +// with LongsSketch and ItemsSketch with ArrayOfLongsSerDe in Java +template +struct serde::value>::type> { + void serialize(std::ostream& os, const T* items, unsigned num) { + os.write((char*)items, sizeof(T) * num); + } + void deserialize(std::istream& is, T* items, unsigned num) { + is.read((char*)items, sizeof(T) * num); + } + size_t size_of_item(T item) { + return sizeof(T); + } + size_t serialize(void* ptr, const T* items, unsigned num) { + memcpy(ptr, items, sizeof(T) * num); + return sizeof(int32_t) * num; + } + size_t deserialize(const void* ptr, T* items, unsigned num) { + memcpy(items, ptr, sizeof(T) * num); + return sizeof(T) * num; + } +}; + +// serde for std::string items +// This should produce sketches binary-compatible with +// ItemsSketch with ArrayOfStringsSerDe in Java. +// The length of each string is stored as a 32-bit integer (historically), +// which may be too wasteful. Treat this as an example. +template<> +struct serde { + void serialize(std::ostream& os, const std::string* items, unsigned num) { + for (unsigned i = 0; i < num; i++) { + uint32_t length = items[i].size(); + os.write((char*)&length, sizeof(length)); + os.write(items[i].c_str(), length); + } + } + void deserialize(std::istream& is, std::string* items, unsigned num) { + for (unsigned i = 0; i < num; i++) { + uint32_t length; + is.read((char*)&length, sizeof(length)); + new (&items[i]) std::string; + items[i].reserve(length); + auto it = std::istreambuf_iterator(is); + for (uint32_t j = 0; j < length; j++) { + items[i].push_back(*it); + ++it; + } + } + } + size_t size_of_item(const std::string& item) { + return sizeof(uint32_t) + item.size(); + } + size_t serialize(void* ptr, const std::string* items, unsigned num) { + size_t size = sizeof(uint32_t) * num; + for (unsigned i = 0; i < num; i++) { + uint32_t length = items[i].size(); + memcpy(ptr, &length, sizeof(length)); + ptr = static_cast(ptr) + sizeof(uint32_t); + memcpy(ptr, items[i].c_str(), length); + ptr = static_cast(ptr) + length; + size += length; + } + return size; + } + size_t deserialize(const void* ptr, std::string* items, unsigned num) { + size_t size = sizeof(uint32_t) * num; + for (unsigned i = 0; i < num; i++) { + uint32_t length; + memcpy(&length, ptr, sizeof(length)); + ptr = static_cast(ptr) + sizeof(uint32_t); + new (&items[i]) std::string(static_cast(ptr), length); + ptr = static_cast(ptr) + length; + size += length; + } + return size; + } +}; + +static inline size_t copy_from_mem(const void* src, void* dst, size_t size) { + memcpy(dst, src, size); + return size; +} + +static inline size_t copy_to_mem(const void* src, void* dst, size_t size) { + memcpy(dst, src, size); + return size; +} + +} /* namespace datasketches */ + +# endif diff --git a/3rd/docopt/docopt.cpp b/3rd/docopt/docopt.cpp new file mode 100644 index 000000000..e875d2fcd --- /dev/null +++ b/3rd/docopt/docopt.cpp @@ -0,0 +1,685 @@ +// +// docopt.cpp +// docopt +// +// Created by Jared Grubb on 2013-11-03. +// Copyright (c) 2013 Jared Grubb. All rights reserved. +// + +#include "docopt.h" +#include "docopt_util.h" +#include "docopt_private.h" + +#include "docopt_value.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace docopt; + +DOCOPT_INLINE +std::ostream& docopt::operator<<(std::ostream& os, value const& val) +{ + if (val.isBool()) { + bool b = val.asBool(); + os << (b ? "true" : "false"); + } else if (val.isLong()) { + long v = val.asLong(); + os << v; + } else if (val.isString()) { + std::string const& str = val.asString(); + os << '"' << str << '"'; + } else if (val.isStringList()) { + auto const& list = val.asStringList(); + os << "["; + bool first = true; + for(auto const& el : list) { + if (first) { + first = false; + } else { + os << ", "; + } + os << '"' << el << '"'; + } + os << "]"; + } else { + os << "null"; + } + return os; +} + +#pragma mark - +#pragma mark Parsing stuff + +class Tokens { +public: + Tokens(std::vector tokens, bool isParsingArgv = true) + : fTokens(std::move(tokens)), + fIsParsingArgv(isParsingArgv) + {} + + explicit operator bool() const { + return fIndex < fTokens.size(); + } + + static Tokens from_pattern(std::string const& source) { + static const std::regex re_separators { + "(?:\\s*)" // any spaces (non-matching subgroup) + "(" + "[\\[\\]\\(\\)\\|]" // one character of brackets or parens or pipe character + "|" + "\\.\\.\\." // elipsis + ")" }; + + static const std::regex re_strings { + "(?:\\s*)" // any spaces (non-matching subgroup) + "(" + "\\S*<.*?>" // strings, but make sure to keep "< >" strings together + "|" + "[^<>\\s]+" // string without <> + ")" }; + + // We do two stages of regex matching. The '[]()' and '...' are strong delimeters + // and need to be split out anywhere they occur (even at the end of a token). We + // first split on those, and then parse the stuff between them to find the string + // tokens. This is a little harder than the python version, since they have regex.split + // and we dont have anything like that. + + std::vector tokens; + std::for_each(std::sregex_iterator{ source.begin(), source.end(), re_separators }, + std::sregex_iterator{}, + [&](std::smatch const& match) + { + // handle anything before the separator (this is the "stuff" between the delimeters) + if (match.prefix().matched) { + std::for_each(std::sregex_iterator{match.prefix().first, match.prefix().second, re_strings}, + std::sregex_iterator{}, + [&](std::smatch const& m) + { + tokens.push_back(m[1].str()); + }); + } + + // handle the delimter token itself + if (match[1].matched) { + tokens.push_back(match[1].str()); + } + }); + + return Tokens(tokens, false); + } + + std::string const& current() const { + if (*this) + return fTokens[fIndex]; + + static std::string const empty; + return empty; + } + + std::string the_rest() const { + if (!*this) + return {}; + return join(fTokens.begin()+static_cast(fIndex), + fTokens.end(), + " "); + } + + std::string pop() { + return std::move(fTokens.at(fIndex++)); + } + + bool isParsingArgv() const { return fIsParsingArgv; } + + struct OptionError : std::runtime_error { using runtime_error::runtime_error; }; + +private: + std::vector fTokens; + size_t fIndex = 0; + bool fIsParsingArgv; +}; + +// Get all instances of 'T' from the pattern +template +std::vector flat_filter(Pattern& pattern) { + std::vector flattened = pattern.flat([](Pattern const* p) -> bool { + return dynamic_cast(p) != nullptr; + }); + + // now, we're guaranteed to have T*'s, so just use static_cast + std::vector ret; + std::transform(flattened.begin(), flattened.end(), std::back_inserter(ret), [](Pattern* p) { + return static_cast(p); + }); + return ret; +} + +static std::vector parse_section(std::string const& name, std::string const& source) { + // ECMAScript regex only has "?=" for a non-matching lookahead. In order to make sure we always have + // a newline to anchor our matching, we have to avoid matching the final newline of each grouping. + // Therefore, our regex is adjusted from the docopt Python one to use ?= to match the newlines before + // the following lines, rather than after. + std::regex const re_section_pattern { + "(?:^|\\n)" // anchored at a linebreak (or start of string) + "(" + "[^\\n]*" + name + "[^\\n]*(?=\\n?)" // a line that contains the name + "(?:\\n[ \\t].*?(?=\\n|$))*" // followed by any number of lines that are indented + ")", + std::regex::icase + }; + + std::vector ret; + std::for_each(std::sregex_iterator(source.begin(), source.end(), re_section_pattern), + std::sregex_iterator(), + [&](std::smatch const& match) + { + ret.push_back(trim(match[1].str())); + }); + + return ret; +} + +static bool is_argument_spec(std::string const& token) { + if (token.empty()) + return false; + + if (token[0]=='<' && token[token.size()-1]=='>') + return true; + + if (std::all_of(token.begin(), token.end(), &::isupper)) + return true; + + return false; +} + +template +std::vector longOptions(I iter, I end) { + std::vector ret; + std::transform(iter, end, + std::back_inserter(ret), + [](typename I::reference opt) { return opt->longOption(); }); + return ret; +} + +static PatternList parse_long(Tokens& tokens, std::vector