diff --git a/.gitignore b/.gitignore index d626575e4f3..4cc78758574 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.swp +build/* core/bin/* core/obj/* core/lib/* diff --git a/.travis.yml b/.travis.yml index 9c5119c0a5b..1e332ddc499 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,10 @@ env: install: # Install cmake, lcov, MPICH - sudo apt-get -y install cmake lcov mpich - # Install zlib, OpenSSL, Gtest and LZ4 - - sudo apt-get -y install zlib1g-dev libssl-dev libgtest-dev liblz4-dev + # Install zlib, bzip2 and LZ4 + - sudo apt-get -y install zlib1g-dev libbz2-dev liblz4-dev + # Install OpenSSL and Gtest + - sudo apt-get -y install libssl-dev libgtest-dev # Install Zstandard - wget https://github.com/facebook/zstd/archive/v1.0.0.tar.gz - tar xf v1.0.0.tar.gz diff --git a/CMakeLists.txt b/CMakeLists.txt index 66490247787..a3399b33772 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,6 +79,9 @@ set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${ZSTD_LIBRARIES}) find_package(OpenSSL REQUIRED) include_directories(${OPENSSL_INCLUDE_DIR}) set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${OPENSSL_LIBRARIES}) +find_package(Bzip2 REQUIRED) +include_directories(${BZIP2_INCLUDE_DIR}) +set(TILEDB_LIB_DEPENDENCIES ${TILEDB_LIB_DEPENDENCIES} ${BZIP2_LIBRARIES}) # Find optional library dependencies find_package(GTest) diff --git a/README.md b/README.md index 75ba989f8ed..7851e20cda6 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,9 @@ # TileDB -[![Travis](https://img.shields.io/travis/Intel-HLS/TileDB.svg?maxAge=2592000)] -(https://travis-ci.org/Intel-HLS/TileDB) +[![Travis](https://travis-ci.org/TileDB-Inc/TileDB.svg?branch=master)](https://travis-ci.org/TileDB-Inc/TileDB) The installation guide for TileDB can be found at this [Github -Wiki](https://github.com/Intel-HLS/TileDB/wiki). +Wiki](https://github.com/TileDB-Inc/TileDB/wiki). The TileDB C API documentation and tutorials can be -found at the [TileDB official website](http://istc-bigdata.org/tiledb). +found at the [TileDB official website](http://www.tiledb.org). diff --git a/cmake/Modules/FindBzip2.cmake b/cmake/Modules/FindBzip2.cmake new file mode 100644 index 00000000000..d09b887210d --- /dev/null +++ b/cmake/Modules/FindBzip2.cmake @@ -0,0 +1,64 @@ +# +# FindBzip2.cmake +# +# +# The MIT License +# +# Copyright (c) 2016 MIT and Intel Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# Finds the Bzip2 library. This module defines: +# - BZIP2_INCLUDE_DIR, directory containing headers +# - BZIP2_LIBRARIES, the Bzip2 library path +# - BZIP2_FOUND, whether Bzip2 has been found + +# Find header files +if(BZIP2_SEARCH_HEADER_PATHS) + find_path( + BZIP2_INCLUDE_DIR bzip2.h + PATHS ${BZIP2_SEARCH_HEADER_PATHS} + NO_DEFAULT_PATH + ) +else() + find_path(BZIP2_INCLUDE_DIR bzlib.h) +endif() + +# Find library +if(BZIP2_SEARCH_LIB_PATH) + find_library( + BZIP2_LIBRARIES NAMES bz2 + PATHS ${BZIP2_SEARCH_LIB_PATH}$ + NO_DEFAULT_PATH + ) +else() + find_library(BZIP2_LIBRARIES NAMES bz2) +endif() + +if(BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES) + message(STATUS "Found Bzip2: ${BZIP2_LIBRARIES}") + set(BZIP2_FOUND TRUE) +else() + set(BZIP2_FOUND FALSE) +endif() + +if(BZIP2_FIND_REQUIRED AND NOT BZIP2_FOUND) + message(FATAL_ERROR "Could not find the Bzip2 library.") +endif() + diff --git a/core/include/array/array_schema.h b/core/include/array/array_schema.h index 73a0db6556e..9dbef8e7797 100644 --- a/core/include/array/array_schema.h +++ b/core/include/array/array_schema.h @@ -35,7 +35,6 @@ #include "array_schema_c.h" #include "metadata_schema_c.h" -#include "hilbert_curve.h" #include #include #include @@ -416,7 +415,6 @@ class ArraySchema { * Sets the tile order. Supported tile orders. * - TILEDB_ROW_MAJOR * - TILEDB_COL_MAJOR - * - TILEDB_HILBSERT * * @param tile_order The tile order. * @return TILEDB_AS_OK for success, and TILEDB_AS_ERR for error. @@ -432,12 +430,24 @@ class ArraySchema { * - TILEDB_INT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 + * - TILEDB_INT8 + * - TILEDB_UINT8 + * - TILEDB_INT16 + * - TILEDB_UINT16 + * - TILEDB_UINT32 + * - TILEDB_UINT64 * * The supported types for the coordinates are: * - TILEDB_INT32 * - TILEDB_INT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 + * - TILEDB_INT8 + * - TILEDB_UINT8 + * - TILEDB_INT16 + * - TILEDB_UINT16 + * - TILEDB_UINT32 + * - TILEDB_UINT64 * * @param types The types. * @return TILEDB_AS_OK for success, and TILEDB_AS_ERR for error. @@ -609,16 +619,6 @@ class ArraySchema { template void get_tile_subarray(const T* tile_coords, T* tile_subarray) const; - /** - * Returns the Hilbert id of the input coordinates. - * - * @tparam T The coordinates type. - * @param coords The coordinates for which the Hilbert id is computed. - * @return The computed Hilbert id. - */ - template - int64_t hilbert_id(const T* coords) const; - /** * Checks the order of the input coordinates. First the tile order is checked * (which, in case of non-regular tiles, is always the same), breaking the @@ -695,7 +695,6 @@ class ArraySchema { * The cell order. It can be one of the following: * - TILEDB_ROW_MAJOR * - TILEDB_COL_MAJOR - * - TILEDB_HILBERT. */ int cell_order_; /** Stores the size of every attribute (plus coordinates in the end). */ @@ -721,10 +720,9 @@ class ArraySchema { * - TILEDB_BLOSC_ZLIB * - TILEDB_BLOSC_ZSTD * - TILEDB_RLE + * - TILEDB_BZIP2 */ std::vector compression_; - /** Auxiliary variable used when calculating Hilbert ids. */ - int* coords_for_hilbert_; /** The size (in bytes) of the coordinates. */ size_t coords_size_; /** @@ -742,13 +740,6 @@ class ArraySchema { * type. */ void* domain_; - /** - * Number of bits used for the calculation of cell ids with the - * Hilbert curve. - */ - int hilbert_bits_; - /** A Hilbert curve object for finding cell ids. */ - HilbertCurve* hilbert_curve_; /** * The array domain. It should contain one [lower, upper] pair per dimension. * The type of the values stored in this buffer should match the coordinates @@ -775,7 +766,7 @@ class ArraySchema { /** * The tile order. It can be one of the following: * - TILEDB_ROW_MAJOR - * - TILEDB_COL_MAJOR. + * - TILEDB_COL_MAJOR */ int tile_order_; /** @@ -786,12 +777,24 @@ class ArraySchema { * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 * - TILEDB_CHAR + * - TILEDB_INT8 + * - TILEDB_UINT8 + * - TILEDB_INT16 + * - TILEDB_UINT16 + * - TILEDB_UINT32 + * - TILEDB_UINT64 * * The coordinate type can be one of the following: * - TILEDB_INT32 * - TILEDB_INT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 + * - TILEDB_INT8 + * - TILEDB_UINT8 + * - TILEDB_INT16 + * - TILEDB_UINT16 + * - TILEDB_UINT32 + * - TILEDB_UINT64 */ std::vector types_; /** Stores the size of every attribute type (plus coordinates in the end). */ @@ -831,15 +834,6 @@ class ArraySchema { /** Computes and returns the size of an attribute (or coordinates). */ size_t compute_cell_size(int attribute_id) const; - /** - * Computes the number of bits per dimension required by the Hilbert curve. - * - * @tparam T The domain type. - * @return void - */ - template - void compute_hilbert_bits(); - /** * Computes the tile domain. Applicable only to arrays with regular tiles. * diff --git a/core/include/array/array_schema_c.h b/core/include/array/array_schema_c.h index d365236e1cc..ad94ea3e743 100644 --- a/core/include/array/array_schema_c.h +++ b/core/include/array/array_schema_c.h @@ -55,7 +55,6 @@ typedef struct ArraySchemaC { * The cell order. It can be one of the following: * - TILEDB_ROW_MAJOR * - TILEDB_COL_MAJOR - * - TILEDB_HILBERT. */ int cell_order_; /** @@ -79,6 +78,7 @@ typedef struct ArraySchemaC { * - TILEDB_BLOSC_ZLIB * - TILEDB_BLOSC_ZSTD * - TILEDB_RLE + * - TILEDB_BZIP2 */ int* compression_; /** @@ -117,12 +117,24 @@ typedef struct ArraySchemaC { * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 * - TILEDB_CHAR + * - TILEDB_INT8 + * - TILEDB_UINT8 + * - TILEDB_INT16 + * - TILEDB_UINT16 + * - TILEDB_UINT32 + * - TILEDB_UINT64 * * The coordinate type can be one of the following: * - TILEDB_INT32 * - TILEDB_INT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 + * - TILEDB_INT8 + * - TILEDB_UINT8 + * - TILEDB_INT16 + * - TILEDB_UINT16 + * - TILEDB_UINT32 + * - TILEDB_UINT64 */ int* types_; } ArraySchemaC; diff --git a/core/include/array/array_sorted_write_state.h b/core/include/array/array_sorted_write_state.h index 7a73bd0dd55..14b3c0c3a64 100644 --- a/core/include/array/array_sorted_write_state.h +++ b/core/include/array/array_sorted_write_state.h @@ -643,8 +643,8 @@ class ArraySortedWriteState { /** * Fills the **a single** cell in a variable-sized buffer of the current copy - * tile slab with the input id with an empty value, based on the template t - * ype. Applicable only to variable-sized attributes. + * tile slab with the input id with an empty value, based on the template + * type. Applicable only to variable-sized attributes. * * @tparam T The attribute type. * @param bid The buffer id corresponding to the targeted attribute. diff --git a/core/include/c_api/tiledb.h b/core/include/c_api/tiledb.h index 3e95ed870af..8526aa2b949 100755 --- a/core/include/c_api/tiledb.h +++ b/core/include/c_api/tiledb.h @@ -222,7 +222,6 @@ typedef struct TileDB_ArraySchema { * The cell order. It can be one of the following: * - TILEDB_ROW_MAJOR * - TILEDB_COL_MAJOR - * - TILEDB_HILBERT. */ int cell_order_; /** @@ -246,6 +245,7 @@ typedef struct TileDB_ArraySchema { * - TILEDB_BLOSC_ZLIB * - TILEDB_BLOSC_ZSTD * - TILEDB_RLE + * - TILEDB_BZIP2 * * If it is *NULL*, then the default TILEDB_NO_COMPRESSION is used for all * attributes. @@ -286,12 +286,24 @@ typedef struct TileDB_ArraySchema { * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 * - TILEDB_CHAR + * - TILEDB_INT8 + * - TILEDB_UINT8 + * - TILEDB_INT16 + * - TILEDB_UINT16 + * - TILEDB_UINT32 + * - TILEDB_UINT64 * * The coordinate type can be one of the following: * - TILEDB_INT32 * - TILEDB_INT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 + * - TILEDB_INT8 + * - TILEDB_UINT8 + * - TILEDB_INT16 + * - TILEDB_UINT16 + * - TILEDB_UINT32 + * - TILEDB_UINT64 */ int* types_; } TileDB_ArraySchema; @@ -759,6 +771,7 @@ typedef struct TileDB_MetadataSchema { * - TILEDB_BLOSC_ZLIB * - TILEDB_BLOSC_ZSTD * - TILEDB_RLE + * - TILEDB_BZIP2 * * If it is *NULL*, then the default TILEDB_NO_COMPRESSION is used for all * attributes. @@ -771,7 +784,13 @@ typedef struct TileDB_MetadataSchema { * - TILEDB_INT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 - * - TILEDB_CHAR. + * - TILEDB_CHAR + * - TILEDB_INT8 + * - TILEDB_UINT8 + * - TILEDB_INT16 + * - TILEDB_UINT16 + * - TILEDB_UINT32 + * - TILEDB_UINT64 */ int* types_; } TileDB_MetadataSchema; @@ -1084,6 +1103,22 @@ TILEDB_EXPORT int tiledb_metadata_iterator_finalize( /* DIRECTORY MANAGEMENT */ /* ********************************* */ +/** + * Returns the type of the input directory. + * + * @param tiledb_ctx The TileDB context. + * @param dir The input directory. + * @return It can be one of the following: + * - TILEDB_WORKSPACE + * - TILEDB_GROUP + * - TILEDB_ARRAY + * - TILEDB_METADATA + * - -1 (none of the above) + */ +TILEDB_EXPORT int tiledb_dir_type( + const TileDB_CTX* tiledb_ctx, + const char* dir); + /** * Clears a TileDB directory. The corresponding TileDB object (workspace, * group, array, or metadata) will still exist after the execution of the diff --git a/core/include/c_api/tiledb_constants.h b/core/include/c_api/tiledb_constants.h index 252353d6ad6..43a153687ea 100644 --- a/core/include/c_api/tiledb_constants.h +++ b/core/include/c_api/tiledb_constants.h @@ -37,7 +37,7 @@ #include /** Version. */ -#define TILEDB_VERSION "0.5.2" +#define TILEDB_VERSION "0.6.0" /**@{*/ /** Return code. */ @@ -102,10 +102,16 @@ /**@{*/ /** Special empty cell value. */ #define TILEDB_EMPTY_INT32 INT_MAX -#define TILEDB_EMPTY_INT64 LLONG_MAX +#define TILEDB_EMPTY_INT64 INT64_MAX #define TILEDB_EMPTY_FLOAT32 FLT_MAX #define TILEDB_EMPTY_FLOAT64 DBL_MAX #define TILEDB_EMPTY_CHAR CHAR_MAX +#define TILEDB_EMPTY_INT8 INT8_MAX +#define TILEDB_EMPTY_UINT8 UINT8_MAX +#define TILEDB_EMPTY_INT16 INT16_MAX +#define TILEDB_EMPTY_UINT16 UINT16_MAX +#define TILEDB_EMPTY_UINT32 UINT32_MAX +#define TILEDB_EMPTY_UINT64 UINT64_MAX /**@}*/ /**@{*/ @@ -121,13 +127,18 @@ #define TILEDB_FLOAT32 2 #define TILEDB_FLOAT64 3 #define TILEDB_CHAR 4 +#define TILEDB_INT8 5 +#define TILEDB_UINT8 6 +#define TILEDB_INT16 7 +#define TILEDB_UINT16 8 +#define TILEDB_UINT32 9 +#define TILEDB_UINT64 10 /**@}*/ /**@{*/ /** Tile or cell order. */ #define TILEDB_ROW_MAJOR 0 #define TILEDB_COL_MAJOR 1 -#define TILEDB_HILBERT 2 /**@}*/ /**@{*/ @@ -143,6 +154,7 @@ #define TILEDB_BLOSC_ZLIB 8 #define TILEDB_BLOSC_ZSTD 9 #define TILEDB_RLE 10 +#define TILEDB_BZIP2 11 /**@}*/ /**@{*/ diff --git a/core/include/fragment/read_state.h b/core/include/fragment/read_state.h index 7c9f0ac7c04..4f7c1c59a88 100644 --- a/core/include/fragment/read_state.h +++ b/core/include/fragment/read_state.h @@ -598,6 +598,7 @@ class ReadState { /** * Decompresses a tile with GZIP. * + * @param attribute_id The attribute id. * @param tile_compressed The compressed tile to be decompressed. * @param tile_compressed_size The size of the compressed tile. * @param tile The resulting decompressed tile. @@ -606,6 +607,7 @@ class ReadState { * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int decompress_tile_gzip( + int attribute_id, unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, @@ -614,6 +616,7 @@ class ReadState { /** * Decompresses a tile with Zstandard. * + * @param attribute_id The attribute id. * @param tile_compressed The compressed tile to be decompressed. * @param tile_compressed_size The size of the compressed tile. * @param tile The resulting decompressed tile. @@ -622,6 +625,7 @@ class ReadState { * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int decompress_tile_zstd( + int attribute_id, unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, @@ -630,6 +634,7 @@ class ReadState { /** * Decompresses a tile with LZ4. * + * @param attribute_id The attribute id. * @param tile_compressed The compressed tile to be decompressed. * @param tile_compressed_size The size of the compressed tile. * @param tile The resulting decompressed tile. @@ -638,6 +643,7 @@ class ReadState { * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int decompress_tile_lz4( + int attribute_id, unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, @@ -646,6 +652,7 @@ class ReadState { /** * Decompresses a tile with Blosc. * + * @param attribute_id The attribute id. * @param tile_compressed The compressed tile to be decompressed. * @param tile_compressed_size The size of the compressed tile. * @param tile The resulting decompressed tile. @@ -655,6 +662,7 @@ class ReadState { * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. */ int decompress_tile_blosc( + int attribute_id, unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, @@ -679,6 +687,24 @@ class ReadState { unsigned char* tile, size_t tile_size); + /** + * Decompresses a tile with BZIP2. + * + * @param attribute_id The attribute id. + * @param tile_compressed The compressed tile to be decompressed. + * @param tile_compressed_size The size of the compressed tile. + * @param tile The resulting decompressed tile. + * @param tile_size The expected size of the decompressed tile (for checking + * for errors). + * @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error. + */ + int decompress_tile_bzip2( + int attribute_id, + unsigned char* tile_compressed, + size_t tile_compressed_size, + unsigned char* tile, + size_t tile_size); + /** * Returns the cell position in the search tile that is after the * input coordinates. diff --git a/core/include/fragment/write_state.h b/core/include/fragment/write_state.h index a05f25b00a6..396bf8bb473 100644 --- a/core/include/fragment/write_state.h +++ b/core/include/fragment/write_state.h @@ -226,12 +226,14 @@ class WriteState { * Compresses with GZIP the input tile buffer, and stores it inside * tile_compressed_ member attribute. * + * @param attribute_id The attribute id. * @param tile The tile buffer to be compressed. * @param tile_size The size of the tile buffer in bytes. * @param tile_compressed_size The size of the resulting compressed tile. * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int compress_tile_gzip( + int attribute_id, unsigned char* tile, size_t tile_size, size_t& tile_compressed_size); @@ -240,12 +242,14 @@ class WriteState { * Compresses with Zstandard the input tile buffer, and stores it inside * tile_compressed_ member attribute. * + * @param attribute_id The attribute id. * @param tile The tile buffer to be compressed. * @param tile_size The size of the tile buffer in bytes. * @param tile_compressed_size The size of the resulting compressed tile. * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int compress_tile_zstd( + int attribute_id, unsigned char* tile, size_t tile_size, size_t& tile_compressed_size); @@ -254,12 +258,14 @@ class WriteState { * Compresses with LZ4 the input tile buffer, and stores it inside * tile_compressed_ member attribute. * + * @param attribute_id The attribute id. * @param tile The tile buffer to be compressed. * @param tile_size The size of the tile buffer in bytes. * @param tile_compressed_size The size of the resulting compressed tile. * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. */ int compress_tile_lz4( + int attribute_id, unsigned char* tile, size_t tile_size, size_t& tile_compressed_size); @@ -298,6 +304,22 @@ class WriteState { size_t tile_size, size_t& tile_compressed_size); + /** + * Compresses with BZIP2 the input tile buffer, and stores it inside + * tile_compressed_ member attribute. + * + * @param attribute_id The attribute id. + * @param tile The tile buffer to be compressed. + * @param tile_size The size of the tile buffer in bytes. + * @param tile_compressed_size The size of the resulting compressed tile. + * @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error. + */ + int compress_tile_bzip2( + int attribute_id, + unsigned char* tile, + size_t tile_size, + size_t& tile_compressed_size); + /** * Compresses the current tile for the input attribute, and writes (appends) * it to its corresponding file on the disk. diff --git a/core/include/metadata/metadata_schema_c.h b/core/include/metadata/metadata_schema_c.h index bef2f065801..6dcdf178376 100644 --- a/core/include/metadata/metadata_schema_c.h +++ b/core/include/metadata/metadata_schema_c.h @@ -71,6 +71,7 @@ typedef struct MetadataSchemaC { * - TILEDB_BLOSC_ZLIB * - TILEDB_BLOSC_ZSTD * - TILEDB_RLE + * - TILEDB_BZIP2 */ int* compression_; /** @@ -80,7 +81,13 @@ typedef struct MetadataSchemaC { * - TILEDB_INT64 * - TILEDB_FLOAT32 * - TILEDB_FLOAT64 - * - TILEDB_CHAR. + * - TILEDB_CHAR + * - TILEDB_INT8 + * - TILEDB_UINT8 + * - TILEDB_INT16 + * - TILEDB_UINT16 + * - TILEDB_UINT32 + * - TILEDB_UINT64 */ int* types_; } MetadataSchemaC; diff --git a/core/include/misc/hilbert_curve.h b/core/include/misc/hilbert_curve.h deleted file mode 100755 index ac723090f54..00000000000 --- a/core/include/misc/hilbert_curve.h +++ /dev/null @@ -1,242 +0,0 @@ -/** - * @file hilbert_curve.h - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2016 MIT and Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file defines the HilbertCurve class. - */ - -#ifndef __HILBERT_CURVE_H__ -#define __HILBERT_CURVE_H__ - -#include -#include -#include - - - - -/* ********************************* */ -/* CONSTANTS */ -/* ********************************* */ - -/** - * Maximum number of dimensions for defining the Hilbert curve. Although the - * Hilbert curve can be defined over arbitrary dimensionality, we limit the - * number of dimensions because they affect the number of bits used to - * represent a Hilbert value; in our class, a Hilbert value is a int64_t - * number and, thus, it cannot exceed 64 bits. - */ -#define HC_MAX_DIM 16 - - - - -/** - * The Hilbert curve fills a multi-dimensional space in a particular manner - * with a 1D line. The typical operations of this class is converting a - * multi-dimensional tuple of coordinates into a 1D Hilbert value, and - * vice versa. - * - * For the 2D case, the Hilbert curve looks as follows: - * - \verbatim - | - 15 | @---@ @---@ @---@ @---@ @---@ @---@ @---@ @---@ - | | | | | | | | | | | | | | | | | - | @ @---@ @ @ @---@ @ @ @---@ @ @ @---@ @ - | | | | | | | | | - | @---@ @---@ @---@ @---@ @---@ @---@ @---@ @---@ - | | | | | | | | | - | @---@ @---@---@---@ @---@ @---@ @---@---@---@ @---@ - | | | | | - | @ @---@---@ @---@---@ @ @ @---@---@ @---@---@ @ - | | | | | | | | | | | | | - Dim[1]| @---@ @---@ @---@ @---@ @---@ @---@ @---@ @---@ - | | | | | - | @---@ @---@ @---@ @---@ @---@ @---@ @---@ @---@ - | | | | | | | | | | | | | - | @ @---@---@ @---@---@ @---@ @---@---@ @---@---@ @ - | | | - | @---@ @---@---@ @---@---@ @---@---@ @---@---@ @---@ - | | | | | | | | | | | - | @---@ @---@ @---@ @---@ @---@ @---@ @---@ @---@ - | | | | | | | - | @ @---@ @ @---@ @---@ @---@ @---@ @ @---@ @ - | | | | | | | | | | | | | | | - | @---@ @---@ @ @---@---@ @---@---@ @ @---@ @---@ - | | | - 3 | 5---6 9---@ @ @---@---@ @---@---@ @ @---@ @---@ - | | | | | | | | | | | | | | | - 2 | 4 7---8 @ @---@ @---@ @---@ @---@ @ @---@ @ - | | | | | | | - 1 | 3---2 @---@ @---@ @---@ @---@ @---@ @---@ @---@ - | | | | | | | | | | | - 0 | 0---1 @---@---@ @---@---@ @---@---@ @---@---@ @--255 - | - ------------------------------------------------------------------- - 0 1 2 3 Dim[0] 15 - - \endverbatim - * - * The Hilbert value of (2,3) is 9, whereas the coordinates corresponding to - * Hilbert value 2 are (1,1). - * - * The class utilizes two functions from John Skilling's work published in: - * John Skilling, "Programming the Hilbert Curve". In AIP, 2004 - */ -class HilbertCurve { - public: - /* ********************************* */ - /* CONSTRUCTORS & DESTRUCTORS */ - /* ********************************* */ - - /** - * Constructor. - * - * @param bits Number of bits used for coordinate values across each dimension - * @param dim_num Number of dimensions - */ - HilbertCurve(int bits, int dim_num); - - /** Destructor. */ - ~HilbertCurve(); - - - - - /* ********************************* */ - /* MAIN FUNCTIONS */ - /* ********************************* */ - - /** - * Converts a set of coordinates to a Hilbert value. - * - * @param coords The coordinates to be converted. - * @param hilbert The output Hilbert value. - * @return void - */ - void coords_to_hilbert(const int* coords, int64_t& hilbert); - - /** - * Converts a Hilbert value into a set of coordinates. - * - * @param hilbert The Hilbert value to be converted. - * @param coords The output coordinates. - * @return void - */ - void hilbert_to_coords(int64_t hilbert, int* coords); - - - - - private: - /* ********************************* */ - /* PRIVATE ATTRIBUTES */ - /* ********************************* */ - - /** Number of bits for representing a coordinate per dimension. */ - int bits_; - /** Number of dimensions. */ - int dim_num_; - /** Temporary buffer. */ - int temp_[HC_MAX_DIM]; - - - - - /* ********************************* */ - /* PRIVATE METHODS */ - /* ********************************* */ - - /** - * Identical to John Skilling's work. It converts the input coordinates - * to what is called the transpose of the Hilbert value. This is done - * in place. - * - * **Example** - * - * Let b=5 and n=3. Let the 15-bit Hilbert value of the input coordinates - * be A B C D E a b c d e 1 2 3 4 5. The function places this number into - * parameter X as follows: - * - \verbatim - X[0] = A D b e 3 X[1]| - X[1] = B E c 1 4 <-------> | /X[2] - X[2] = C a d 2 5 axes | / - high low |/______ - - X[0] - \endverbatim - * - * The X value after the function terminates is called the transpose form - * of the Hilbert value. - * - * @param X Input coordinates, and output transpose (the conversion is - * done in place). - * @param b Number of bits for representing a coordinate per dimension - * (it should be equal to HilbertCurve::bits_). - * @param n Number of dimensions (it should be equal to - * HilbertCurve::dim_num_). - * @return void - */ - void AxestoTranspose(int* X, int b, int n); - - /** - * Identical to John Skilling's work. It converts the transpose of a - * Hilbert value into the corresponding coordinates. This is done in place. - * - * **Example** - * - * Let b=5 and n=3. Let the 15-bit Hilbert value of the output coordinates - * be A B C D E a b c d e 1 2 3 4 5. The function takes as input the tranpose - * form of the Hilbert value, which is stored in X as follows: - * - \verbatim - X[0] = A D b e 3 X[1]| - X[1] = B E c 1 4 <-------> | /X[2] - X[2] = C a d 2 5 axes | / - high low |/______ - - X[0] - \endverbatim - * - * The X value after the function terminates will contain the coordinates - * corresponding to the Hilbert value. - * - * @param X Input transpose, and output coordinates (the conversion is - * done in place). - * @param b Number of bits for representing a coordinate per dimension - * (it should be equal to HilbertCurve::bits_). - * @param n Number of dimensions (it should be equal to - * HilbertCurve::dim_num_). - * @return void - */ - void TransposetoAxes(int* X, int b, int n); -}; - -#endif diff --git a/core/include/misc/utils.h b/core/include/misc/utils.h index b095394ed7b..f98a6a02daf 100644 --- a/core/include/misc/utils.h +++ b/core/include/misc/utils.h @@ -748,6 +748,21 @@ int RLE_decompress_coords_row( size_t value_size, int dim_num); +/** + * Receives a tile of coordinate tuples and splits it across the dimensions. + * + * @param tile The input tile. + * @param tile_size The tile size. + * @param dim_num The number of dimensions. + * @param coords_size The size in bytes of a tuple of coordinates. + * @return void + */ +void split_coordinates( + void* tile, + size_t tile_size, + int dim_num, + size_t coords_size); + /** * Checks if a string starts with a certain prefix. * @@ -792,4 +807,20 @@ int write_to_file_cmp_gzip( const void* buffer, size_t buffer_size); +/** + * Creates tuples of coordinates from vertically partitioned dimensions. + * + * @param tile The input tile. + * @param tile_size The tile size. + * @param dim_num The number of dimensions. + * @param coords_size The size in bytes of a tuple of coordinates. + * @return void + */ +void zip_coordinates( + void* tile, + size_t tile_size, + int dim_num, + size_t coords_size); + + #endif diff --git a/core/include/storage_manager/storage_manager.h b/core/include/storage_manager/storage_manager.h index f3afcdad078..07d6059623d 100755 --- a/core/include/storage_manager/storage_manager.h +++ b/core/include/storage_manager/storage_manager.h @@ -475,6 +475,19 @@ class StorageManager { /* MISC */ /* ********************************* */ + /** + * Returns the type of the input directory. + * + * @param dir The input directory. + * @return It can be one of the following: + * - TILEDB_WORKSPACE + * - TILEDB_GROUP + * - TILEDB_ARRAY + * - TILEDB_METADATA + * - -1 (not a TileDB directory) + */ + int dir_type(const char* dir); + /** * Lists all the TileDB objects in a directory, copying them into the input * buffers. diff --git a/core/src/array/array_read_state.cc b/core/src/array/array_read_state.cc index a016dbc9b95..9ef84c52ffa 100644 --- a/core/src/array/array_read_state.cc +++ b/core/src/array/array_read_state.cc @@ -48,6 +48,8 @@ # define PRINT_ERROR(x) do { } while(0) #endif +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define MAX(a,b) ((a) > (b) ? (a) : (b)) @@ -442,6 +444,42 @@ int ArrayReadState::copy_cells( buffer, buffer_size, buffer_offset); + else if(type == TILEDB_INT8) + rc = copy_cells( + attribute_id, + buffer, + buffer_size, + buffer_offset); + else if(type == TILEDB_UINT8) + rc = copy_cells( + attribute_id, + buffer, + buffer_size, + buffer_offset); + else if(type == TILEDB_INT16) + rc = copy_cells( + attribute_id, + buffer, + buffer_size, + buffer_offset); + else if(type == TILEDB_UINT16) + rc = copy_cells( + attribute_id, + buffer, + buffer_size, + buffer_offset); + else if(type == TILEDB_UINT32) + rc = copy_cells( + attribute_id, + buffer, + buffer_size, + buffer_offset); + else if(type == TILEDB_UINT64) + rc = copy_cells( + attribute_id, + buffer, + buffer_size, + buffer_offset); else rc = TILEDB_ARS_ERR; @@ -579,6 +617,60 @@ int ArrayReadState::copy_cells_var( buffer_var, buffer_var_size, buffer_var_offset); + else if(type == TILEDB_INT8) + rc = copy_cells_var( + attribute_id, + buffer, + buffer_size, + buffer_offset, + buffer_var, + buffer_var_size, + buffer_var_offset); + else if(type == TILEDB_UINT8) + rc = copy_cells_var( + attribute_id, + buffer, + buffer_size, + buffer_offset, + buffer_var, + buffer_var_size, + buffer_var_offset); + else if(type == TILEDB_INT16) + rc = copy_cells_var( + attribute_id, + buffer, + buffer_size, + buffer_offset, + buffer_var, + buffer_var_size, + buffer_var_offset); + else if(type == TILEDB_UINT16) + rc = copy_cells_var( + attribute_id, + buffer, + buffer_size, + buffer_offset, + buffer_var, + buffer_var_size, + buffer_var_offset); + else if(type == TILEDB_UINT32) + rc = copy_cells_var( + attribute_id, + buffer, + buffer_size, + buffer_offset, + buffer_var, + buffer_var_size, + buffer_var_offset); + else if(type == TILEDB_UINT64) + rc = copy_cells_var( + attribute_id, + buffer, + buffer_size, + buffer_offset, + buffer_var, + buffer_var_size, + buffer_var_offset); else rc = TILEDB_ARS_ERR; @@ -696,7 +788,7 @@ void ArrayReadState::copy_cells_with_empty( int64_t cell_num_left_to_copy = cell_num_in_range - empty_cells_written_[attribute_id]; size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; - size_t bytes_to_copy = std::min(bytes_left_to_copy, buffer_free_space); + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); int64_t cell_num_to_copy = bytes_to_copy / cell_size; // Copy empty cells to buffer @@ -745,7 +837,7 @@ void ArrayReadState::copy_cells_with_empty( int64_t cell_num_left_to_copy = cell_num_in_range - empty_cells_written_[attribute_id]; size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; - size_t bytes_to_copy = std::min(bytes_left_to_copy, buffer_free_space); + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); int64_t cell_num_to_copy = bytes_to_copy / cell_size; // Copy empty cells to buffer @@ -787,134 +879,832 @@ void ArrayReadState::copy_cells_with_empty( } // Sanity check - assert(!array_schema_->var_size(attribute_id)); + assert(!array_schema_->var_size(attribute_id)); + + // Calculate number of empty cells to write + int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; + int64_t cell_num_left_to_copy = + cell_num_in_range - empty_cells_written_[attribute_id]; + size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + int64_t cell_num_to_copy = bytes_to_copy / cell_size; + + // Copy empty cells to buffer + float empty = TILEDB_EMPTY_FLOAT32; + for(int64_t i=0; i +void ArrayReadState::copy_cells_with_empty( + int attribute_id, + void* buffer, + size_t buffer_size, + size_t& buffer_offset, + const CellPosRange& cell_pos_range) { + // For easy reference + size_t cell_size = array_schema_->cell_size(attribute_id); + char* buffer_c = static_cast(buffer); + int cell_val_num = array_schema_->cell_val_num(attribute_id); + + // Calculate free space in buffer + size_t buffer_free_space = buffer_size - buffer_offset; + buffer_free_space = (buffer_free_space / cell_size) * cell_size; + if(buffer_free_space == 0) { // Overflow + overflow_[attribute_id] = true; + return; + } + + // Sanity check + assert(!array_schema_->var_size(attribute_id)); + + // Calculate number of empty cells to write + int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; + int64_t cell_num_left_to_copy = + cell_num_in_range - empty_cells_written_[attribute_id]; + size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + int64_t cell_num_to_copy = bytes_to_copy / cell_size; + + // Copy empty cells to buffer + double empty = TILEDB_EMPTY_FLOAT64; + for(int64_t i=0; i +void ArrayReadState::copy_cells_with_empty( + int attribute_id, + void* buffer, + size_t buffer_size, + size_t& buffer_offset, + const CellPosRange& cell_pos_range) { + // For easy reference + size_t cell_size = array_schema_->cell_size(attribute_id); + char* buffer_c = static_cast(buffer); + int cell_val_num = array_schema_->cell_val_num(attribute_id); + + // Calculate free space in buffer + size_t buffer_free_space = buffer_size - buffer_offset; + buffer_free_space = (buffer_free_space / cell_size) * cell_size; + if(buffer_free_space == 0) { // Overflow + overflow_[attribute_id] = true; + return; + } + + // Sanity check + assert(!array_schema_->var_size(attribute_id)); + + // Calculate number of empty cells to write + int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; + int64_t cell_num_left_to_copy = + cell_num_in_range - empty_cells_written_[attribute_id]; + size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + int64_t cell_num_to_copy = bytes_to_copy / cell_size; + + // Copy empty cells to buffer + char empty = TILEDB_EMPTY_CHAR; + for(int64_t i=0; i +void ArrayReadState::copy_cells_with_empty( + int attribute_id, + void* buffer, + size_t buffer_size, + size_t& buffer_offset, + const CellPosRange& cell_pos_range) { + // For easy reference + size_t cell_size = array_schema_->cell_size(attribute_id); + char* buffer_c = static_cast(buffer); + int cell_val_num = array_schema_->cell_val_num(attribute_id); + + // Calculate free space in buffer + size_t buffer_free_space = buffer_size - buffer_offset; + buffer_free_space = (buffer_free_space / cell_size) * cell_size; + if(buffer_free_space == 0) { // Overflow + overflow_[attribute_id] = true; + return; + } + + // Sanity check + assert(!array_schema_->var_size(attribute_id)); + + // Calculate number of empty cells to write + int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; + int64_t cell_num_left_to_copy = + cell_num_in_range - empty_cells_written_[attribute_id]; + size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + int64_t cell_num_to_copy = bytes_to_copy / cell_size; + + // Copy empty cells to buffer + int8_t empty = TILEDB_EMPTY_INT8; + for(int64_t i=0; i +void ArrayReadState::copy_cells_with_empty( + int attribute_id, + void* buffer, + size_t buffer_size, + size_t& buffer_offset, + const CellPosRange& cell_pos_range) { + // For easy reference + size_t cell_size = array_schema_->cell_size(attribute_id); + char* buffer_c = static_cast(buffer); + int cell_val_num = array_schema_->cell_val_num(attribute_id); + + // Calculate free space in buffer + size_t buffer_free_space = buffer_size - buffer_offset; + buffer_free_space = (buffer_free_space / cell_size) * cell_size; + if(buffer_free_space == 0) { // Overflow + overflow_[attribute_id] = true; + return; + } + + // Sanity check + assert(!array_schema_->var_size(attribute_id)); + + // Calculate number of empty cells to write + int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; + int64_t cell_num_left_to_copy = + cell_num_in_range - empty_cells_written_[attribute_id]; + size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + int64_t cell_num_to_copy = bytes_to_copy / cell_size; + + // Copy empty cells to buffer + uint8_t empty = TILEDB_EMPTY_UINT8; + for(int64_t i=0; i +void ArrayReadState::copy_cells_with_empty( + int attribute_id, + void* buffer, + size_t buffer_size, + size_t& buffer_offset, + const CellPosRange& cell_pos_range) { + // For easy reference + size_t cell_size = array_schema_->cell_size(attribute_id); + char* buffer_c = static_cast(buffer); + int cell_val_num = array_schema_->cell_val_num(attribute_id); + + // Calculate free space in buffer + size_t buffer_free_space = buffer_size - buffer_offset; + buffer_free_space = (buffer_free_space / cell_size) * cell_size; + if(buffer_free_space == 0) { // Overflow + overflow_[attribute_id] = true; + return; + } + + // Sanity check + assert(!array_schema_->var_size(attribute_id)); + + // Calculate number of empty cells to write + int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; + int64_t cell_num_left_to_copy = + cell_num_in_range - empty_cells_written_[attribute_id]; + size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + int64_t cell_num_to_copy = bytes_to_copy / cell_size; + + // Copy empty cells to buffer + int16_t empty = TILEDB_EMPTY_INT16; + for(int64_t i=0; i +void ArrayReadState::copy_cells_with_empty( + int attribute_id, + void* buffer, + size_t buffer_size, + size_t& buffer_offset, + const CellPosRange& cell_pos_range) { + // For easy reference + size_t cell_size = array_schema_->cell_size(attribute_id); + char* buffer_c = static_cast(buffer); + int cell_val_num = array_schema_->cell_val_num(attribute_id); + + // Calculate free space in buffer + size_t buffer_free_space = buffer_size - buffer_offset; + buffer_free_space = (buffer_free_space / cell_size) * cell_size; + if(buffer_free_space == 0) { // Overflow + overflow_[attribute_id] = true; + return; + } + + // Sanity check + assert(!array_schema_->var_size(attribute_id)); + + // Calculate number of empty cells to write + int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; + int64_t cell_num_left_to_copy = + cell_num_in_range - empty_cells_written_[attribute_id]; + size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + int64_t cell_num_to_copy = bytes_to_copy / cell_size; + + // Copy empty cells to buffer + uint16_t empty = TILEDB_EMPTY_UINT16; + for(int64_t i=0; i +void ArrayReadState::copy_cells_with_empty( + int attribute_id, + void* buffer, + size_t buffer_size, + size_t& buffer_offset, + const CellPosRange& cell_pos_range) { + // For easy reference + size_t cell_size = array_schema_->cell_size(attribute_id); + char* buffer_c = static_cast(buffer); + int cell_val_num = array_schema_->cell_val_num(attribute_id); + + // Calculate free space in buffer + size_t buffer_free_space = buffer_size - buffer_offset; + buffer_free_space = (buffer_free_space / cell_size) * cell_size; + if(buffer_free_space == 0) { // Overflow + overflow_[attribute_id] = true; + return; + } + + // Sanity check + assert(!array_schema_->var_size(attribute_id)); + + // Calculate number of empty cells to write + int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; + int64_t cell_num_left_to_copy = + cell_num_in_range - empty_cells_written_[attribute_id]; + size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + int64_t cell_num_to_copy = bytes_to_copy / cell_size; + + // Copy empty cells to buffer + uint32_t empty = TILEDB_EMPTY_UINT32; + for(int64_t i=0; i +void ArrayReadState::copy_cells_with_empty( + int attribute_id, + void* buffer, + size_t buffer_size, + size_t& buffer_offset, + const CellPosRange& cell_pos_range) { + // For easy reference + size_t cell_size = array_schema_->cell_size(attribute_id); + char* buffer_c = static_cast(buffer); + int cell_val_num = array_schema_->cell_val_num(attribute_id); + + // Calculate free space in buffer + size_t buffer_free_space = buffer_size - buffer_offset; + buffer_free_space = (buffer_free_space / cell_size) * cell_size; + if(buffer_free_space == 0) { // Overflow + overflow_[attribute_id] = true; + return; + } + + // Sanity check + assert(!array_schema_->var_size(attribute_id)); + + // Calculate number of empty cells to write + int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; + int64_t cell_num_left_to_copy = + cell_num_in_range - empty_cells_written_[attribute_id]; + size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + int64_t cell_num_to_copy = bytes_to_copy / cell_size; + + // Copy empty cells to buffer + uint64_t empty = TILEDB_EMPTY_UINT64; + for(int64_t i=0; i +void ArrayReadState::copy_cells_with_empty_var( + int attribute_id, + void* buffer, + size_t buffer_size, + size_t& buffer_offset, + void* buffer_var, + size_t buffer_var_size, + size_t& buffer_var_offset, + const CellPosRange& cell_pos_range) { + // For easy reference + size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; + size_t cell_size_var = sizeof(int); + char* buffer_c = static_cast(buffer); + char* buffer_var_c = static_cast(buffer_var); + + // Calculate free space in buffer + size_t buffer_free_space = buffer_size - buffer_offset; + buffer_free_space = (buffer_free_space / cell_size) * cell_size; + size_t buffer_var_free_space = buffer_var_size - buffer_var_offset; + buffer_var_free_space = (buffer_var_free_space/cell_size_var)*cell_size_var; + + // Handle overflow + if(buffer_free_space == 0 || buffer_var_free_space == 0) { // Overflow + overflow_[attribute_id] = true; + return; + } + + // Sanity check + assert(array_schema_->var_size(attribute_id)); + + // Calculate cell number to copy + int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; + int64_t cell_num_left_to_copy = + cell_num_in_range - empty_cells_written_[attribute_id]; + size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; + size_t bytes_left_to_copy_var = cell_num_left_to_copy * cell_size_var; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + size_t bytes_to_copy_var = + MIN(bytes_left_to_copy_var, buffer_var_free_space); + int64_t cell_num_to_copy = bytes_to_copy / cell_size; + int64_t cell_num_to_copy_var = bytes_to_copy_var / cell_size_var; + cell_num_to_copy = MIN(cell_num_to_copy, cell_num_to_copy_var); + + // Copy empty cells to buffers + int empty = TILEDB_EMPTY_INT32; + for(int64_t i=0; i +void ArrayReadState::copy_cells_with_empty_var( + int attribute_id, + void* buffer, + size_t buffer_size, + size_t& buffer_offset, + void* buffer_var, + size_t buffer_var_size, + size_t& buffer_var_offset, + const CellPosRange& cell_pos_range) { + // For easy reference + size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; + size_t cell_size_var = sizeof(int64_t); + char* buffer_c = static_cast(buffer); + char* buffer_var_c = static_cast(buffer_var); + + // Calculate free space in buffer + size_t buffer_free_space = buffer_size - buffer_offset; + buffer_free_space = (buffer_free_space / cell_size) * cell_size; + size_t buffer_var_free_space = buffer_var_size - buffer_var_offset; + buffer_var_free_space = (buffer_var_free_space/cell_size_var)*cell_size_var; + + // Handle overflow + if(buffer_free_space == 0 || buffer_var_free_space == 0) { // Overflow + overflow_[attribute_id] = true; + return; + } + + // Sanity check + assert(array_schema_->var_size(attribute_id)); + + // Calculate cell number to copy + int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; + int64_t cell_num_left_to_copy = + cell_num_in_range - empty_cells_written_[attribute_id]; + size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; + size_t bytes_left_to_copy_var = cell_num_left_to_copy * cell_size_var; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + size_t bytes_to_copy_var = + MIN(bytes_left_to_copy_var, buffer_var_free_space); + int64_t cell_num_to_copy = bytes_to_copy / cell_size; + int64_t cell_num_to_copy_var = bytes_to_copy_var / cell_size_var; + cell_num_to_copy = MIN(cell_num_to_copy, cell_num_to_copy_var); + + // Copy empty cells to buffers + int64_t empty = TILEDB_EMPTY_INT64; + for(int64_t i=0; i +void ArrayReadState::copy_cells_with_empty_var( + int attribute_id, + void* buffer, + size_t buffer_size, + size_t& buffer_offset, + void* buffer_var, + size_t buffer_var_size, + size_t& buffer_var_offset, + const CellPosRange& cell_pos_range) { + // For easy reference + size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; + size_t cell_size_var = sizeof(float); + char* buffer_c = static_cast(buffer); + char* buffer_var_c = static_cast(buffer_var); + + // Calculate free space in buffer + size_t buffer_free_space = buffer_size - buffer_offset; + buffer_free_space = (buffer_free_space / cell_size) * cell_size; + size_t buffer_var_free_space = buffer_var_size - buffer_var_offset; + buffer_var_free_space = (buffer_var_free_space/cell_size_var)*cell_size_var; + + // Handle overflow + if(buffer_free_space == 0 || buffer_var_free_space == 0) { // Overflow + overflow_[attribute_id] = true; + return; + } + + // Sanity check + assert(array_schema_->var_size(attribute_id)); + + // Calculate cell number to copy + int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; + int64_t cell_num_left_to_copy = + cell_num_in_range - empty_cells_written_[attribute_id]; + size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; + size_t bytes_left_to_copy_var = cell_num_left_to_copy * cell_size_var; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + size_t bytes_to_copy_var = + MIN(bytes_left_to_copy_var, buffer_var_free_space); + int64_t cell_num_to_copy = bytes_to_copy / cell_size; + int64_t cell_num_to_copy_var = bytes_to_copy_var / cell_size_var; + cell_num_to_copy = MIN(cell_num_to_copy, cell_num_to_copy_var); + + // Copy empty cells to buffers + float empty = TILEDB_EMPTY_FLOAT32; + for(int64_t i=0; i +void ArrayReadState::copy_cells_with_empty_var( + int attribute_id, + void* buffer, + size_t buffer_size, + size_t& buffer_offset, + void* buffer_var, + size_t buffer_var_size, + size_t& buffer_var_offset, + const CellPosRange& cell_pos_range) { + // For easy reference + size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; + size_t cell_size_var = sizeof(double); + char* buffer_c = static_cast(buffer); + char* buffer_var_c = static_cast(buffer_var); + + // Calculate free space in buffer + size_t buffer_free_space = buffer_size - buffer_offset; + buffer_free_space = (buffer_free_space / cell_size) * cell_size; + size_t buffer_var_free_space = buffer_var_size - buffer_var_offset; + buffer_var_free_space = (buffer_var_free_space/cell_size_var)*cell_size_var; + + // Handle overflow + if(buffer_free_space == 0 || buffer_var_free_space == 0) { // Overflow + overflow_[attribute_id] = true; + return; + } + + // Sanity check + assert(array_schema_->var_size(attribute_id)); - // Calculate number of empty cells to write + // Calculate cell number to copy int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; int64_t cell_num_left_to_copy = cell_num_in_range - empty_cells_written_[attribute_id]; size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; - size_t bytes_to_copy = std::min(bytes_left_to_copy, buffer_free_space); + size_t bytes_left_to_copy_var = cell_num_left_to_copy * cell_size_var; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + size_t bytes_to_copy_var = + MIN(bytes_left_to_copy_var, buffer_var_free_space); int64_t cell_num_to_copy = bytes_to_copy / cell_size; + int64_t cell_num_to_copy_var = bytes_to_copy_var / cell_size_var; + cell_num_to_copy = MIN(cell_num_to_copy, cell_num_to_copy_var); - // Copy empty cells to buffer - float empty = TILEDB_EMPTY_FLOAT32; + // Copy empty cells to buffers + double empty = TILEDB_EMPTY_FLOAT64; for(int64_t i=0; i -void ArrayReadState::copy_cells_with_empty( +void ArrayReadState::copy_cells_with_empty_var( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, + void* buffer_var, + size_t buffer_var_size, + size_t& buffer_var_offset, const CellPosRange& cell_pos_range) { // For easy reference - size_t cell_size = array_schema_->cell_size(attribute_id); + size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; + size_t cell_size_var = sizeof(char); char* buffer_c = static_cast(buffer); - int cell_val_num = array_schema_->cell_val_num(attribute_id); + char* buffer_var_c = static_cast(buffer_var); // Calculate free space in buffer size_t buffer_free_space = buffer_size - buffer_offset; buffer_free_space = (buffer_free_space / cell_size) * cell_size; - if(buffer_free_space == 0) { // Overflow - overflow_[attribute_id] = true; + size_t buffer_var_free_space = buffer_var_size - buffer_var_offset; + buffer_var_free_space = (buffer_var_free_space/cell_size_var)*cell_size_var; + + // Handle overflow + if(buffer_free_space == 0 || buffer_var_free_space == 0) { // Overflow + overflow_[attribute_id] = true; return; } // Sanity check - assert(!array_schema_->var_size(attribute_id)); + assert(array_schema_->var_size(attribute_id)); - // Calculate number of empty cells to write + // Calculate cell number to copy int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; int64_t cell_num_left_to_copy = cell_num_in_range - empty_cells_written_[attribute_id]; size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; - size_t bytes_to_copy = std::min(bytes_left_to_copy, buffer_free_space); + size_t bytes_left_to_copy_var = cell_num_left_to_copy * cell_size_var; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + size_t bytes_to_copy_var = + MIN(bytes_left_to_copy_var, buffer_var_free_space); int64_t cell_num_to_copy = bytes_to_copy / cell_size; + int64_t cell_num_to_copy_var = bytes_to_copy_var / cell_size_var; + cell_num_to_copy = MIN(cell_num_to_copy, cell_num_to_copy_var); - // Copy empty cells to buffer - double empty = TILEDB_EMPTY_FLOAT64; + // Copy empty cells to buffers + char empty = TILEDB_EMPTY_CHAR; for(int64_t i=0; i -void ArrayReadState::copy_cells_with_empty( +void ArrayReadState::copy_cells_with_empty_var( int attribute_id, void* buffer, size_t buffer_size, size_t& buffer_offset, + void* buffer_var, + size_t buffer_var_size, + size_t& buffer_var_offset, const CellPosRange& cell_pos_range) { // For easy reference - size_t cell_size = array_schema_->cell_size(attribute_id); + size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; + size_t cell_size_var = sizeof(int8_t); char* buffer_c = static_cast(buffer); - int cell_val_num = array_schema_->cell_val_num(attribute_id); + char* buffer_var_c = static_cast(buffer_var); // Calculate free space in buffer size_t buffer_free_space = buffer_size - buffer_offset; buffer_free_space = (buffer_free_space / cell_size) * cell_size; - if(buffer_free_space == 0) { // Overflow - overflow_[attribute_id] = true; + size_t buffer_var_free_space = buffer_var_size - buffer_var_offset; + buffer_var_free_space = (buffer_var_free_space/cell_size_var)*cell_size_var; + + // Handle overflow + if(buffer_free_space == 0 || buffer_var_free_space == 0) { // Overflow + overflow_[attribute_id] = true; return; } // Sanity check - assert(!array_schema_->var_size(attribute_id)); + assert(array_schema_->var_size(attribute_id)); - // Calculate number of empty cells to write + // Calculate cell number to copy int64_t cell_num_in_range = cell_pos_range.second - cell_pos_range.first + 1; int64_t cell_num_left_to_copy = cell_num_in_range - empty_cells_written_[attribute_id]; size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; - size_t bytes_to_copy = std::min(bytes_left_to_copy, buffer_free_space); + size_t bytes_left_to_copy_var = cell_num_left_to_copy * cell_size_var; + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); + size_t bytes_to_copy_var = + MIN(bytes_left_to_copy_var, buffer_var_free_space); int64_t cell_num_to_copy = bytes_to_copy / cell_size; + int64_t cell_num_to_copy_var = bytes_to_copy_var / cell_size_var; + cell_num_to_copy = MIN(cell_num_to_copy, cell_num_to_copy_var); - // Copy empty cells to buffer - char empty = TILEDB_EMPTY_CHAR; + // Copy empty cells to buffers + int8_t empty = TILEDB_EMPTY_INT8; for(int64_t i=0; i -void ArrayReadState::copy_cells_with_empty_var( +void ArrayReadState::copy_cells_with_empty_var( int attribute_id, void* buffer, size_t buffer_size, @@ -925,7 +1715,7 @@ void ArrayReadState::copy_cells_with_empty_var( const CellPosRange& cell_pos_range) { // For easy reference size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; - size_t cell_size_var = sizeof(int); + size_t cell_size_var = sizeof(uint8_t); char* buffer_c = static_cast(buffer); char* buffer_var_c = static_cast(buffer_var); @@ -950,15 +1740,15 @@ void ArrayReadState::copy_cells_with_empty_var( cell_num_in_range - empty_cells_written_[attribute_id]; size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; size_t bytes_left_to_copy_var = cell_num_left_to_copy * cell_size_var; - size_t bytes_to_copy = std::min(bytes_left_to_copy, buffer_free_space); + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); size_t bytes_to_copy_var = - std::min(bytes_left_to_copy_var, buffer_var_free_space); + MIN(bytes_left_to_copy_var, buffer_var_free_space); int64_t cell_num_to_copy = bytes_to_copy / cell_size; int64_t cell_num_to_copy_var = bytes_to_copy_var / cell_size_var; - cell_num_to_copy = std::min(cell_num_to_copy, cell_num_to_copy_var); + cell_num_to_copy = MIN(cell_num_to_copy, cell_num_to_copy_var); // Copy empty cells to buffers - int empty = TILEDB_EMPTY_INT32; + uint8_t empty = TILEDB_EMPTY_UINT8; for(int64_t i=0; i( empty_cells_written_[attribute_id] += cell_num_to_copy; // Handle buffer overflow - if(empty_cells_written_[attribute_id] != cell_num_in_range) { + if(empty_cells_written_[attribute_id] != cell_num_in_range) overflow_[attribute_id] = true; - } else { // Done copying this range + else // Done copying this range empty_cells_written_[attribute_id] = 0; - } } template<> -void ArrayReadState::copy_cells_with_empty_var( +void ArrayReadState::copy_cells_with_empty_var( int attribute_id, void* buffer, size_t buffer_size, @@ -993,7 +1782,7 @@ void ArrayReadState::copy_cells_with_empty_var( const CellPosRange& cell_pos_range) { // For easy reference size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; - size_t cell_size_var = sizeof(int64_t); + size_t cell_size_var = sizeof(int16_t); char* buffer_c = static_cast(buffer); char* buffer_var_c = static_cast(buffer_var); @@ -1018,15 +1807,15 @@ void ArrayReadState::copy_cells_with_empty_var( cell_num_in_range - empty_cells_written_[attribute_id]; size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; size_t bytes_left_to_copy_var = cell_num_left_to_copy * cell_size_var; - size_t bytes_to_copy = std::min(bytes_left_to_copy, buffer_free_space); + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); size_t bytes_to_copy_var = - std::min(bytes_left_to_copy_var, buffer_var_free_space); + MIN(bytes_left_to_copy_var, buffer_var_free_space); int64_t cell_num_to_copy = bytes_to_copy / cell_size; int64_t cell_num_to_copy_var = bytes_to_copy_var / cell_size_var; - cell_num_to_copy = std::min(cell_num_to_copy, cell_num_to_copy_var); + cell_num_to_copy = MIN(cell_num_to_copy, cell_num_to_copy_var); // Copy empty cells to buffers - int64_t empty = TILEDB_EMPTY_INT64; + int16_t empty = TILEDB_EMPTY_INT16; for(int64_t i=0; i( empty_cells_written_[attribute_id] += cell_num_to_copy; // Handle buffer overflow - if(empty_cells_written_[attribute_id] != cell_num_in_range) { + if(empty_cells_written_[attribute_id] != cell_num_in_range) overflow_[attribute_id] = true; - } else { // Done copying this range + else // Done copying this range empty_cells_written_[attribute_id] = 0; - } } template<> -void ArrayReadState::copy_cells_with_empty_var( +void ArrayReadState::copy_cells_with_empty_var( int attribute_id, void* buffer, size_t buffer_size, @@ -1061,7 +1849,7 @@ void ArrayReadState::copy_cells_with_empty_var( const CellPosRange& cell_pos_range) { // For easy reference size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; - size_t cell_size_var = sizeof(float); + size_t cell_size_var = sizeof(uint16_t); char* buffer_c = static_cast(buffer); char* buffer_var_c = static_cast(buffer_var); @@ -1086,15 +1874,15 @@ void ArrayReadState::copy_cells_with_empty_var( cell_num_in_range - empty_cells_written_[attribute_id]; size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; size_t bytes_left_to_copy_var = cell_num_left_to_copy * cell_size_var; - size_t bytes_to_copy = std::min(bytes_left_to_copy, buffer_free_space); + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); size_t bytes_to_copy_var = - std::min(bytes_left_to_copy_var, buffer_var_free_space); + MIN(bytes_left_to_copy_var, buffer_var_free_space); int64_t cell_num_to_copy = bytes_to_copy / cell_size; int64_t cell_num_to_copy_var = bytes_to_copy_var / cell_size_var; - cell_num_to_copy = std::min(cell_num_to_copy, cell_num_to_copy_var); + cell_num_to_copy = MIN(cell_num_to_copy, cell_num_to_copy_var); // Copy empty cells to buffers - float empty = TILEDB_EMPTY_FLOAT32; + uint16_t empty = TILEDB_EMPTY_UINT16; for(int64_t i=0; i( } template<> -void ArrayReadState::copy_cells_with_empty_var( +void ArrayReadState::copy_cells_with_empty_var( int attribute_id, void* buffer, size_t buffer_size, @@ -1128,7 +1916,7 @@ void ArrayReadState::copy_cells_with_empty_var( const CellPosRange& cell_pos_range) { // For easy reference size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; - size_t cell_size_var = sizeof(double); + size_t cell_size_var = sizeof(uint32_t); char* buffer_c = static_cast(buffer); char* buffer_var_c = static_cast(buffer_var); @@ -1153,15 +1941,15 @@ void ArrayReadState::copy_cells_with_empty_var( cell_num_in_range - empty_cells_written_[attribute_id]; size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; size_t bytes_left_to_copy_var = cell_num_left_to_copy * cell_size_var; - size_t bytes_to_copy = std::min(bytes_left_to_copy, buffer_free_space); + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); size_t bytes_to_copy_var = - std::min(bytes_left_to_copy_var, buffer_var_free_space); + MIN(bytes_left_to_copy_var, buffer_var_free_space); int64_t cell_num_to_copy = bytes_to_copy / cell_size; int64_t cell_num_to_copy_var = bytes_to_copy_var / cell_size_var; - cell_num_to_copy = std::min(cell_num_to_copy, cell_num_to_copy_var); + cell_num_to_copy = MIN(cell_num_to_copy, cell_num_to_copy_var); // Copy empty cells to buffers - double empty = TILEDB_EMPTY_FLOAT64; + uint32_t empty = TILEDB_EMPTY_UINT32; for(int64_t i=0; i( } template<> -void ArrayReadState::copy_cells_with_empty_var( +void ArrayReadState::copy_cells_with_empty_var( int attribute_id, void* buffer, size_t buffer_size, @@ -1195,7 +1983,7 @@ void ArrayReadState::copy_cells_with_empty_var( const CellPosRange& cell_pos_range) { // For easy reference size_t cell_size = TILEDB_CELL_VAR_OFFSET_SIZE; - size_t cell_size_var = sizeof(char); + size_t cell_size_var = sizeof(uint64_t); char* buffer_c = static_cast(buffer); char* buffer_var_c = static_cast(buffer_var); @@ -1220,15 +2008,15 @@ void ArrayReadState::copy_cells_with_empty_var( cell_num_in_range - empty_cells_written_[attribute_id]; size_t bytes_left_to_copy = cell_num_left_to_copy * cell_size; size_t bytes_left_to_copy_var = cell_num_left_to_copy * cell_size_var; - size_t bytes_to_copy = std::min(bytes_left_to_copy, buffer_free_space); + size_t bytes_to_copy = MIN(bytes_left_to_copy, buffer_free_space); size_t bytes_to_copy_var = - std::min(bytes_left_to_copy_var, buffer_var_free_space); + MIN(bytes_left_to_copy_var, buffer_var_free_space); int64_t cell_num_to_copy = bytes_to_copy / cell_size; int64_t cell_num_to_copy_var = bytes_to_copy_var / cell_size_var; - cell_num_to_copy = std::min(cell_num_to_copy, cell_num_to_copy_var); + cell_num_to_copy = MIN(cell_num_to_copy, cell_num_to_copy_var); // Copy empty cells to buffers - char empty = TILEDB_EMPTY_CHAR; + uint64_t empty = TILEDB_EMPTY_UINT64; for(int64_t i=0; i( empty_cells_written_[attribute_id] += cell_num_to_copy; // Handle buffer overflow - if(empty_cells_written_[attribute_id] != cell_num_in_range) + if(empty_cells_written_[attribute_id] != cell_num_in_range) overflow_[attribute_id] = true; else // Done copying this range empty_cells_written_[attribute_id] = 0; @@ -1698,6 +2486,36 @@ int ArrayReadState::read_dense_attr( attribute_id, buffer, buffer_size); + } else if(coords_type == TILEDB_INT8) { + return read_dense_attr( + attribute_id, + buffer, + buffer_size); + } else if(coords_type == TILEDB_UINT8) { + return read_dense_attr( + attribute_id, + buffer, + buffer_size); + } else if(coords_type == TILEDB_INT16) { + return read_dense_attr( + attribute_id, + buffer, + buffer_size); + } else if(coords_type == TILEDB_UINT16) { + return read_dense_attr( + attribute_id, + buffer, + buffer_size); + } else if(coords_type == TILEDB_UINT32) { + return read_dense_attr( + attribute_id, + buffer, + buffer_size); + } else if(coords_type == TILEDB_UINT64) { + return read_dense_attr( + attribute_id, + buffer, + buffer_size); } else { std::string errmsg = "Cannot read from array; Invalid coordinates type"; PRINT_ERROR(errmsg); @@ -1787,6 +2605,48 @@ int ArrayReadState::read_dense_attr_var( buffer_size, buffer_var, buffer_var_size); + } else if(coords_type == TILEDB_INT8) { + return read_dense_attr_var( + attribute_id, + buffer, + buffer_size, + buffer_var, + buffer_var_size); + } else if(coords_type == TILEDB_UINT8) { + return read_dense_attr_var( + attribute_id, + buffer, + buffer_size, + buffer_var, + buffer_var_size); + } else if(coords_type == TILEDB_INT16) { + return read_dense_attr_var( + attribute_id, + buffer, + buffer_size, + buffer_var, + buffer_var_size); + } else if(coords_type == TILEDB_UINT16) { + return read_dense_attr_var( + attribute_id, + buffer, + buffer_size, + buffer_var, + buffer_var_size); + } else if(coords_type == TILEDB_UINT32) { + return read_dense_attr_var( + attribute_id, + buffer, + buffer_size, + buffer_var, + buffer_var_size); + } else if(coords_type == TILEDB_UINT64) { + return read_dense_attr_var( + attribute_id, + buffer, + buffer_size, + buffer_var, + buffer_var_size); } else { std::string errmsg = "Cannot read from array; Invalid coordinates type"; PRINT_ERROR(errmsg); @@ -1956,6 +2816,36 @@ int ArrayReadState::read_sparse_attr( attribute_id, buffer, buffer_size); + } else if(coords_type == TILEDB_INT8) { + return read_sparse_attr( + attribute_id, + buffer, + buffer_size); + } else if(coords_type == TILEDB_UINT8) { + return read_sparse_attr( + attribute_id, + buffer, + buffer_size); + } else if(coords_type == TILEDB_INT16) { + return read_sparse_attr( + attribute_id, + buffer, + buffer_size); + } else if(coords_type == TILEDB_UINT16) { + return read_sparse_attr( + attribute_id, + buffer, + buffer_size); + } else if(coords_type == TILEDB_UINT32) { + return read_sparse_attr( + attribute_id, + buffer, + buffer_size); + } else if(coords_type == TILEDB_UINT64) { + return read_sparse_attr( + attribute_id, + buffer, + buffer_size); } else { std::string errmsg = "Cannot read from array; Invalid coordinates type"; PRINT_ERROR(errmsg); @@ -2059,6 +2949,48 @@ int ArrayReadState::read_sparse_attr_var( buffer_size, buffer_var, buffer_var_size); + } else if(coords_type == TILEDB_INT8) { + return read_sparse_attr_var( + attribute_id, + buffer, + buffer_size, + buffer_var, + buffer_var_size); + } else if(coords_type == TILEDB_UINT8) { + return read_sparse_attr_var( + attribute_id, + buffer, + buffer_size, + buffer_var, + buffer_var_size); + } else if(coords_type == TILEDB_INT16) { + return read_sparse_attr_var( + attribute_id, + buffer, + buffer_size, + buffer_var, + buffer_var_size); + } else if(coords_type == TILEDB_UINT16) { + return read_sparse_attr_var( + attribute_id, + buffer, + buffer_size, + buffer_var, + buffer_var_size); + } else if(coords_type == TILEDB_UINT32) { + return read_sparse_attr_var( + attribute_id, + buffer, + buffer_size, + buffer_var, + buffer_var_size); + } else if(coords_type == TILEDB_UINT64) { + return read_sparse_attr_var( + attribute_id, + buffer, + buffer_size, + buffer_var, + buffer_var_size); } else { std::string errmsg = "Cannot read from array; Invalid coordinates type"; PRINT_ERROR(errmsg); @@ -2712,9 +3644,21 @@ template class ArrayReadState::PQFragmentCellRange; template class ArrayReadState::PQFragmentCellRange; template class ArrayReadState::PQFragmentCellRange; template class ArrayReadState::PQFragmentCellRange; +template class ArrayReadState::PQFragmentCellRange; +template class ArrayReadState::PQFragmentCellRange; +template class ArrayReadState::PQFragmentCellRange; +template class ArrayReadState::PQFragmentCellRange; +template class ArrayReadState::PQFragmentCellRange; +template class ArrayReadState::PQFragmentCellRange; template class ArrayReadState::SmallerPQFragmentCellRange; template class ArrayReadState::SmallerPQFragmentCellRange; template class ArrayReadState::SmallerPQFragmentCellRange; template class ArrayReadState::SmallerPQFragmentCellRange; +template class ArrayReadState::SmallerPQFragmentCellRange; +template class ArrayReadState::SmallerPQFragmentCellRange; +template class ArrayReadState::SmallerPQFragmentCellRange; +template class ArrayReadState::SmallerPQFragmentCellRange; +template class ArrayReadState::SmallerPQFragmentCellRange; +template class ArrayReadState::SmallerPQFragmentCellRange; diff --git a/core/src/array/array_schema.cc b/core/src/array/array_schema.cc index 005f356d9a7..0457e7336f7 100644 --- a/core/src/array/array_schema.cc +++ b/core/src/array/array_schema.cc @@ -52,6 +52,8 @@ # define PRINT_ERROR(x) do { } while(0) #endif +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define MAX(a,b) ((a) > (b) ? (a) : (b)) @@ -70,24 +72,16 @@ std::string tiledb_as_errmsg = ""; ArraySchema::ArraySchema() { cell_num_per_tile_ = -1; - coords_for_hilbert_ = NULL; domain_ = NULL; - hilbert_curve_ = NULL; tile_extents_ = NULL; tile_domain_ = NULL; tile_coords_aux_ = NULL; } ArraySchema::~ArraySchema() { - if(coords_for_hilbert_ != NULL) - delete [] coords_for_hilbert_; - if(domain_ != NULL) free(domain_); - if(hilbert_curve_ != NULL) - delete hilbert_curve_; - if(tile_extents_ != NULL) free(tile_extents_); @@ -344,6 +338,18 @@ bool ArraySchema::is_contained_in_tile_slab_col(const void* range) const { return is_contained_in_tile_slab_col(static_cast(range)); else if(types_[attribute_num_] == TILEDB_FLOAT64) return is_contained_in_tile_slab_col(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_INT8) + return is_contained_in_tile_slab_col(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_UINT8) + return is_contained_in_tile_slab_col(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_INT16) + return is_contained_in_tile_slab_col(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_UINT16) + return is_contained_in_tile_slab_col(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_UINT32) + return is_contained_in_tile_slab_col(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_UINT64) + return is_contained_in_tile_slab_col(static_cast(range)); else return false; } @@ -357,6 +363,18 @@ bool ArraySchema::is_contained_in_tile_slab_row(const void* range) const { return is_contained_in_tile_slab_row(static_cast(range)); else if(types_[attribute_num_] == TILEDB_FLOAT64) return is_contained_in_tile_slab_row(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_INT8) + return is_contained_in_tile_slab_row(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_UINT8) + return is_contained_in_tile_slab_row(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_INT16) + return is_contained_in_tile_slab_row(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_UINT16) + return is_contained_in_tile_slab_row(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_UINT32) + return is_contained_in_tile_slab_row(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_UINT64) + return is_contained_in_tile_slab_row(static_cast(range)); else return false; } @@ -398,6 +416,42 @@ void ArraySchema::print() const { std::cout << "\t" << dimensions_[i] << ": [" << domain_double[2*i] << "," << domain_double[2*i+1] << "]\n"; } + } else if(types_[attribute_num_] == TILEDB_INT8) { + int8_t* domain_int8 = (int8_t*) domain_; + for(int i=0; i(); else if(types_[attribute_num_] == TILEDB_INT64) return tile_num(); + else if(types_[attribute_num_] == TILEDB_INT8) + return tile_num(); + else if(types_[attribute_num_] == TILEDB_UINT8) + return tile_num(); + else if(types_[attribute_num_] == TILEDB_INT16) + return tile_num(); + else if(types_[attribute_num_] == TILEDB_UINT16) + return tile_num(); + else if(types_[attribute_num_] == TILEDB_UINT32) + return tile_num(); + else if(types_[attribute_num_] == TILEDB_UINT64) + return tile_num(); assert(0); std::string errmsg = @@ -785,7 +905,18 @@ int64_t ArraySchema::tile_num(const void* range) const { return tile_num(static_cast(range)); else if(types_[attribute_num_] == TILEDB_INT64) return tile_num(static_cast(range)); - + else if(types_[attribute_num_] == TILEDB_INT8) + return tile_num(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_UINT8) + return tile_num(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_INT16) + return tile_num(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_UINT16) + return tile_num(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_UINT32) + return tile_num(static_cast(range)); + else if(types_[attribute_num_] == TILEDB_UINT64) + return tile_num(static_cast(range)); assert(0); std::string errmsg = @@ -827,6 +958,18 @@ int64_t ArraySchema::tile_slab_col_cell_num(const void* subarray) const { return tile_slab_col_cell_num(static_cast(subarray)); else if(types_[attribute_num_] == TILEDB_FLOAT64) return tile_slab_col_cell_num(static_cast(subarray)); + else if(types_[attribute_num_] == TILEDB_INT8) + return tile_slab_col_cell_num(static_cast(subarray)); + else if(types_[attribute_num_] == TILEDB_UINT8) + return tile_slab_col_cell_num(static_cast(subarray)); + else if(types_[attribute_num_] == TILEDB_INT16) + return tile_slab_col_cell_num(static_cast(subarray)); + else if(types_[attribute_num_] == TILEDB_UINT16) + return tile_slab_col_cell_num(static_cast(subarray)); + else if(types_[attribute_num_] == TILEDB_UINT32) + return tile_slab_col_cell_num(static_cast(subarray)); + else if(types_[attribute_num_] == TILEDB_UINT64) + return tile_slab_col_cell_num(static_cast(subarray)); else return TILEDB_AS_ERR; } @@ -841,6 +984,18 @@ int64_t ArraySchema::tile_slab_row_cell_num(const void* subarray) const { return tile_slab_row_cell_num(static_cast(subarray)); else if(types_[attribute_num_] == TILEDB_FLOAT64) return tile_slab_row_cell_num(static_cast(subarray)); + else if(types_[attribute_num_] == TILEDB_INT8) + return tile_slab_row_cell_num(static_cast(subarray)); + else if(types_[attribute_num_] == TILEDB_UINT8) + return tile_slab_row_cell_num(static_cast(subarray)); + else if(types_[attribute_num_] == TILEDB_INT16) + return tile_slab_row_cell_num(static_cast(subarray)); + else if(types_[attribute_num_] == TILEDB_UINT16) + return tile_slab_row_cell_num(static_cast(subarray)); + else if(types_[attribute_num_] == TILEDB_UINT32) + return tile_slab_row_cell_num(static_cast(subarray)); + else if(types_[attribute_num_] == TILEDB_UINT64) + return tile_slab_row_cell_num(static_cast(subarray)); else return TILEDB_AS_ERR; @@ -1041,9 +1196,6 @@ int ArraySchema::deserialize( // Compute tile offsets compute_tile_offsets(); - // Initialize Hilbert curve - init_hilbert_curve(); - // Initialize static auxiliary variables if(tile_coords_aux_ != NULL) free(tile_coords_aux_); @@ -1100,9 +1252,6 @@ int ArraySchema::init(const ArraySchemaC* array_schema_c) { // Compute tile offsets compute_tile_offsets(); - // Initialize Hilbert curve - init_hilbert_curve(); - // Initialize static auxiliary variables if(tile_coords_aux_ != NULL) free(tile_coords_aux_); @@ -1170,7 +1319,7 @@ int ArraySchema::init(const MetadataSchemaC* metadata_schema_c) { for(int i=0; iattribute_num_; ++i) types[i] = metadata_schema_c->types_[i]; types[metadata_schema_c->attribute_num_] = TILEDB_CHAR; - types[metadata_schema_c->attribute_num_+1] = TILEDB_INT32; + types[metadata_schema_c->attribute_num_+1] = TILEDB_UINT32; array_schema_c.types_ = types; // Set cell num val @@ -1299,8 +1448,7 @@ void ArraySchema::set_cell_val_num(const int* cell_val_num) { int ArraySchema::set_cell_order(int cell_order) { // Set cell order if(cell_order != TILEDB_ROW_MAJOR && - cell_order != TILEDB_COL_MAJOR && - cell_order != TILEDB_HILBERT) { + cell_order != TILEDB_COL_MAJOR) { std::string errmsg = "Cannot set cell order; Invalid cell order"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; @@ -1329,7 +1477,8 @@ int ArraySchema::set_compression(int* compression) { compression[i] != TILEDB_BLOSC_SNAPPY && compression[i] != TILEDB_BLOSC_ZLIB && compression[i] != TILEDB_BLOSC_ZSTD && - compression[i] != TILEDB_RLE) { + compression[i] != TILEDB_RLE && + compression[i] != TILEDB_BZIP2) { std::string errmsg = "Cannot set compression; Invalid compression type"; PRINT_ERROR(errmsg); tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; @@ -1461,6 +1610,78 @@ int ArraySchema::set_domain(const void* domain) { return TILEDB_AS_ERR; } } + } else if(types_[attribute_num_] == TILEDB_INT8) { + int8_t* domain_int8 = (int8_t*) domain_; + for(int i=0; i domain_int8[2*i+1]) { + std::string errmsg = + "Cannot set domain; Lower domain bound larger than its " + "corresponding upper"; + PRINT_ERROR(errmsg); + tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; + return TILEDB_AS_ERR; + } + } + } else if(types_[attribute_num_] == TILEDB_UINT8) { + uint8_t* domain_uint8 = (uint8_t*) domain_; + for(int i=0; i domain_uint8[2*i+1]) { + std::string errmsg = + "Cannot set domain; Lower domain bound larger than its " + "corresponding upper"; + PRINT_ERROR(errmsg); + tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; + return TILEDB_AS_ERR; + } + } + } else if(types_[attribute_num_] == TILEDB_INT16) { + int16_t* domain_int16 = (int16_t*) domain_; + for(int i=0; i domain_int16[2*i+1]) { + std::string errmsg = + "Cannot set domain; Lower domain bound larger than its " + "corresponding upper"; + PRINT_ERROR(errmsg); + tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; + return TILEDB_AS_ERR; + } + } + } else if(types_[attribute_num_] == TILEDB_UINT16) { + uint16_t* domain_uint16 = (uint16_t*) domain_; + for(int i=0; i domain_uint16[2*i+1]) { + std::string errmsg = + "Cannot set domain; Lower domain bound larger than its " + "corresponding upper"; + PRINT_ERROR(errmsg); + tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; + return TILEDB_AS_ERR; + } + } + } else if(types_[attribute_num_] == TILEDB_UINT32) { + uint32_t* domain_uint32 = (uint32_t*) domain_; + for(int i=0; i domain_uint32[2*i+1]) { + std::string errmsg = + "Cannot set domain; Lower domain bound larger than its " + "corresponding upper"; + PRINT_ERROR(errmsg); + tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; + return TILEDB_AS_ERR; + } + } + } else if(types_[attribute_num_] == TILEDB_UINT64) { + uint64_t* domain_uint64 = (uint64_t*) domain_; + for(int i=0; i domain_uint64[2*i+1]) { + std::string errmsg = + "Cannot set domain; Lower domain bound larger than its " + "corresponding upper"; + PRINT_ERROR(errmsg); + tiledb_as_errmsg = TILEDB_AS_ERRMSG + errmsg; + return TILEDB_AS_ERR; + } + } } else { std::string errmsg = "Cannot set domain; Invalid coordinates type"; PRINT_ERROR(errmsg); @@ -1525,10 +1746,16 @@ int ArraySchema::set_types(const int* types) { // Set attribute types for(int i=0; i coords_b[i]) return 1; } - } else if(cell_order_ == TILEDB_HILBERT) { // HILBERT - // Check hilbert ids - int64_t id_a = hilbert_id(coords_a); - int64_t id_b = hilbert_id(coords_b); - - if(id_a < id_b) - return -1; - else if(id_a > id_b) - return 1; - - // Hilbert ids match - check coordinates in row-major order - for(int i=0; i coords_b[i]) - return 1; - } } else { // Invalid cell order assert(0); } @@ -1625,6 +1841,18 @@ void ArraySchema::expand_domain(void* domain) const { expand_domain(static_cast(domain)); else if(types_[attribute_num_] == TILEDB_INT64) expand_domain(static_cast(domain)); + else if(types_[attribute_num_] == TILEDB_INT8) + expand_domain(static_cast(domain)); + else if(types_[attribute_num_] == TILEDB_UINT8) + expand_domain(static_cast(domain)); + else if(types_[attribute_num_] == TILEDB_INT16) + expand_domain(static_cast(domain)); + else if(types_[attribute_num_] == TILEDB_UINT16) + expand_domain(static_cast(domain)); + else if(types_[attribute_num_] == TILEDB_UINT32) + expand_domain(static_cast(domain)); + else if(types_[attribute_num_] == TILEDB_UINT64) + expand_domain(static_cast(domain)); } template @@ -1739,10 +1967,10 @@ void ArraySchema::get_subarray_tile_domain( // Calculate subarray in tile domain for(int i=0; i -int64_t ArraySchema::hilbert_id(const T* coords) const { - // For easy reference - const T* domain = static_cast(domain_); - - // Normalize coordinates - for(int i = 0; i < dim_num_; ++i) - coords_for_hilbert_[i] = static_cast(coords[i] - domain[2*i]); - - // Compute Hilber id - int64_t id; - hilbert_curve_->coords_to_hilbert(coords_for_hilbert_, id); - - // Return - return id; -} - template int ArraySchema::tile_cell_order_cmp( const T* coords_a, @@ -1848,262 +2059,30 @@ int64_t ArraySchema::tile_id(const T* cell_coords) const { for(int i=0; i -int ArraySchema::tile_order_cmp( - const int* coords_a, - const int* coords_b) const { - // For easy reference - int diff; - int norm; - const int* domain = static_cast(domain_); - const int* tile_extents = static_cast(tile_extents_); - - // If there are regular tiles, first check tile order - if(tile_extents_ != NULL) { - // ROW-MAJOR - if(tile_order_ == TILEDB_ROW_MAJOR) { - // Check if the cells are definitely IN the same tile - for(int i=0; i 0) - norm = (coords_b[i] - domain[2*i]) % tile_extents[i]; - - if(diff < 0 && (norm - diff) >= tile_extents[i]) - return -1; - else if(diff > 0 && (norm + diff) >= tile_extents[i]) - return 1; - } - } else { // COLUMN-MAJOR - // Check if the cells are definitely IN the same tile - for(int i=dim_num_-1; i>=0; --i) { - diff = coords_a[i] - coords_b[i]; - - if(diff < 0) - norm = (coords_a[i] - domain[2*i]) % tile_extents[i]; - else if(diff > 0) - norm = (coords_b[i] - domain[2*i]) % tile_extents[i]; - - if(diff < 0 && (norm - diff) >= tile_extents[i]) - return -1; - else if(diff > 0 && (norm + diff) >= tile_extents[i]) - return 1; - - - if(diff < 0 && (norm - diff) >= tile_extents[i]) - return -1; - else if(diff > 0 && (norm + diff) >= tile_extents[i]) - return 1; - } - } - } - - // Same tile order - return 0; -} - -template<> -int ArraySchema::tile_order_cmp( - const int64_t* coords_a, - const int64_t* coords_b) const { - // For easy reference - int64_t diff; - int64_t norm; - const int64_t* domain = static_cast(domain_); - const int64_t* tile_extents = static_cast(tile_extents_); - - // If there are regular tiles, first check tile order - if(tile_extents_ != NULL) { - // ROW-MAJOR - if(tile_order_ == TILEDB_ROW_MAJOR) { - // Check if the cells are definitely IN the same tile - for(int i=0; i 0) - norm = (coords_b[i] - domain[2*i]) % tile_extents[i]; - - if(diff < 0 && (norm - diff) >= tile_extents[i]) - return -1; - else if(diff > 0 && (norm + diff) >= tile_extents[i]) - return 1; - } - } else { // COLUMN-MAJOR - // Check if the cells are definitely IN the same tile - for(int i=dim_num_-1; i>=0; --i) { - diff = coords_a[i] - coords_b[i]; - - if(diff < 0) - norm = (coords_a[i] - domain[2*i]) % tile_extents[i]; - else if(diff > 0) - norm = (coords_b[i] - domain[2*i]) % tile_extents[i]; - - if(diff < 0 && (norm - diff) >= tile_extents[i]) - return -1; - else if(diff > 0 && (norm + diff) >= tile_extents[i]) - return 1; - - - if(diff < 0 && (norm - diff) >= tile_extents[i]) - return -1; - else if(diff > 0 && (norm + diff) >= tile_extents[i]) - return 1; - } - } - } - - // Same tile order - return 0; -} - -template<> -int ArraySchema::tile_order_cmp( - const float* coords_a, - const float* coords_b) const { - // For easy reference - float diff; - float norm, norm_temp; - const float* domain = static_cast(domain_); - const float* tile_extents = static_cast(tile_extents_); - - // If there are regular tiles, first check tile order - if(tile_extents_ != NULL) { - // ROW-MAJOR - if(tile_order_ == TILEDB_ROW_MAJOR) { - // Check if the cells are definitely IN the same tile - for(int i=0; i= domain[2*i]); - } else if(diff > 0) { - norm_temp = coords_b[i]; - do { - norm = norm_temp; - norm_temp -= tile_extents[i]; - } while(norm_temp >= domain[2*i]); - } - - if(diff < 0 && (norm - diff) >= tile_extents[i]) - return -1; - else if(diff > 0 && (norm + diff) >= tile_extents[i]) - return 1; - } - } else { // COLUMN-MAJOR - // Check if the cells are definitely IN the same tile - for(int i=dim_num_-1; i>=0; --i) { - diff = coords_a[i] - coords_b[i]; - - if(diff < 0) { - norm_temp = coords_a[i]; - do { - norm = norm_temp; - norm_temp -= tile_extents[i]; - } while(norm_temp >= domain[2*i]); - } else if(diff > 0) { - norm_temp = coords_b[i]; - do { - norm = norm_temp; - norm_temp -= tile_extents[i]; - } while(norm_temp >= domain[2*i]); - } - - if(diff < 0 && (norm - diff) >= tile_extents[i]) - return -1; - else if(diff > 0 && (norm + diff) >= tile_extents[i]) - return 1; - } - } - } - - // Same tile order - return 0; -} -template<> +template int ArraySchema::tile_order_cmp( - const double* coords_a, - const double* coords_b) const { - // For easy reference - double diff; - double norm, norm_temp; - const double* domain = static_cast(domain_); - const double* tile_extents = static_cast(tile_extents_); - - // If there are regular tiles, first check tile order - if(tile_extents_ != NULL) { - // ROW-MAJOR - if(tile_order_ == TILEDB_ROW_MAJOR) { - // Check if the cells are definitely IN the same tile - for(int i=0; i= domain[2*i]); - } else if(diff > 0) { - norm_temp = coords_b[i]; - do { - norm = norm_temp; - norm_temp -= tile_extents[i]; - } while(norm_temp >= domain[2*i]); - } - - if(diff < 0 && (norm - diff) >= tile_extents[i]) - return -1; - else if(diff > 0 && (norm + diff) >= tile_extents[i]) - return 1; - } - } else { // COLUMN-MAJOR - // Check if the cells are definitely IN the same tile - for(int i=dim_num_-1; i>=0; --i) { - diff = coords_a[i] - coords_b[i]; - - if(diff < 0) { - norm_temp = coords_a[i]; - do { - norm = norm_temp; - norm_temp -= tile_extents[i]; - } while(norm_temp >= domain[2*i]); - } else if(diff > 0) { - norm_temp = coords_b[i]; - do { - norm = norm_temp; - norm_temp -= tile_extents[i]; - } while(norm_temp >= domain[2*i]); - } - - if(diff < 0 && (norm - diff) >= tile_extents[i]) - return -1; - else if(diff > 0 && (norm + diff) >= tile_extents[i]) - return 1; - } - } - } - - // Same tile order - return 0; + const T* coords_a, + const T* coords_b) const { + // Calculate tile ids + int64_t id_a = tile_id(coords_a); + int64_t id_b = tile_id(coords_b); + + // Compare ids + if(id_a < id_b) + return -1; + else if(id_a > id_b) + return 1; + else // id_a == id_b + return 0; } - - /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ @@ -2177,6 +2156,18 @@ void ArraySchema::compute_cell_num_per_tile() { compute_cell_num_per_tile(); else if(coords_type == TILEDB_INT64) compute_cell_num_per_tile(); + else if(coords_type == TILEDB_INT8) + compute_cell_num_per_tile(); + else if(coords_type == TILEDB_UINT8) + compute_cell_num_per_tile(); + else if(coords_type == TILEDB_INT16) + compute_cell_num_per_tile(); + else if(coords_type == TILEDB_UINT16) + compute_cell_num_per_tile(); + else if(coords_type == TILEDB_UINT32) + compute_cell_num_per_tile(); + else if(coords_type == TILEDB_UINT64) + compute_cell_num_per_tile(); else // Sanity check assert(0); } @@ -2212,6 +2203,18 @@ size_t ArraySchema::compute_cell_size(int i) const { size = cell_val_num_[i] * sizeof(float); else if(types_[i] == TILEDB_FLOAT64) size = cell_val_num_[i] * sizeof(double); + else if(types_[i] == TILEDB_INT8) + size = cell_val_num_[i] * sizeof(int8_t); + else if(types_[i] == TILEDB_UINT8) + size = cell_val_num_[i] * sizeof(uint8_t); + else if(types_[i] == TILEDB_INT16) + size = cell_val_num_[i] * sizeof(int16_t); + else if(types_[i] == TILEDB_UINT16) + size = cell_val_num_[i] * sizeof(uint16_t); + else if(types_[i] == TILEDB_UINT32) + size = cell_val_num_[i] * sizeof(uint32_t); + else if(types_[i] == TILEDB_UINT64) + size = cell_val_num_[i] * sizeof(uint64_t); } else { // Coordinates if(types_[i] == TILEDB_INT32) size = dim_num_ * sizeof(int); @@ -2221,27 +2224,23 @@ size_t ArraySchema::compute_cell_size(int i) const { size = dim_num_ * sizeof(float); else if(types_[i] == TILEDB_FLOAT64) size = dim_num_ * sizeof(double); + else if(types_[i] == TILEDB_INT8) + size = dim_num_ * sizeof(int8_t); + else if(types_[i] == TILEDB_UINT8) + size = dim_num_ * sizeof(uint8_t); + else if(types_[i] == TILEDB_INT16) + size = dim_num_ * sizeof(int16_t); + else if(types_[i] == TILEDB_UINT16) + size = dim_num_ * sizeof(uint16_t); + else if(types_[i] == TILEDB_UINT32) + size = dim_num_ * sizeof(uint32_t); + else if(types_[i] == TILEDB_UINT64) + size = dim_num_ * sizeof(uint64_t); } return size; } -template -void ArraySchema::compute_hilbert_bits() { - // For easy reference - const T* domain = static_cast(domain_); - T max_domain_range = 0; - T domain_range; - - for(int i = 0; i < dim_num_; ++i) { - domain_range = domain[2*i+1] - domain[2*i] + 1; - if(max_domain_range < domain_range) - max_domain_range = domain_range; - } - - hilbert_bits_ = ceil(log2(int64_t(max_domain_range+0.5))); -} - void ArraySchema::compute_tile_domain() { // For easy reference int coords_type = types_[attribute_num_]; @@ -2255,6 +2254,18 @@ void ArraySchema::compute_tile_domain() { compute_tile_domain(); else if(coords_type == TILEDB_FLOAT64) compute_tile_domain(); + else if(coords_type == TILEDB_INT8) + compute_tile_domain(); + else if(coords_type == TILEDB_UINT8) + compute_tile_domain(); + else if(coords_type == TILEDB_INT16) + compute_tile_domain(); + else if(coords_type == TILEDB_UINT16) + compute_tile_domain(); + else if(coords_type == TILEDB_UINT32) + compute_tile_domain(); + else if(coords_type == TILEDB_UINT64) + compute_tile_domain(); } template @@ -2292,6 +2303,18 @@ void ArraySchema::compute_tile_offsets() { compute_tile_offsets(); } else if(types_[attribute_num_] == TILEDB_FLOAT64) { compute_tile_offsets(); + } else if(types_[attribute_num_] == TILEDB_INT8) { + compute_tile_offsets(); + } else if(types_[attribute_num_] == TILEDB_UINT8) { + compute_tile_offsets(); + } else if(types_[attribute_num_] == TILEDB_INT16) { + compute_tile_offsets(); + } else if(types_[attribute_num_] == TILEDB_UINT16) { + compute_tile_offsets(); + } else if(types_[attribute_num_] == TILEDB_UINT32) { + compute_tile_offsets(); + } else if(types_[attribute_num_] == TILEDB_UINT64) { + compute_tile_offsets(); } else { // The program should never reach this point assert(0); } @@ -2340,6 +2363,18 @@ size_t ArraySchema::compute_type_size(int i) const { return sizeof(float); } else if(types_[i] == TILEDB_FLOAT64) { return sizeof(double); + } else if(types_[i] == TILEDB_INT8) { + return sizeof(int8_t); + } else if(types_[i] == TILEDB_UINT8) { + return sizeof(uint8_t); + } else if(types_[i] == TILEDB_INT16) { + return sizeof(int16_t); + } else if(types_[i] == TILEDB_UINT16) { + return sizeof(uint16_t); + } else if(types_[i] == TILEDB_UINT32) { + return sizeof(uint32_t); + } else if(types_[i] == TILEDB_UINT64) { + return sizeof(uint64_t); } else { // The program should never reach this point assert(0); return 0; @@ -2568,29 +2603,6 @@ int64_t ArraySchema::get_tile_pos_row( return pos; } -void ArraySchema::init_hilbert_curve() { - // Applicable only to Hilbert cell order - if(cell_order_ != TILEDB_HILBERT) - return; - - // Allocate some space for the Hilbert coordinates - if(coords_for_hilbert_ == NULL) - coords_for_hilbert_ = new int[dim_num_]; - - // Compute Hilbert bits, invoking the proper templated function - if(types_[attribute_num_] == TILEDB_INT32) - compute_hilbert_bits(); - else if(types_[attribute_num_] == TILEDB_INT64) - compute_hilbert_bits(); - else if(types_[attribute_num_] == TILEDB_FLOAT32) - compute_hilbert_bits(); - else if(types_[attribute_num_] == TILEDB_FLOAT64) - compute_hilbert_bits(); - - // Create new Hilberrt curve - hilbert_curve_ = new HilbertCurve(hilbert_bits_, dim_num_); -} - template bool ArraySchema::is_contained_in_tile_slab_col(const T* range) const { // For easy reference @@ -2637,7 +2649,7 @@ int64_t ArraySchema::tile_slab_col_cell_num(const T* subarray) const { // Initialize the cell num to be returned to the maximum number of rows // in the slab int64_t cell_num = - std::min( + MIN( tile_extents[dim_num_-1], subarray[2*(dim_num_-1)+1] - subarray[2*(dim_num_-1)] + 1); @@ -2656,7 +2668,7 @@ int64_t ArraySchema::tile_slab_row_cell_num(const T* subarray) const { // Initialize the cell num to be returned to the maximum number of rows // in the slab - int64_t cell_num = std::min(tile_extents[0], subarray[1] - subarray[0] + 1); + int64_t cell_num = MIN(tile_extents[0], subarray[1] - subarray[0] + 1); // Calculate the number of cells in the slab for(int i=1; i( template int ArraySchema::cell_order_cmp( const double* coords_a, const double* coords_b) const; +template int ArraySchema::cell_order_cmp( + const int8_t* coords_a, + const int8_t* coords_b) const; +template int ArraySchema::cell_order_cmp( + const uint8_t* coords_a, + const uint8_t* coords_b) const; +template int ArraySchema::cell_order_cmp( + const int16_t* coords_a, + const int16_t* coords_b) const; +template int ArraySchema::cell_order_cmp( + const uint16_t* coords_a, + const uint16_t* coords_b) const; +template int ArraySchema::cell_order_cmp( + const uint32_t* coords_a, + const uint32_t* coords_b) const; +template int ArraySchema::cell_order_cmp( + const uint64_t* coords_a, + const uint64_t* coords_b) const; template int64_t ArraySchema::get_cell_pos( const int* coords) const; @@ -2692,6 +2722,18 @@ template int64_t ArraySchema::get_cell_pos( const float* coords) const; template int64_t ArraySchema::get_cell_pos( const double* coords) const; +template int64_t ArraySchema::get_cell_pos( + const int8_t* coords) const; +template int64_t ArraySchema::get_cell_pos( + const uint8_t* coords) const; +template int64_t ArraySchema::get_cell_pos( + const int16_t* coords) const; +template int64_t ArraySchema::get_cell_pos( + const uint16_t* coords) const; +template int64_t ArraySchema::get_cell_pos( + const uint32_t* coords) const; +template int64_t ArraySchema::get_cell_pos( + const uint64_t* coords) const; template void ArraySchema::get_next_cell_coords( const int* domain, @@ -2710,6 +2752,31 @@ template void ArraySchema::get_next_cell_coords( double* cell_coords, bool& coords_retrieved) const; +template void ArraySchema::get_next_cell_coords( + const int8_t* domain, + int8_t* cell_coords, + bool& coords_retrieved) const; +template void ArraySchema::get_next_cell_coords( + const uint8_t* domain, + uint8_t* cell_coords, + bool& coords_retrieved) const; +template void ArraySchema::get_next_cell_coords( + const int16_t* domain, + int16_t* cell_coords, + bool& coords_retrieved) const; +template void ArraySchema::get_next_cell_coords( + const uint16_t* domain, + uint16_t* cell_coords, + bool& coords_retrieved) const; +template void ArraySchema::get_next_cell_coords( + const uint32_t* domain, + uint32_t* cell_coords, + bool& coords_retrieved) const; +template void ArraySchema::get_next_cell_coords( + const uint64_t* domain, + uint64_t* cell_coords, + bool& coords_retrieved) const; + template void ArraySchema::get_next_tile_coords( const int* domain, int* tile_coords) const; @@ -2722,6 +2789,24 @@ template void ArraySchema::get_next_tile_coords( template void ArraySchema::get_next_tile_coords( const double* domain, double* tile_coords) const; +template void ArraySchema::get_next_tile_coords( + const int8_t* domain, + int8_t* tile_coords) const; +template void ArraySchema::get_next_tile_coords( + const uint8_t* domain, + uint8_t* tile_coords) const; +template void ArraySchema::get_next_tile_coords( + const int16_t* domain, + int16_t* tile_coords) const; +template void ArraySchema::get_next_tile_coords( + const uint16_t* domain, + uint16_t* tile_coords) const; +template void ArraySchema::get_next_tile_coords( + const uint32_t* domain, + uint32_t* tile_coords) const; +template void ArraySchema::get_next_tile_coords( + const uint64_t* domain, + uint64_t* tile_coords) const; template void ArraySchema::get_previous_cell_coords( const int* domain, @@ -2735,6 +2820,24 @@ template void ArraySchema::get_previous_cell_coords( template void ArraySchema::get_previous_cell_coords( const double* domain, double* cell_coords) const; +template void ArraySchema::get_previous_cell_coords( + const int8_t* domain, + int8_t* cell_coords) const; +template void ArraySchema::get_previous_cell_coords( + const uint8_t* domain, + uint8_t* cell_coords) const; +template void ArraySchema::get_previous_cell_coords( + const int16_t* domain, + int16_t* cell_coords) const; +template void ArraySchema::get_previous_cell_coords( + const uint16_t* domain, + uint16_t* cell_coords) const; +template void ArraySchema::get_previous_cell_coords( + const uint32_t* domain, + uint32_t* cell_coords) const; +template void ArraySchema::get_previous_cell_coords( + const uint64_t* domain, + uint64_t* cell_coords) const; template void ArraySchema::get_subarray_tile_domain( const int* subarray, @@ -2744,6 +2847,30 @@ template void ArraySchema::get_subarray_tile_domain( const int64_t* subarray, int64_t* tile_domain, int64_t* subarray_tile_domain) const; +template void ArraySchema::get_subarray_tile_domain( + const int8_t* subarray, + int8_t* tile_domain, + int8_t* subarray_tile_domain) const; +template void ArraySchema::get_subarray_tile_domain( + const uint8_t* subarray, + uint8_t* tile_domain, + uint8_t* subarray_tile_domain) const; +template void ArraySchema::get_subarray_tile_domain( + const int16_t* subarray, + int16_t* tile_domain, + int16_t* subarray_tile_domain) const; +template void ArraySchema::get_subarray_tile_domain( + const uint16_t* subarray, + uint16_t* tile_domain, + uint16_t* subarray_tile_domain) const; +template void ArraySchema::get_subarray_tile_domain( + const uint32_t* subarray, + uint32_t* tile_domain, + uint32_t* subarray_tile_domain) const; +template void ArraySchema::get_subarray_tile_domain( + const uint64_t* subarray, + uint64_t* tile_domain, + uint64_t* subarray_tile_domain) const; template int64_t ArraySchema::get_tile_pos( const int* domain, @@ -2757,6 +2884,24 @@ template int64_t ArraySchema::get_tile_pos( template int64_t ArraySchema::get_tile_pos( const double* domain, const double* tile_coords) const; +template int64_t ArraySchema::get_tile_pos( + const int8_t* domain, + const int8_t* tile_coords) const; +template int64_t ArraySchema::get_tile_pos( + const uint8_t* domain, + const uint8_t* tile_coords) const; +template int64_t ArraySchema::get_tile_pos( + const int16_t* domain, + const int16_t* tile_coords) const; +template int64_t ArraySchema::get_tile_pos( + const uint16_t* domain, + const uint16_t* tile_coords) const; +template int64_t ArraySchema::get_tile_pos( + const uint32_t* domain, + const uint32_t* tile_coords) const; +template int64_t ArraySchema::get_tile_pos( + const uint64_t* domain, + const uint64_t* tile_coords) const; template void ArraySchema::get_tile_subarray( const int* tile_coords, @@ -2764,15 +2909,24 @@ template void ArraySchema::get_tile_subarray( template void ArraySchema::get_tile_subarray( const int64_t* tile_coords, int64_t* tile_subarray) const; - -template int64_t ArraySchema::hilbert_id( - const int* coords) const; -template int64_t ArraySchema::hilbert_id( - const int64_t* coords) const; -template int64_t ArraySchema::hilbert_id( - const float* coords) const; -template int64_t ArraySchema::hilbert_id( - const double* coords) const; +template void ArraySchema::get_tile_subarray( + const int8_t* tile_coords, + int8_t* tile_subarray) const; +template void ArraySchema::get_tile_subarray( + const uint8_t* tile_coords, + uint8_t* tile_subarray) const; +template void ArraySchema::get_tile_subarray( + const int16_t* tile_coords, + int16_t* tile_subarray) const; +template void ArraySchema::get_tile_subarray( + const uint16_t* tile_coords, + uint16_t* tile_subarray) const; +template void ArraySchema::get_tile_subarray( + const uint32_t* tile_coords, + uint32_t* tile_subarray) const; +template void ArraySchema::get_tile_subarray( + const uint64_t* tile_coords, + uint64_t* tile_subarray) const; template bool ArraySchema::is_contained_in_tile_slab_col( const int* range) const; @@ -2782,6 +2936,18 @@ template bool ArraySchema::is_contained_in_tile_slab_col( const float* range) const; template bool ArraySchema::is_contained_in_tile_slab_col( const double* range) const; +template bool ArraySchema::is_contained_in_tile_slab_col( + const int8_t* range) const; +template bool ArraySchema::is_contained_in_tile_slab_col( + const uint8_t* range) const; +template bool ArraySchema::is_contained_in_tile_slab_col( + const int16_t* range) const; +template bool ArraySchema::is_contained_in_tile_slab_col( + const uint16_t* range) const; +template bool ArraySchema::is_contained_in_tile_slab_col( + const uint32_t* range) const; +template bool ArraySchema::is_contained_in_tile_slab_col( + const uint64_t* range) const; template bool ArraySchema::is_contained_in_tile_slab_row( const int* range) const; @@ -2791,6 +2957,18 @@ template bool ArraySchema::is_contained_in_tile_slab_row( const float* range) const; template bool ArraySchema::is_contained_in_tile_slab_row( const double* range) const; +template bool ArraySchema::is_contained_in_tile_slab_row( + const int8_t* range) const; +template bool ArraySchema::is_contained_in_tile_slab_row( + const uint8_t* range) const; +template bool ArraySchema::is_contained_in_tile_slab_row( + const int16_t* range) const; +template bool ArraySchema::is_contained_in_tile_slab_row( + const uint16_t* range) const; +template bool ArraySchema::is_contained_in_tile_slab_row( + const uint32_t* range) const; +template bool ArraySchema::is_contained_in_tile_slab_row( + const uint64_t* range) const; template int ArraySchema::subarray_overlap( const int* subarray_a, @@ -2808,6 +2986,30 @@ template int ArraySchema::subarray_overlap( const double* subarray_a, const double* subarray_b, double* overlap_subarray) const; +template int ArraySchema::subarray_overlap( + const int8_t* subarray_a, + const int8_t* subarray_b, + int8_t* overlap_subarray) const; +template int ArraySchema::subarray_overlap( + const uint8_t* subarray_a, + const uint8_t* subarray_b, + uint8_t* overlap_subarray) const; +template int ArraySchema::subarray_overlap( + const int16_t* subarray_a, + const int16_t* subarray_b, + int16_t* overlap_subarray) const; +template int ArraySchema::subarray_overlap( + const uint16_t* subarray_a, + const uint16_t* subarray_b, + uint16_t* overlap_subarray) const; +template int ArraySchema::subarray_overlap( + const uint32_t* subarray_a, + const uint32_t* subarray_b, + uint32_t* overlap_subarray) const; +template int ArraySchema::subarray_overlap( + const uint64_t* subarray_a, + const uint64_t* subarray_b, + uint64_t* overlap_subarray) const; template int ArraySchema::tile_cell_order_cmp( const int* coords_a, @@ -2821,6 +3023,24 @@ template int ArraySchema::tile_cell_order_cmp( template int ArraySchema::tile_cell_order_cmp( const double* coords_a, const double* coords_b) const; +template int ArraySchema::tile_cell_order_cmp( + const int8_t* coords_a, + const int8_t* coords_b) const; +template int ArraySchema::tile_cell_order_cmp( + const uint8_t* coords_a, + const uint8_t* coords_b) const; +template int ArraySchema::tile_cell_order_cmp( + const int16_t* coords_a, + const int16_t* coords_b) const; +template int ArraySchema::tile_cell_order_cmp( + const uint16_t* coords_a, + const uint16_t* coords_b) const; +template int ArraySchema::tile_cell_order_cmp( + const uint32_t* coords_a, + const uint32_t* coords_b) const; +template int ArraySchema::tile_cell_order_cmp( + const uint64_t* coords_a, + const uint64_t* coords_b) const; template int64_t ArraySchema::tile_id( const int* cell_coords) const; @@ -2830,5 +3050,17 @@ template int64_t ArraySchema::tile_id( const float* cell_coords) const; template int64_t ArraySchema::tile_id( const double* cell_coords) const; +template int64_t ArraySchema::tile_id( + const int8_t* cell_coords) const; +template int64_t ArraySchema::tile_id( + const uint8_t* cell_coords) const; +template int64_t ArraySchema::tile_id( + const int16_t* cell_coords) const; +template int64_t ArraySchema::tile_id( + const uint16_t* cell_coords) const; +template int64_t ArraySchema::tile_id( + const uint32_t* cell_coords) const; +template int64_t ArraySchema::tile_id( + const uint64_t* cell_coords) const; diff --git a/core/src/array/array_sorted_read_state.cc b/core/src/array/array_sorted_read_state.cc index 6f65129f39b..e3d8347f6c1 100644 --- a/core/src/array/array_sorted_read_state.cc +++ b/core/src/array/array_sorted_read_state.cc @@ -57,6 +57,9 @@ #define SORT(first, last, comp) std::sort((first), (last), (comp)) #endif +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define MAX(a,b) ((a) > (b) ? (a) : (b)) + /* ****************************** */ @@ -287,6 +290,18 @@ int ArraySortedReadState::read(void** buffers, size_t* buffer_sizes) { return read(); } else if(type == TILEDB_FLOAT64) { return read(); + } else if(type == TILEDB_INT8) { + return read(); + } else if(type == TILEDB_UINT8) { + return read(); + } else if(type == TILEDB_INT16) { + return read(); + } else if(type == TILEDB_UINT16) { + return read(); + } else if(type == TILEDB_UINT32) { + return read(); + } else if(type == TILEDB_UINT64) { + return read(); } else { assert(0); return TILEDB_ASRS_ERR; @@ -382,6 +397,42 @@ int ArraySortedReadState::init() { (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_row_row_s : calculate_cell_slab_info_row_col_s; + } else if(coords_type == TILEDB_INT8) { + advance_cell_slab_ = advance_cell_slab_row_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_row_row_s : + calculate_cell_slab_info_row_col_s; + } else if(coords_type == TILEDB_UINT8) { + advance_cell_slab_ = advance_cell_slab_row_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_row_row_s : + calculate_cell_slab_info_row_col_s; + } else if(coords_type == TILEDB_INT16) { + advance_cell_slab_ = advance_cell_slab_row_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_row_row_s : + calculate_cell_slab_info_row_col_s; + } else if(coords_type == TILEDB_UINT16) { + advance_cell_slab_ = advance_cell_slab_row_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_row_row_s : + calculate_cell_slab_info_row_col_s; + } else if(coords_type == TILEDB_UINT32) { + advance_cell_slab_ = advance_cell_slab_row_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_row_row_s : + calculate_cell_slab_info_row_col_s; + } else if(coords_type == TILEDB_UINT64) { + advance_cell_slab_ = advance_cell_slab_row_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_row_row_s : + calculate_cell_slab_info_row_col_s; } else { assert(0); } @@ -410,6 +461,42 @@ int ArraySortedReadState::init() { (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_col_row_s : calculate_cell_slab_info_col_col_s; + } else if(coords_type == TILEDB_INT8) { + advance_cell_slab_ = advance_cell_slab_col_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_col_row_s : + calculate_cell_slab_info_col_col_s; + } else if(coords_type == TILEDB_UINT8) { + advance_cell_slab_ = advance_cell_slab_col_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_col_row_s : + calculate_cell_slab_info_col_col_s; + } else if(coords_type == TILEDB_INT16) { + advance_cell_slab_ = advance_cell_slab_col_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_col_row_s : + calculate_cell_slab_info_col_col_s; + } else if(coords_type == TILEDB_UINT16) { + advance_cell_slab_ = advance_cell_slab_col_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_col_row_s : + calculate_cell_slab_info_col_col_s; + } else if(coords_type == TILEDB_UINT32) { + advance_cell_slab_ = advance_cell_slab_col_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_col_row_s : + calculate_cell_slab_info_col_col_s; + } else if(coords_type == TILEDB_UINT64) { + advance_cell_slab_ = advance_cell_slab_col_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_col_row_s : + calculate_cell_slab_info_col_col_s; } else { assert(0); } @@ -423,6 +510,18 @@ int ArraySortedReadState::init() { calculate_tile_slab_info_ = calculate_tile_slab_info_row; else if(coords_type == TILEDB_FLOAT64) calculate_tile_slab_info_ = calculate_tile_slab_info_row; + else if(coords_type == TILEDB_INT8) + calculate_tile_slab_info_ = calculate_tile_slab_info_row; + else if(coords_type == TILEDB_UINT8) + calculate_tile_slab_info_ = calculate_tile_slab_info_row; + else if(coords_type == TILEDB_INT16) + calculate_tile_slab_info_ = calculate_tile_slab_info_row; + else if(coords_type == TILEDB_UINT16) + calculate_tile_slab_info_ = calculate_tile_slab_info_row; + else if(coords_type == TILEDB_UINT32) + calculate_tile_slab_info_ = calculate_tile_slab_info_row; + else if(coords_type == TILEDB_UINT64) + calculate_tile_slab_info_ = calculate_tile_slab_info_row; else assert(0); } else { // tile_order == TILEDB_COL_MAJOR @@ -434,6 +533,18 @@ int ArraySortedReadState::init() { calculate_tile_slab_info_ = calculate_tile_slab_info_col; else if(coords_type == TILEDB_FLOAT64) calculate_tile_slab_info_ = calculate_tile_slab_info_col; + else if(coords_type == TILEDB_INT8) + calculate_tile_slab_info_ = calculate_tile_slab_info_col; + else if(coords_type == TILEDB_UINT8) + calculate_tile_slab_info_ = calculate_tile_slab_info_col; + else if(coords_type == TILEDB_INT16) + calculate_tile_slab_info_ = calculate_tile_slab_info_col; + else if(coords_type == TILEDB_UINT16) + calculate_tile_slab_info_ = calculate_tile_slab_info_col; + else if(coords_type == TILEDB_UINT32) + calculate_tile_slab_info_ = calculate_tile_slab_info_col; + else if(coords_type == TILEDB_UINT64) + calculate_tile_slab_info_ = calculate_tile_slab_info_col; else assert(0); } @@ -988,9 +1099,9 @@ void ArraySortedReadState::calculate_tile_slab_info_col(int id) { for(int i=0; ihandle_copy_requests_dense(); else if(coords_type == TILEDB_FLOAT64) asrs->handle_copy_requests_dense(); + else if(coords_type == TILEDB_INT8) + asrs->handle_copy_requests_dense(); + else if(coords_type == TILEDB_UINT8) + asrs->handle_copy_requests_dense(); + else if(coords_type == TILEDB_INT16) + asrs->handle_copy_requests_dense(); + else if(coords_type == TILEDB_UINT16) + asrs->handle_copy_requests_dense(); + else if(coords_type == TILEDB_UINT32) + asrs->handle_copy_requests_dense(); + else if(coords_type == TILEDB_UINT64) + asrs->handle_copy_requests_dense(); else assert(0); } else { // SPARSE @@ -1123,6 +1246,18 @@ void *ArraySortedReadState::copy_handler(void* context) { asrs->handle_copy_requests_sparse(); else if(coords_type == TILEDB_FLOAT64) asrs->handle_copy_requests_sparse(); + else if(coords_type == TILEDB_INT8) + asrs->handle_copy_requests_sparse(); + else if(coords_type == TILEDB_UINT8) + asrs->handle_copy_requests_sparse(); + else if(coords_type == TILEDB_INT16) + asrs->handle_copy_requests_sparse(); + else if(coords_type == TILEDB_UINT16) + asrs->handle_copy_requests_sparse(); + else if(coords_type == TILEDB_UINT32) + asrs->handle_copy_requests_sparse(); + else if(coords_type == TILEDB_UINT64) + asrs->handle_copy_requests_sparse(); else assert(0); } @@ -1815,7 +1950,7 @@ bool ArraySortedReadState::next_tile_slab_dense_col() { (upper - domain[2*(dim_num_-1)]) / tile_extents[dim_num_-1] * tile_extents[dim_num_-1] + domain[2*(dim_num_-1)]; tile_slab[aio_id_][2*(dim_num_-1)+1] = - std::min(cropped_upper - 1, subarray[2*(dim_num_-1)+1]); + MIN(cropped_upper - 1, subarray[2*(dim_num_-1)+1]); // Leave the rest of the subarray extents intact for(int i=0; i() { floor((upper - domain[2*(dim_num_-1)]) / tile_extents[dim_num_-1]) * tile_extents[dim_num_-1] + domain[2*(dim_num_-1)]; tile_slab[aio_id_][2*(dim_num_-1)+1] = - std::min(cropped_upper - FLT_MIN, subarray[2*(dim_num_-1)+1]); + MIN(cropped_upper - FLT_MIN, subarray[2*(dim_num_-1)+1]); // Leave the rest of the subarray extents intact for(int i=0; i() { tile_slab[aio_id_][2*(dim_num_-1)] = tile_slab[aio_id_][2*(dim_num_-1)+1] + FLT_MIN; tile_slab[aio_id_][2*(dim_num_-1)+1] = - std::min( + MIN( tile_slab[aio_id_][2*(dim_num_-1)] + tile_extents[dim_num_-1] - FLT_MIN, subarray[2*(dim_num_-1)+1]); @@ -2134,7 +2269,7 @@ bool ArraySortedReadState::next_tile_slab_sparse_col() { floor((upper - domain[2*(dim_num_-1)]) / tile_extents[dim_num_-1]) * tile_extents[dim_num_-1] + domain[2*(dim_num_-1)]; tile_slab[aio_id_][2*(dim_num_-1)+1] = - std::min(cropped_upper - DBL_MIN, subarray[2*(dim_num_-1)+1]); + MIN(cropped_upper - DBL_MIN, subarray[2*(dim_num_-1)+1]); // Leave the rest of the subarray extents intact for(int i=0; i() { tile_slab[aio_id_][2*(dim_num_-1)] = tile_slab[aio_id_][2*(dim_num_-1)+1] + DBL_MIN; tile_slab[aio_id_][2*(dim_num_-1)+1] = - std::min( + MIN( tile_slab[aio_id_][2*(dim_num_-1)] + tile_extents[dim_num_-1] - DBL_MIN, subarray[2*(dim_num_-1)+1]); @@ -2207,7 +2342,7 @@ bool ArraySortedReadState::next_tile_slab_sparse_row() { T upper = subarray[0] + tile_extents[0]; T cropped_upper = (upper - domain[0]) / tile_extents[0] * tile_extents[0] + domain[0]; - tile_slab[aio_id_][1] = std::min(cropped_upper - 1, subarray[1]); + tile_slab[aio_id_][1] = MIN(cropped_upper - 1, subarray[1]); // Leave the rest of the subarray extents intact for(int i=1; i() { float cropped_upper = floor((upper - domain[0]) / tile_extents[0]) * tile_extents[0] + domain[0]; - tile_slab[aio_id_][1] = std::min(cropped_upper - FLT_MIN, subarray[1]); + tile_slab[aio_id_][1] = MIN(cropped_upper - FLT_MIN, subarray[1]); // Leave the rest of the subarray extents intact for(int i=1; i() { // Advance tile slab tile_slab[aio_id_][0] = tile_slab[aio_id_][1] + FLT_MIN; tile_slab[aio_id_][1] = - std::min( + MIN( tile_slab[aio_id_][0] + tile_extents[0] - FLT_MIN, subarray[1]); } @@ -2350,7 +2485,7 @@ bool ArraySortedReadState::next_tile_slab_sparse_row() { double cropped_upper = floor((upper - domain[0]) / tile_extents[0]) * tile_extents[0] + domain[0]; - tile_slab[aio_id_][1] = std::min(cropped_upper - DBL_MIN, subarray[1]); + tile_slab[aio_id_][1] = MIN(cropped_upper - DBL_MIN, subarray[1]); // Leave the rest of the subarray extents intact for(int i=1; i() { // Advance tile slab tile_slab[aio_id_][0] = tile_slab[aio_id_][1] + DBL_MIN; tile_slab[aio_id_][1] = - std::min( + MIN( tile_slab[aio_id_][0] + tile_extents[0] - DBL_MIN, subarray[1]); } @@ -2909,8 +3044,20 @@ template int ArraySortedReadState::read_dense_sorted_col(); template int ArraySortedReadState::read_dense_sorted_col(); template int ArraySortedReadState::read_dense_sorted_col(); template int ArraySortedReadState::read_dense_sorted_col(); +template int ArraySortedReadState::read_dense_sorted_col(); +template int ArraySortedReadState::read_dense_sorted_col(); +template int ArraySortedReadState::read_dense_sorted_col(); +template int ArraySortedReadState::read_dense_sorted_col(); +template int ArraySortedReadState::read_dense_sorted_col(); +template int ArraySortedReadState::read_dense_sorted_col(); template int ArraySortedReadState::read_dense_sorted_row(); template int ArraySortedReadState::read_dense_sorted_row(); template int ArraySortedReadState::read_dense_sorted_row(); template int ArraySortedReadState::read_dense_sorted_row(); +template int ArraySortedReadState::read_dense_sorted_row(); +template int ArraySortedReadState::read_dense_sorted_row(); +template int ArraySortedReadState::read_dense_sorted_row(); +template int ArraySortedReadState::read_dense_sorted_row(); +template int ArraySortedReadState::read_dense_sorted_row(); +template int ArraySortedReadState::read_dense_sorted_row(); diff --git a/core/src/array/array_sorted_write_state.cc b/core/src/array/array_sorted_write_state.cc index c394191ca4a..ebfe47d73f2 100644 --- a/core/src/array/array_sorted_write_state.cc +++ b/core/src/array/array_sorted_write_state.cc @@ -48,7 +48,8 @@ # define PRINT_ERROR(x) do { } while(0) #endif - +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define MAX(a,b) ((a) > (b) ? (a) : (b)) /* ****************************** */ @@ -225,6 +226,42 @@ int ArraySortedWriteState::init() { (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_row_row_s : calculate_cell_slab_info_row_col_s; + } else if(coords_type == TILEDB_INT8) { + advance_cell_slab_ = advance_cell_slab_row_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_row_row_s : + calculate_cell_slab_info_row_col_s; + } else if(coords_type == TILEDB_UINT8) { + advance_cell_slab_ = advance_cell_slab_row_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_row_row_s : + calculate_cell_slab_info_row_col_s; + } else if(coords_type == TILEDB_INT16) { + advance_cell_slab_ = advance_cell_slab_row_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_row_row_s : + calculate_cell_slab_info_row_col_s; + } else if(coords_type == TILEDB_UINT16) { + advance_cell_slab_ = advance_cell_slab_row_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_row_row_s : + calculate_cell_slab_info_row_col_s; + } else if(coords_type == TILEDB_UINT32) { + advance_cell_slab_ = advance_cell_slab_row_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_row_row_s : + calculate_cell_slab_info_row_col_s; + } else if(coords_type == TILEDB_UINT64) { + advance_cell_slab_ = advance_cell_slab_row_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_row_row_s : + calculate_cell_slab_info_row_col_s; } else { assert(0); } @@ -241,6 +278,42 @@ int ArraySortedWriteState::init() { (cell_order == TILEDB_ROW_MAJOR) ? calculate_cell_slab_info_col_row_s : calculate_cell_slab_info_col_col_s; + } else if(coords_type == TILEDB_INT8) { + advance_cell_slab_ = advance_cell_slab_col_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_col_row_s : + calculate_cell_slab_info_col_col_s; + } else if(coords_type == TILEDB_UINT8) { + advance_cell_slab_ = advance_cell_slab_col_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_col_row_s : + calculate_cell_slab_info_col_col_s; + } else if(coords_type == TILEDB_INT16) { + advance_cell_slab_ = advance_cell_slab_col_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_col_row_s : + calculate_cell_slab_info_col_col_s; + } else if(coords_type == TILEDB_UINT16) { + advance_cell_slab_ = advance_cell_slab_col_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_col_row_s : + calculate_cell_slab_info_col_col_s; + } else if(coords_type == TILEDB_UINT32) { + advance_cell_slab_ = advance_cell_slab_col_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_col_row_s : + calculate_cell_slab_info_col_col_s; + } else if(coords_type == TILEDB_UINT64) { + advance_cell_slab_ = advance_cell_slab_col_s; + calculate_cell_slab_info_ = + (cell_order == TILEDB_ROW_MAJOR) ? + calculate_cell_slab_info_col_row_s : + calculate_cell_slab_info_col_col_s; } else { assert(0); } @@ -250,6 +323,18 @@ int ArraySortedWriteState::init() { calculate_tile_slab_info_ = calculate_tile_slab_info_row; else if(coords_type == TILEDB_INT64) calculate_tile_slab_info_ = calculate_tile_slab_info_row; + else if(coords_type == TILEDB_INT8) + calculate_tile_slab_info_ = calculate_tile_slab_info_row; + else if(coords_type == TILEDB_UINT8) + calculate_tile_slab_info_ = calculate_tile_slab_info_row; + else if(coords_type == TILEDB_INT16) + calculate_tile_slab_info_ = calculate_tile_slab_info_row; + else if(coords_type == TILEDB_UINT16) + calculate_tile_slab_info_ = calculate_tile_slab_info_row; + else if(coords_type == TILEDB_UINT32) + calculate_tile_slab_info_ = calculate_tile_slab_info_row; + else if(coords_type == TILEDB_UINT64) + calculate_tile_slab_info_ = calculate_tile_slab_info_row; else assert(0); } else { // tile_order == TILEDB_COL_MAJOR @@ -257,6 +342,18 @@ int ArraySortedWriteState::init() { calculate_tile_slab_info_ = calculate_tile_slab_info_col; else if(coords_type == TILEDB_INT64) calculate_tile_slab_info_ = calculate_tile_slab_info_col; + else if(coords_type == TILEDB_INT8) + calculate_tile_slab_info_ = calculate_tile_slab_info_col; + else if(coords_type == TILEDB_UINT8) + calculate_tile_slab_info_ = calculate_tile_slab_info_col; + else if(coords_type == TILEDB_INT16) + calculate_tile_slab_info_ = calculate_tile_slab_info_col; + else if(coords_type == TILEDB_UINT16) + calculate_tile_slab_info_ = calculate_tile_slab_info_col; + else if(coords_type == TILEDB_UINT32) + calculate_tile_slab_info_ = calculate_tile_slab_info_col; + else if(coords_type == TILEDB_UINT64) + calculate_tile_slab_info_ = calculate_tile_slab_info_col; else assert(0); } @@ -297,6 +394,18 @@ int ArraySortedWriteState::write( return write(); } else if(type == TILEDB_INT64) { return write(); + } else if(type == TILEDB_INT8) { + return write(); + } else if(type == TILEDB_UINT8) { + return write(); + } else if(type == TILEDB_INT16) { + return write(); + } else if(type == TILEDB_UINT16) { + return write(); + } else if(type == TILEDB_UINT32) { + return write(); + } else if(type == TILEDB_UINT64) { + return write(); } else { assert(0); return TILEDB_ASWS_ERR; @@ -637,9 +746,9 @@ void ArraySortedWriteState::calculate_tile_slab_info_col(int id) { for(int i=0; ihandle_aio_requests(); else if(coords_type == TILEDB_INT64) asws->handle_aio_requests(); + else if(coords_type == TILEDB_INT8) + asws->handle_aio_requests(); + else if(coords_type == TILEDB_UINT8) + asws->handle_aio_requests(); + else if(coords_type == TILEDB_INT16) + asws->handle_aio_requests(); + else if(coords_type == TILEDB_UINT16) + asws->handle_aio_requests(); + else if(coords_type == TILEDB_UINT32) + asws->handle_aio_requests(); + else if(coords_type == TILEDB_UINT64) + asws->handle_aio_requests(); else assert(0); @@ -782,6 +903,20 @@ void ArraySortedWriteState::copy_tile_slab() { copy_tile_slab(i, b); else if(type == TILEDB_CHAR) copy_tile_slab(i, b); + else if(type == TILEDB_INT8) + copy_tile_slab(i, b); + else if(type == TILEDB_INT8) + copy_tile_slab(i, b); + else if(type == TILEDB_UINT8) + copy_tile_slab(i, b); + else if(type == TILEDB_INT16) + copy_tile_slab(i, b); + else if(type == TILEDB_UINT16) + copy_tile_slab(i, b); + else if(type == TILEDB_UINT32) + copy_tile_slab(i, b); + else if(type == TILEDB_UINT64) + copy_tile_slab(i, b); ++b; } else { if(type == TILEDB_INT32) @@ -794,6 +929,18 @@ void ArraySortedWriteState::copy_tile_slab() { copy_tile_slab_var(i, b); else if(type == TILEDB_CHAR) copy_tile_slab_var(i, b); + else if(type == TILEDB_INT8) + copy_tile_slab_var(i, b); + else if(type == TILEDB_UINT8) + copy_tile_slab_var(i, b); + else if(type == TILEDB_INT16) + copy_tile_slab_var(i, b); + else if(type == TILEDB_UINT16) + copy_tile_slab_var(i, b); + else if(type == TILEDB_UINT32) + copy_tile_slab_var(i, b); + else if(type == TILEDB_UINT64) + copy_tile_slab_var(i, b); b += 2; } } @@ -1072,6 +1219,84 @@ void ArraySortedWriteState::fill_with_empty(int bid) { local_buffer[i] = empty; } +template<> +void ArraySortedWriteState::fill_with_empty(int bid) { + // For easy reference + int8_t* local_buffer = (int8_t*) copy_state_.buffers_[copy_id_][bid]; + size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; + int8_t empty = TILEDB_EMPTY_INT8; + + // Fill with empty values + size_t offset = 0; + for(int64_t i=0; offset < local_buffer_size; offset += sizeof(int8_t), ++i) + local_buffer[i] = empty; +} + +template<> +void ArraySortedWriteState::fill_with_empty(int bid) { + // For easy reference + uint8_t* local_buffer = (uint8_t*) copy_state_.buffers_[copy_id_][bid]; + size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; + uint8_t empty = TILEDB_EMPTY_UINT8; + + // Fill with empty values + size_t offset = 0; + for(int64_t i=0; offset < local_buffer_size; offset += sizeof(uint8_t), ++i) + local_buffer[i] = empty; +} + +template<> +void ArraySortedWriteState::fill_with_empty(int bid) { + // For easy reference + int16_t* local_buffer = (int16_t*) copy_state_.buffers_[copy_id_][bid]; + size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; + int16_t empty = TILEDB_EMPTY_INT16; + + // Fill with empty values + size_t offset = 0; + for(int64_t i=0; offset < local_buffer_size; offset += sizeof(int16_t), ++i) + local_buffer[i] = empty; +} + +template<> +void ArraySortedWriteState::fill_with_empty(int bid) { + // For easy reference + uint16_t* local_buffer = (uint16_t*) copy_state_.buffers_[copy_id_][bid]; + size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; + uint16_t empty = TILEDB_EMPTY_UINT16; + + // Fill with empty values + size_t offset = 0; + for(int64_t i=0; offset < local_buffer_size; offset += sizeof(uint16_t), ++i) + local_buffer[i] = empty; +} + +template<> +void ArraySortedWriteState::fill_with_empty(int bid) { + // For easy reference + uint32_t* local_buffer = (uint32_t*) copy_state_.buffers_[copy_id_][bid]; + size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; + uint32_t empty = TILEDB_EMPTY_UINT32; + + // Fill with empty values + size_t offset = 0; + for(int64_t i=0; offset < local_buffer_size; offset += sizeof(uint32_t), ++i) + local_buffer[i] = empty; +} + +template<> +void ArraySortedWriteState::fill_with_empty(int bid) { + // For easy reference + uint64_t* local_buffer = (uint64_t*) copy_state_.buffers_[copy_id_][bid]; + size_t local_buffer_size = copy_state_.buffer_sizes_[copy_id_][bid]; + uint64_t empty = TILEDB_EMPTY_UINT64; + + // Fill with empty values + size_t offset = 0; + for(int64_t i=0; offset < local_buffer_size; offset += sizeof(uint64_t), ++i) + local_buffer[i] = empty; +} + template<> void ArraySortedWriteState::fill_with_empty_var(int bid) { // For easy reference @@ -1127,6 +1352,72 @@ void ArraySortedWriteState::fill_with_empty_var(int bid) { memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(char)); } +template<> +void ArraySortedWriteState::fill_with_empty_var(int bid) { + // For easy reference + char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; + size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; + int8_t empty = TILEDB_EMPTY_INT8; + + // Fill an empty value + memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(int8_t)); +} + +template<> +void ArraySortedWriteState::fill_with_empty_var(int bid) { + // For easy reference + char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; + size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; + uint8_t empty = TILEDB_EMPTY_UINT8; + + // Fill an empty value + memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(uint8_t)); +} + +template<> +void ArraySortedWriteState::fill_with_empty_var(int bid) { + // For easy reference + char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; + size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; + int16_t empty = TILEDB_EMPTY_INT16; + + // Fill an empty value + memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(int16_t)); +} + +template<> +void ArraySortedWriteState::fill_with_empty_var(int bid) { + // For easy reference + char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; + size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; + uint16_t empty = TILEDB_EMPTY_UINT16; + + // Fill an empty value + memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(uint16_t)); +} + +template<> +void ArraySortedWriteState::fill_with_empty_var(int bid) { + // For easy reference + char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; + size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; + uint32_t empty = TILEDB_EMPTY_UINT32; + + // Fill an empty value + memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(uint32_t)); +} + +template<> +void ArraySortedWriteState::fill_with_empty_var(int bid) { + // For easy reference + char* local_buffer_var = (char*) copy_state_.buffers_[copy_id_][bid+1]; + size_t local_buffer_offset_var = copy_state_.buffer_offsets_[copy_id_][bid+1]; + uint64_t empty = TILEDB_EMPTY_UINT64; + + // Fill an empty value + memcpy(local_buffer_var + local_buffer_offset_var, &empty, sizeof(uint64_t)); +} + void ArraySortedWriteState::free_copy_state() { for(int i=0; i<2; ++i) { if(copy_state_.buffer_sizes_[i] != NULL) @@ -1424,7 +1715,7 @@ bool ArraySortedWriteState::next_tile_slab_col() { (upper - domain[2*(dim_num_-1)]) / tile_extents[dim_num_-1] * tile_extents[dim_num_-1] + domain[2*(dim_num_-1)]; tile_slab[copy_id_][2*(dim_num_-1)+1] = - std::min(cropped_upper - 1, subarray[2*(dim_num_-1)+1]); + MIN(cropped_upper - 1, subarray[2*(dim_num_-1)+1]); // Leave the rest of the subarray extents intact for(int i=0; i(aid); else if(coords_type == TILEDB_FLOAT64) update_current_tile_and_offset(aid); + else if(coords_type == TILEDB_INT8) + update_current_tile_and_offset(aid); + else if(coords_type == TILEDB_UINT8) + update_current_tile_and_offset(aid); + else if(coords_type == TILEDB_INT16) + update_current_tile_and_offset(aid); + else if(coords_type == TILEDB_UINT16) + update_current_tile_and_offset(aid); + else if(coords_type == TILEDB_UINT32) + update_current_tile_and_offset(aid); + else if(coords_type == TILEDB_UINT64) + update_current_tile_and_offset(aid); else assert(0); } @@ -1842,7 +2145,19 @@ int ArraySortedWriteState::wait_copy(int id) { template int ArraySortedWriteState::write_sorted_col(); template int ArraySortedWriteState::write_sorted_col(); +template int ArraySortedWriteState::write_sorted_col(); +template int ArraySortedWriteState::write_sorted_col(); +template int ArraySortedWriteState::write_sorted_col(); +template int ArraySortedWriteState::write_sorted_col(); +template int ArraySortedWriteState::write_sorted_col(); +template int ArraySortedWriteState::write_sorted_col(); template int ArraySortedWriteState::write_sorted_row(); template int ArraySortedWriteState::write_sorted_row(); +template int ArraySortedWriteState::write_sorted_row(); +template int ArraySortedWriteState::write_sorted_row(); +template int ArraySortedWriteState::write_sorted_row(); +template int ArraySortedWriteState::write_sorted_row(); +template int ArraySortedWriteState::write_sorted_row(); +template int ArraySortedWriteState::write_sorted_row(); diff --git a/core/src/c_api/tiledb.cc b/core/src/c_api/tiledb.cc index 0e08e985cd7..0ca47cdc45e 100644 --- a/core/src/c_api/tiledb.cc +++ b/core/src/c_api/tiledb.cc @@ -50,8 +50,6 @@ #endif - - /* ****************************** */ /* GLOBAL VARIABLES */ /* ****************************** */ @@ -1380,6 +1378,13 @@ int tiledb_metadata_iterator_finalize( /* DIRECTORY MANAGEMENT */ /* ****************************** */ +int tiledb_dir_type( + const TileDB_CTX* tiledb_ctx, + const char* dir) { + // Return the directory type + return tiledb_ctx->storage_manager_->dir_type(dir); +} + int tiledb_clear( const TileDB_CTX* tiledb_ctx, const char* dir) { diff --git a/core/src/fragment/read_state.cc b/core/src/fragment/read_state.cc index 02af3289dc3..615e8e0611a 100644 --- a/core/src/fragment/read_state.cc +++ b/core/src/fragment/read_state.cc @@ -33,6 +33,7 @@ #include "utils.h" #include "read_state.h" #include +#include #include #include #include @@ -1238,6 +1239,18 @@ void ReadState::compute_tile_search_range() { compute_tile_search_range(); } else if(coords_type == TILEDB_FLOAT64) { compute_tile_search_range(); + } else if(coords_type == TILEDB_INT8) { + compute_tile_search_range(); + } else if(coords_type == TILEDB_UINT8) { + compute_tile_search_range(); + } else if(coords_type == TILEDB_INT16) { + compute_tile_search_range(); + } else if(coords_type == TILEDB_UINT16) { + compute_tile_search_range(); + } else if(coords_type == TILEDB_UINT32) { + compute_tile_search_range(); + } else if(coords_type == TILEDB_UINT64) { + compute_tile_search_range(); } else { // The code should never reach here assert(0); @@ -1246,14 +1259,8 @@ void ReadState::compute_tile_search_range() { template void ReadState::compute_tile_search_range() { - // For easy reference - int cell_order = array_schema_->cell_order(); - // Initialize the tile search range - if(cell_order == TILEDB_HILBERT) // HILBERT CELL ORDER - compute_tile_search_range_hil(); - else // COLUMN- OR ROW-MAJOR - compute_tile_search_range_col_or_row(); + compute_tile_search_range_col_or_row(); // Handle no overlap if(tile_search_range_[0] == -1 || @@ -1439,26 +1446,34 @@ int ReadState::decompress_tile( // For easy reference int compression = array_schema_->compression(attribute_id); + // Special case for search coordinate tiles + if(attribute_id == attribute_num_ + 1) + attribute_id = attribute_num_; + if(compression == TILEDB_GZIP) return decompress_tile_gzip( + attribute_id, tile_compressed, tile_compressed_size, tile, tile_size); else if(compression == TILEDB_ZSTD) return decompress_tile_zstd( + attribute_id, tile_compressed, tile_compressed_size, tile, tile_size); else if(compression == TILEDB_LZ4) return decompress_tile_lz4( + attribute_id, tile_compressed, tile_compressed_size, tile, tile_size); else if(compression == TILEDB_BLOSC) return decompress_tile_blosc( + attribute_id, tile_compressed, tile_compressed_size, tile, @@ -1466,6 +1481,7 @@ int ReadState::decompress_tile( "blosclz"); else if(compression == TILEDB_BLOSC_LZ4) return decompress_tile_blosc( + attribute_id, tile_compressed, tile_compressed_size, tile, @@ -1473,6 +1489,7 @@ int ReadState::decompress_tile( "lz4"); else if(compression == TILEDB_BLOSC_LZ4HC) return decompress_tile_blosc( + attribute_id, tile_compressed, tile_compressed_size, tile, @@ -1480,6 +1497,7 @@ int ReadState::decompress_tile( "lz4hc"); else if(compression == TILEDB_BLOSC_SNAPPY) return decompress_tile_blosc( + attribute_id, tile_compressed, tile_compressed_size, tile, @@ -1487,6 +1505,7 @@ int ReadState::decompress_tile( "snappy"); else if(compression == TILEDB_BLOSC_ZLIB) return decompress_tile_blosc( + attribute_id, tile_compressed, tile_compressed_size, tile, @@ -1494,6 +1513,7 @@ int ReadState::decompress_tile( "zlib"); else if(compression == TILEDB_BLOSC_ZSTD) return decompress_tile_blosc( + attribute_id, tile_compressed, tile_compressed_size, tile, @@ -1506,6 +1526,13 @@ int ReadState::decompress_tile( tile_compressed_size, tile, tile_size); + else if(compression == TILEDB_BZIP2) + return decompress_tile_bzip2( + attribute_id, + tile_compressed, + tile_compressed_size, + tile, + tile_size); // Error assert(0); @@ -1513,10 +1540,18 @@ int ReadState::decompress_tile( } int ReadState::decompress_tile_gzip( + int attribute_id, unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) { + // For easy reference + const ArraySchema* array_schema = fragment_->array()->array_schema(); + int dim_num = array_schema->dim_num(); + int attribute_num = array_schema->attribute_num(); + bool coords = (attribute_id == attribute_num); + size_t coords_size = array_schema->coords_size(); + // Decompress tile size_t gunzip_out_size; if(gunzip( @@ -1529,15 +1564,27 @@ int ReadState::decompress_tile_gzip( return TILEDB_RS_ERR; } + // Zip coordinates + if(coords) + zip_coordinates(tile, tile_size, dim_num, coords_size); + // Success return TILEDB_RS_OK; } int ReadState::decompress_tile_zstd( + int attribute_id, unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) { + // For easy reference + const ArraySchema* array_schema = fragment_->array()->array_schema(); + int dim_num = array_schema->dim_num(); + int attribute_num = array_schema->attribute_num(); + bool coords = (attribute_id == attribute_num); + size_t coords_size = array_schema->coords_size(); + // Decompress tile size_t zstd_size = ZSTD_decompress( @@ -1552,15 +1599,27 @@ int ReadState::decompress_tile_zstd( return TILEDB_RS_ERR; } + // Zip coordinates + if(coords) + zip_coordinates(tile, tile_size, dim_num, coords_size); + // Success return TILEDB_RS_OK; } int ReadState::decompress_tile_lz4( + int attribute_id, unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size) { + // For easy reference + const ArraySchema* array_schema = fragment_->array()->array_schema(); + int dim_num = array_schema->dim_num(); + int attribute_num = array_schema->attribute_num(); + bool coords = (attribute_id == attribute_num); + size_t coords_size = array_schema->coords_size(); + // Decompress tile if(LZ4_decompress_safe( (const char*) tile_compressed, @@ -1573,16 +1632,28 @@ int ReadState::decompress_tile_lz4( return TILEDB_RS_ERR; } + // Zip coordinates + if(coords) + zip_coordinates(tile, tile_size, dim_num, coords_size); + // Success return TILEDB_RS_OK; } int ReadState::decompress_tile_blosc( + int attribute_id, unsigned char* tile_compressed, size_t tile_compressed_size, unsigned char* tile, size_t tile_size, const char* compressor) { + // For easy reference + const ArraySchema* array_schema = fragment_->array()->array_schema(); + int dim_num = array_schema->dim_num(); + int attribute_num = array_schema->attribute_num(); + bool coords = (attribute_id == attribute_num); + size_t coords_size = array_schema->coords_size(); + // Initialization blosc_init(); @@ -1601,6 +1672,10 @@ int ReadState::decompress_tile_blosc( // Clean up blosc_destroy(); + // Zip coordinates + if(coords) + zip_coordinates(tile, tile_size, dim_num, coords_size); + // Success return TILEDB_RS_OK; } @@ -1611,10 +1686,6 @@ int ReadState::decompress_tile_rle( size_t tile_compressed_size, unsigned char* tile, size_t tile_size) { - // Special case for search coordinate tiles - if(attribute_id == attribute_num_ + 1) - attribute_id = attribute_num_; - // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); int dim_num = array_schema->dim_num(); @@ -1671,6 +1742,45 @@ int ReadState::decompress_tile_rle( return TILEDB_RS_OK; } +int ReadState::decompress_tile_bzip2( + int attribute_id, + unsigned char* tile_compressed, + size_t tile_compressed_size, + unsigned char* tile, + size_t tile_size) { + // For easy reference + const ArraySchema* array_schema = fragment_->array()->array_schema(); + int dim_num = array_schema->dim_num(); + int attribute_num = array_schema->attribute_num(); + bool coords = (attribute_id == attribute_num); + size_t coords_size = array_schema->coords_size(); + + // Decompress tile + unsigned int destLen = tile_size; + int rc = BZ2_bzBuffToBuffDecompress( + (char*) tile, + &destLen, + (char*) tile_compressed, + tile_compressed_size, + 0, + 0); + + // Check for error + if(rc != BZ_OK) { + std::string errmsg = "Failed decompressing with BZIP2"; + PRINT_ERROR(errmsg); + tiledb_rs_errmsg = TILEDB_RS_ERRMSG + errmsg; + return TILEDB_RS_ERR; + } + + // Zip coordinates + if(coords) + zip_coordinates(tile, tile_size, dim_num, coords_size); + + // Success + return TILEDB_RS_OK; +} + template int64_t ReadState::get_cell_pos_after(const T* coords) { // For easy reference @@ -3070,6 +3180,30 @@ template int ReadState::get_coords_after( const double* coords, double* coords_after, bool& coords_retrieved); +template int ReadState::get_coords_after( + const int8_t* coords, + int8_t* coords_after, + bool& coords_retrieved); +template int ReadState::get_coords_after( + const uint8_t* coords, + uint8_t* coords_after, + bool& coords_retrieved); +template int ReadState::get_coords_after( + const int16_t* coords, + int16_t* coords_after, + bool& coords_retrieved); +template int ReadState::get_coords_after( + const uint16_t* coords, + uint16_t* coords_after, + bool& coords_retrieved); +template int ReadState::get_coords_after( + const uint32_t* coords, + uint32_t* coords_after, + bool& coords_retrieved); +template int ReadState::get_coords_after( + const uint64_t* coords, + uint64_t* coords_after, + bool& coords_retrieved); template int ReadState::get_enclosing_coords( int tile_i, @@ -3111,6 +3245,66 @@ template int ReadState::get_enclosing_coords( bool& left_retrieved, bool& right_retrieved, bool& target_exists); +template int ReadState::get_enclosing_coords( + int tile_i, + const int8_t* target_coords, + const int8_t* start_coords, + const int8_t* end_coords, + int8_t* left_coords, + int8_t* right_coords, + bool& left_retrieved, + bool& right_retrieved, + bool& target_exists); +template int ReadState::get_enclosing_coords( + int tile_i, + const uint8_t* target_coords, + const uint8_t* start_coords, + const uint8_t* end_coords, + uint8_t* left_coords, + uint8_t* right_coords, + bool& left_retrieved, + bool& right_retrieved, + bool& target_exists); +template int ReadState::get_enclosing_coords( + int tile_i, + const int16_t* target_coords, + const int16_t* start_coords, + const int16_t* end_coords, + int16_t* left_coords, + int16_t* right_coords, + bool& left_retrieved, + bool& right_retrieved, + bool& target_exists); +template int ReadState::get_enclosing_coords( + int tile_i, + const uint16_t* target_coords, + const uint16_t* start_coords, + const uint16_t* end_coords, + uint16_t* left_coords, + uint16_t* right_coords, + bool& left_retrieved, + bool& right_retrieved, + bool& target_exists); +template int ReadState::get_enclosing_coords( + int tile_i, + const uint32_t* target_coords, + const uint32_t* start_coords, + const uint32_t* end_coords, + uint32_t* left_coords, + uint32_t* right_coords, + bool& left_retrieved, + bool& right_retrieved, + bool& target_exists); +template int ReadState::get_enclosing_coords( + int tile_i, + const uint64_t* target_coords, + const uint64_t* start_coords, + const uint64_t* end_coords, + uint64_t* left_coords, + uint64_t* right_coords, + bool& left_retrieved, + bool& right_retrieved, + bool& target_exists); template int ReadState::get_fragment_cell_pos_range_sparse( const FragmentInfo& fragment_info, @@ -3128,6 +3322,30 @@ template int ReadState::get_fragment_cell_pos_range_sparse( const FragmentInfo& fragment_info, const double* cell_range, FragmentCellPosRange& fragment_cell_pos_range); +template int ReadState::get_fragment_cell_pos_range_sparse( + const FragmentInfo& fragment_info, + const int8_t* cell_range, + FragmentCellPosRange& fragment_cell_pos_range); +template int ReadState::get_fragment_cell_pos_range_sparse( + const FragmentInfo& fragment_info, + const uint8_t* cell_range, + FragmentCellPosRange& fragment_cell_pos_range); +template int ReadState::get_fragment_cell_pos_range_sparse( + const FragmentInfo& fragment_info, + const int16_t* cell_range, + FragmentCellPosRange& fragment_cell_pos_range); +template int ReadState::get_fragment_cell_pos_range_sparse( + const FragmentInfo& fragment_info, + const uint16_t* cell_range, + FragmentCellPosRange& fragment_cell_pos_range); +template int ReadState::get_fragment_cell_pos_range_sparse( + const FragmentInfo& fragment_info, + const uint32_t* cell_range, + FragmentCellPosRange& fragment_cell_pos_range); +template int ReadState::get_fragment_cell_pos_range_sparse( + const FragmentInfo& fragment_info, + const uint64_t* cell_range, + FragmentCellPosRange& fragment_cell_pos_range); template int ReadState::get_fragment_cell_ranges_sparse( int fragment_i, @@ -3149,6 +3367,36 @@ template int ReadState::get_fragment_cell_ranges_sparse( const double* start_coords, const double* end_coords, FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_sparse( + int fragment_i, + const int8_t* start_coords, + const int8_t* end_coords, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_sparse( + int fragment_i, + const uint8_t* start_coords, + const uint8_t* end_coords, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_sparse( + int fragment_i, + const int16_t* start_coords, + const int16_t* end_coords, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_sparse( + int fragment_i, + const uint16_t* start_coords, + const uint16_t* end_coords, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_sparse( + int fragment_i, + const uint32_t* start_coords, + const uint32_t* end_coords, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_sparse( + int fragment_i, + const uint64_t* start_coords, + const uint64_t* end_coords, + FragmentCellRanges& fragment_cell_ranges); template int ReadState::get_fragment_cell_ranges_sparse( int fragment_i, @@ -3156,6 +3404,24 @@ template int ReadState::get_fragment_cell_ranges_sparse( template int ReadState::get_fragment_cell_ranges_sparse( int fragment_i, FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_sparse( + int fragment_i, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_sparse( + int fragment_i, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_sparse( + int fragment_i, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_sparse( + int fragment_i, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_sparse( + int fragment_i, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_sparse( + int fragment_i, + FragmentCellRanges& fragment_cell_ranges); template int ReadState::get_fragment_cell_ranges_dense( int fragment_i, @@ -3163,19 +3429,66 @@ template int ReadState::get_fragment_cell_ranges_dense( template int ReadState::get_fragment_cell_ranges_dense( int fragment_i, FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_dense( + int fragment_i, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_dense( + int fragment_i, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_dense( + int fragment_i, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_dense( + int fragment_i, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_dense( + int fragment_i, + FragmentCellRanges& fragment_cell_ranges); +template int ReadState::get_fragment_cell_ranges_dense( + int fragment_i, + FragmentCellRanges& fragment_cell_ranges); template void ReadState::get_next_overlapping_tile_dense( const int* tile_coords); template void ReadState::get_next_overlapping_tile_dense( const int64_t* tile_coords); +template void ReadState::get_next_overlapping_tile_dense( + const int8_t* tile_coords); +template void ReadState::get_next_overlapping_tile_dense( + const uint8_t* tile_coords); +template void ReadState::get_next_overlapping_tile_dense( + const int16_t* tile_coords); +template void ReadState::get_next_overlapping_tile_dense( + const uint16_t* tile_coords); +template void ReadState::get_next_overlapping_tile_dense( + const uint32_t* tile_coords); +template void ReadState::get_next_overlapping_tile_dense( + const uint64_t* tile_coords); template void ReadState::get_next_overlapping_tile_sparse( const int* tile_coords); template void ReadState::get_next_overlapping_tile_sparse( const int64_t* tile_coords); +template void ReadState::get_next_overlapping_tile_sparse( + const int8_t* tile_coords); +template void ReadState::get_next_overlapping_tile_sparse( + const uint8_t* tile_coords); +template void ReadState::get_next_overlapping_tile_sparse( + const int16_t* tile_coords); +template void ReadState::get_next_overlapping_tile_sparse( + const uint16_t* tile_coords); +template void ReadState::get_next_overlapping_tile_sparse( + const uint32_t* tile_coords); +template void ReadState::get_next_overlapping_tile_sparse( + const uint64_t* tile_coords); template void ReadState::get_next_overlapping_tile_sparse(); template void ReadState::get_next_overlapping_tile_sparse(); template void ReadState::get_next_overlapping_tile_sparse(); template void ReadState::get_next_overlapping_tile_sparse(); - +template void ReadState::get_next_overlapping_tile_sparse(); +template void ReadState::get_next_overlapping_tile_sparse(); +template void ReadState::get_next_overlapping_tile_sparse(); +template void ReadState::get_next_overlapping_tile_sparse(); +template void ReadState::get_next_overlapping_tile_sparse(); +template void ReadState::get_next_overlapping_tile_sparse(); diff --git a/core/src/fragment/write_state.cc b/core/src/fragment/write_state.cc index b2c7dea8289..9e181fbf92e 100644 --- a/core/src/fragment/write_state.cc +++ b/core/src/fragment/write_state.cc @@ -35,6 +35,7 @@ #include "utils.h" #include "write_state.h" #include +#include #include #include #include @@ -463,11 +464,23 @@ int WriteState::compress_tile( // Handle different compression if(compression == TILEDB_GZIP) - return compress_tile_gzip(tile, tile_size, tile_compressed_size); + return compress_tile_gzip( + attribute_id, + tile, + tile_size, + tile_compressed_size); else if(compression == TILEDB_ZSTD) - return compress_tile_zstd(tile, tile_size, tile_compressed_size); + return compress_tile_zstd( + attribute_id, + tile, + tile_size, + tile_compressed_size); else if(compression == TILEDB_LZ4) - return compress_tile_lz4(tile, tile_size, tile_compressed_size); + return compress_tile_lz4( + attribute_id, + tile, + tile_size, + tile_compressed_size); else if(compression == TILEDB_BLOSC) return compress_tile_blosc( attribute_id, @@ -516,6 +529,12 @@ int WriteState::compress_tile( tile, tile_size, tile_compressed_size); + else if(compression == TILEDB_BZIP2) + return compress_tile_bzip2( + attribute_id, + tile, + tile_size, + tile_compressed_size); // Error assert(0); @@ -523,9 +542,17 @@ int WriteState::compress_tile( } int WriteState::compress_tile_gzip( + int attribute_id, unsigned char* tile, size_t tile_size, size_t& tile_compressed_size) { + // For easy reference + const ArraySchema* array_schema = fragment_->array()->array_schema(); + int dim_num = array_schema->dim_num(); + int attribute_num = array_schema->attribute_num(); + bool coords = (attribute_id == attribute_num); + size_t coords_size = array_schema->coords_size(); + // Allocate space to store the compressed tile if(tile_compressed_ == NULL) { tile_compressed_allocated_size_ = @@ -546,6 +573,10 @@ int WriteState::compress_tile_gzip( unsigned char* tile_compressed = static_cast(tile_compressed_); + // Split dimensions + if(coords) + split_coordinates(tile, tile_size, dim_num, coords_size); + // Compress tile ssize_t gzip_size = gzip(tile, tile_size, tile_compressed, tile_compressed_allocated_size_); @@ -560,9 +591,17 @@ int WriteState::compress_tile_gzip( } int WriteState::compress_tile_zstd( + int attribute_id, unsigned char* tile, size_t tile_size, size_t& tile_compressed_size) { + // For easy reference + const ArraySchema* array_schema = fragment_->array()->array_schema(); + int dim_num = array_schema->dim_num(); + int attribute_num = array_schema->attribute_num(); + bool coords = (attribute_id == attribute_num); + size_t coords_size = array_schema->coords_size(); + // Allocate space to store the compressed tile size_t compress_bound = ZSTD_compressBound(tile_size); if(tile_compressed_ == NULL) { @@ -580,6 +619,10 @@ int WriteState::compress_tile_zstd( unsigned char* tile_compressed = static_cast(tile_compressed_); + // Split dimensions + if(coords) + split_coordinates(tile, tile_size, dim_num, coords_size); + // Compress tile size_t zstd_size = ZSTD_compress( @@ -601,9 +644,17 @@ int WriteState::compress_tile_zstd( } int WriteState::compress_tile_lz4( + int attribute_id, unsigned char* tile, size_t tile_size, size_t& tile_compressed_size) { + // For easy reference + const ArraySchema* array_schema = fragment_->array()->array_schema(); + int dim_num = array_schema->dim_num(); + int attribute_num = array_schema->attribute_num(); + bool coords = (attribute_id == attribute_num); + size_t coords_size = array_schema->coords_size(); + // Allocate space to store the compressed tile size_t compress_bound = LZ4_compressBound(tile_size); if(tile_compressed_ == NULL) { @@ -617,6 +668,10 @@ int WriteState::compress_tile_lz4( tile_compressed_ = realloc(tile_compressed_, compress_bound); } + // Split dimensions + if(coords) + split_coordinates(tile, tile_size, dim_num, coords_size); + // Compress tile int lz4_size = LZ4_compress( @@ -643,6 +698,10 @@ int WriteState::compress_tile_blosc( const char* compressor) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); + int dim_num = array_schema->dim_num(); + int attribute_num = array_schema->attribute_num(); + bool coords = (attribute_id == attribute_num); + size_t coords_size = array_schema->coords_size(); // Allocate space to store the compressed tile size_t compress_bound = tile_size + BLOSC_MAX_OVERHEAD; @@ -673,6 +732,10 @@ int WriteState::compress_tile_blosc( unsigned char* tile_compressed = static_cast(tile_compressed_); + // Split dimensions + if(coords) + split_coordinates(tile, tile_size, dim_num, coords_size); + // Compress tile int blosc_size = blosc_compress( @@ -781,6 +844,58 @@ int WriteState::compress_tile_rle( return TILEDB_WS_OK; } +int WriteState::compress_tile_bzip2( + int attribute_id, + unsigned char* tile, + size_t tile_size, + size_t& tile_compressed_size) { + // For easy reference + const ArraySchema* array_schema = fragment_->array()->array_schema(); + int dim_num = array_schema->dim_num(); + int attribute_num = array_schema->attribute_num(); + bool coords = (attribute_id == attribute_num); + size_t coords_size = array_schema->coords_size(); + + // Allocate space to store the compressed tile + if(tile_compressed_ == NULL) { + tile_compressed_allocated_size_ = tile_size; + tile_compressed_ = malloc(tile_size); + } + + // Split dimensions + if(coords) + split_coordinates(tile, tile_size, dim_num, coords_size); + + // Compress tile + unsigned int destLen = tile_compressed_allocated_size_; + int rc; + while((rc = BZ2_bzBuffToBuffCompress( + (char*) tile_compressed_, + &destLen, + (char*) tile, + tile_size, + 9, + 0, + 30)) == BZ_OUTBUFF_FULL) { + expand_buffer(tile_compressed_, tile_compressed_allocated_size_); + destLen = tile_compressed_allocated_size_; + } + + // Check for error + if(rc != BZ_OK) { + std::string errmsg = "Failed compressing with BZIP2"; + PRINT_ERROR(errmsg); + tiledb_ws_errmsg = TILEDB_WS_ERRMSG + errmsg; + return TILEDB_WS_ERR; + } + + // Set tile compressed size + tile_compressed_size = destLen; + + // Success + return TILEDB_WS_OK; +} + int WriteState::compress_and_write_tile(int attribute_id) { // For easy reference const ArraySchema* array_schema = fragment_->array()->array_schema(); @@ -967,6 +1082,18 @@ void WriteState::sort_cell_pos( sort_cell_pos(buffer, buffer_size, cell_pos); else if(coords_type == TILEDB_FLOAT64) sort_cell_pos(buffer, buffer_size, cell_pos); + else if(coords_type == TILEDB_INT8) + sort_cell_pos(buffer, buffer_size, cell_pos); + else if(coords_type == TILEDB_UINT8) + sort_cell_pos(buffer, buffer_size, cell_pos); + else if(coords_type == TILEDB_INT16) + sort_cell_pos(buffer, buffer_size, cell_pos); + else if(coords_type == TILEDB_UINT16) + sort_cell_pos(buffer, buffer_size, cell_pos); + else if(coords_type == TILEDB_UINT32) + sort_cell_pos(buffer, buffer_size, cell_pos); + else if(coords_type == TILEDB_UINT64) + sort_cell_pos(buffer, buffer_size, cell_pos); } template @@ -1001,18 +1128,6 @@ void WriteState::sort_cell_pos( cell_pos.begin(), cell_pos.end(), SmallerCol(buffer_T, dim_num)); - } else if(cell_order == TILEDB_HILBERT) { - // Get hilbert ids - std::vector ids; - ids.resize(buffer_cell_num); - for(int i=0; ihilbert_id(&buffer_T[i * dim_num]); - - // Sort cell positions - SORT( - cell_pos.begin(), - cell_pos.end(), - SmallerIdRow(buffer_T, dim_num, ids)); } else { assert(0); // The code should never reach here } @@ -1056,6 +1171,18 @@ void WriteState::update_book_keeping( update_book_keeping(buffer, buffer_size); else if(coords_type == TILEDB_FLOAT64) update_book_keeping(buffer, buffer_size); + else if(coords_type == TILEDB_INT8) + update_book_keeping(buffer, buffer_size); + else if(coords_type == TILEDB_UINT8) + update_book_keeping(buffer, buffer_size); + else if(coords_type == TILEDB_INT16) + update_book_keeping(buffer, buffer_size); + else if(coords_type == TILEDB_UINT16) + update_book_keeping(buffer, buffer_size); + else if(coords_type == TILEDB_UINT32) + update_book_keeping(buffer, buffer_size); + else if(coords_type == TILEDB_UINT64) + update_book_keeping(buffer, buffer_size); } template diff --git a/core/src/misc/hilbert_curve.cc b/core/src/misc/hilbert_curve.cc deleted file mode 100755 index 20fff287d20..00000000000 --- a/core/src/misc/hilbert_curve.cc +++ /dev/null @@ -1,162 +0,0 @@ -/** - * @file hilbert_curve.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2016 MIT and Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file implements the HilbertCurve class. - */ - -#include "hilbert_curve.h" -#include -#include - - - - -/* ****************************** */ -/* CONSTRUCTORS & DESTRUCTORS */ -/* ****************************** */ - -HilbertCurve::HilbertCurve(int bits, int dim_num) - : bits_(bits), dim_num_(dim_num) { - assert(dim_num >=0 && dim_num < HC_MAX_DIM); - assert(bits * dim_num <= int(sizeof(int64_t)*8)); -} - -HilbertCurve::~HilbertCurve() { -} - - - - -/* ****************************** */ -/* BASIC FUNCTIONS */ -/* ****************************** */ - -void HilbertCurve::coords_to_hilbert(const int* coords, int64_t& hilbert) { - // Copy coords to temporary storage - memcpy(temp_, coords, dim_num_ * sizeof(int)); - - // Convert coords to the transpose form of the hilbert value - AxestoTranspose(temp_, bits_, dim_num_); - - // Convert the hilbert transpose form into an int64_t hilbert value - hilbert = 0; - int64_t c = 1; // This is a bit shifted from right to left over temp_[i] - int64_t h = 1; // This is a bit shifted from right to left over hilbert - for(int j=0; j=0; --i, h <<= 1) { - if(temp_[i] & c) - hilbert |= h; - } - } -} - -void HilbertCurve::hilbert_to_coords(int64_t hilbert, int* coords) { - // Initialization - for(int i=0; i=0; --i, h <<= 1) { - if(hilbert & h) - temp_[i] |= c; - } - } - - // Convert coords to the transpose form of the hilbert value - TransposetoAxes(temp_, bits_, dim_num_); - - // Copy from the temporary storage to the (output) coords - memcpy(coords, temp_, dim_num_ * sizeof(int)); -} - - - - -/* ****************************** */ -/* PRIVATE METHODS */ -/* ****************************** */ - -void HilbertCurve::AxestoTranspose(int* X, int b, int n) { - int P, Q, t, i; - - // Inverse undo - for(Q = 1 << (b - 1); Q > 1; Q >>= 1) { - P = Q - 1; - if(X[0] & Q) // invert - X[0] ^= P; - for(i = 1; i < n; i++) - if(X[i] & Q) // invert - X[0] ^= P; - else { // exchange - t = (X[0] ^ X[i]) & P; - X[0] ^= t; - X[i] ^= t; - } - } - - // Gray encode (inverse of decode) - for(i = 1; i < n; i++) - X[i] ^= X[i-1]; - t = X[n-1]; - for(i = 1; i < b; i <<= 1) - X[n-1] ^= X[n-1] >> i; - t ^= X[n-1]; - for(i = n-2; i >= 0; i--) - X[i] ^= t; -} - -void HilbertCurve::TransposetoAxes(int* X, int b, int n) { - int M, P, Q, t, i; - - // Gray decode by H ^ (H/2) - t = X[n-1] >> 1; - for(i = n-1; i; i--) - X[i] ^= X[i-1]; - X[0] ^= t; - - // Undo excess work - M = 2 << (b - 1); - for(Q = 2; Q != M; Q <<= 1) { - P = Q - 1; - for(i = n-1; i; i--) - if(X[i] & Q) // invert - X[0] ^= P; - else { // exchange - t = (X[0] ^ X[i]) & P; - X[0] ^= t; - X[i] ^= t; - } - if(X[0] & Q) // invert - X[0] ^= P; - } -} - diff --git a/core/src/misc/utils.cc b/core/src/misc/utils.cc index e2df8348de5..6986bb5113d 100644 --- a/core/src/misc/utils.cc +++ b/core/src/misc/utils.cc @@ -349,6 +349,18 @@ bool empty_value(T value) { return value == T(TILEDB_EMPTY_FLOAT32); else if(&typeid(T) == &typeid(double)) return value == T(TILEDB_EMPTY_FLOAT64); + else if(&typeid(T) == &typeid(int8_t)) + return value == T(TILEDB_EMPTY_INT8); + else if(&typeid(T) == &typeid(uint8_t)) + return value == T(TILEDB_EMPTY_UINT8); + else if(&typeid(T) == &typeid(int16_t)) + return value == T(TILEDB_EMPTY_INT16); + else if(&typeid(T) == &typeid(uint16_t)) + return value == T(TILEDB_EMPTY_UINT16); + else if(&typeid(T) == &typeid(uint32_t)) + return value == T(TILEDB_EMPTY_UINT32); + else if(&typeid(T) == &typeid(uint64_t)) + return value == T(TILEDB_EMPTY_UINT64); else return false; } @@ -1809,6 +1821,35 @@ int RLE_decompress_coords_row( return TILEDB_UT_OK; } +void split_coordinates( + void* tile, + size_t tile_size, + int dim_num, + size_t coords_size) { + // For easy reference + size_t coord_size = coords_size / dim_num; + int64_t cell_num = tile_size / coords_size; + char* tile_c = (char*) tile; + size_t ptr = 0, ptr_tmp = 0; + + // Create a tile clone + char* tile_tmp = (char*) malloc(tile_size); + memcpy(tile_tmp, tile, tile_size); + + // Split coordinates + for(int j=0; j value.size()) return false; @@ -1971,6 +2012,36 @@ int write_to_file_cmp_gzip( return TILEDB_UT_OK; } +void zip_coordinates( + void* tile, + size_t tile_size, + int dim_num, + size_t coords_size) { + // For easy reference + size_t coord_size = coords_size / dim_num; + int64_t cell_num = tile_size / coords_size; + char* tile_c = (char*) tile; + size_t ptr = 0, ptr_tmp = 0; + + // Create a tile clone + char* tile_tmp = (char*) malloc(tile_size); + memcpy(tile_tmp, tile, tile_size); + + // Zip coordinates + for(int j=0; j( const int* subarray, @@ -1984,6 +2055,24 @@ template int64_t cell_num_in_subarray( template int64_t cell_num_in_subarray( const double* subarray, int dim_num); +template int64_t cell_num_in_subarray( + const int8_t* subarray, + int dim_num); +template int64_t cell_num_in_subarray( + const uint8_t* subarray, + int dim_num); +template int64_t cell_num_in_subarray( + const int16_t* subarray, + int dim_num); +template int64_t cell_num_in_subarray( + const uint16_t* subarray, + int dim_num); +template int64_t cell_num_in_subarray( + const uint32_t* subarray, + int dim_num); +template int64_t cell_num_in_subarray( + const uint64_t* subarray, + int dim_num); template bool cell_in_subarray( const int* cell, @@ -2001,6 +2090,30 @@ template bool cell_in_subarray( const double* cell, const double* subarray, int dim_num); +template bool cell_in_subarray( + const int8_t* cell, + const int8_t* subarray, + int dim_num); +template bool cell_in_subarray( + const uint8_t* cell, + const uint8_t* subarray, + int dim_num); +template bool cell_in_subarray( + const int16_t* cell, + const int16_t* subarray, + int dim_num); +template bool cell_in_subarray( + const uint16_t* cell, + const uint16_t* subarray, + int dim_num); +template bool cell_in_subarray( + const uint32_t* cell, + const uint32_t* subarray, + int dim_num); +template bool cell_in_subarray( + const uint64_t* cell, + const uint64_t* subarray, + int dim_num); template int cmp_col_order( const int* coords_a, @@ -2018,6 +2131,30 @@ template int cmp_col_order( const double* coords_a, const double* coords_b, int dim_num); +template int cmp_col_order( + const int8_t* coords_a, + const int8_t* coords_b, + int dim_num); +template int cmp_col_order( + const uint8_t* coords_a, + const uint8_t* coords_b, + int dim_num); +template int cmp_col_order( + const int16_t* coords_a, + const int16_t* coords_b, + int dim_num); +template int cmp_col_order( + const uint16_t* coords_a, + const uint16_t* coords_b, + int dim_num); +template int cmp_col_order( + const uint32_t* coords_a, + const uint32_t* coords_b, + int dim_num); +template int cmp_col_order( + const uint64_t* coords_a, + const uint64_t* coords_b, + int dim_num); template int cmp_col_order( int64_t id_a, @@ -2043,6 +2180,45 @@ template int cmp_col_order( int64_t id_b, const double* coords_b, int dim_num); +template int cmp_col_order( + int64_t id_a, + const int8_t* coords_a, + int64_t id_b, + const int8_t* coords_b, + int dim_num); +template int cmp_col_order( + int64_t id_a, + const uint8_t* coords_a, + int64_t id_b, + const uint8_t* coords_b, + int dim_num); +template int cmp_col_order( + int64_t id_a, + const int16_t* coords_a, + int64_t id_b, + const int16_t* coords_b, + int dim_num); +template int cmp_col_order( + int64_t id_a, + const uint16_t* coords_a, + int64_t id_b, + const uint16_t* coords_b, + int dim_num); +template int cmp_col_order( + int64_t id_a, + const uint32_t* coords_a, + int64_t id_b, + const uint32_t* coords_b, + int dim_num); +template int cmp_col_order( + int64_t id_a, + const uint64_t* coords_a, + int64_t id_b, + const uint64_t* coords_b, + int dim_num); + + + template int cmp_row_order( const int* coords_a, @@ -2060,6 +2236,30 @@ template int cmp_row_order( const double* coords_a, const double* coords_b, int dim_num); +template int cmp_row_order( + const int8_t* coords_a, + const int8_t* coords_b, + int dim_num); +template int cmp_row_order( + const uint8_t* coords_a, + const uint8_t* coords_b, + int dim_num); +template int cmp_row_order( + const int16_t* coords_a, + const int16_t* coords_b, + int dim_num); +template int cmp_row_order( + const uint16_t* coords_a, + const uint16_t* coords_b, + int dim_num); +template int cmp_row_order( + const uint32_t* coords_a, + const uint32_t* coords_b, + int dim_num); +template int cmp_row_order( + const uint64_t* coords_a, + const uint64_t* coords_b, + int dim_num); template int cmp_row_order( int64_t id_a, @@ -2085,11 +2285,53 @@ template int cmp_row_order( int64_t id_b, const double* coords_b, int dim_num); +template int cmp_row_order( + int64_t id_a, + const int8_t* coords_a, + int64_t id_b, + const int8_t* coords_b, + int dim_num); +template int cmp_row_order( + int64_t id_a, + const uint8_t* coords_a, + int64_t id_b, + const uint8_t* coords_b, + int dim_num); +template int cmp_row_order( + int64_t id_a, + const int16_t* coords_a, + int64_t id_b, + const int16_t* coords_b, + int dim_num); +template int cmp_row_order( + int64_t id_a, + const uint16_t* coords_a, + int64_t id_b, + const uint16_t* coords_b, + int dim_num); +template int cmp_row_order( + int64_t id_a, + const uint32_t* coords_a, + int64_t id_b, + const uint32_t* coords_b, + int dim_num); +template int cmp_row_order( + int64_t id_a, + const uint64_t* coords_a, + int64_t id_b, + const uint64_t* coords_b, + int dim_num); template bool empty_value(int value); template bool empty_value(int64_t value); template bool empty_value(float value); template bool empty_value(double value); +template bool empty_value(int8_t value); +template bool empty_value(uint8_t value); +template bool empty_value(int16_t value); +template bool empty_value(uint16_t value); +template bool empty_value(uint32_t value); +template bool empty_value(uint64_t value); template void expand_mbr( int* mbr, @@ -2107,6 +2349,30 @@ template void expand_mbr( double* mbr, const double* coords, int dim_num); +template void expand_mbr( + int8_t* mbr, + const int8_t* coords, + int dim_num); +template void expand_mbr( + uint8_t* mbr, + const uint8_t* coords, + int dim_num); +template void expand_mbr( + int16_t* mbr, + const int16_t* coords, + int dim_num); +template void expand_mbr( + uint16_t* mbr, + const uint16_t* coords, + int dim_num); +template void expand_mbr( + uint32_t* mbr, + const uint32_t* coords, + int dim_num); +template void expand_mbr( + uint64_t* mbr, + const uint64_t* coords, + int dim_num); template bool has_duplicates(const std::vector& v); @@ -2126,6 +2392,30 @@ template bool inside_subarray( const double* coords, const double* subarray, int dim_num); +template bool inside_subarray( + const int8_t* coords, + const int8_t* subarray, + int dim_num); +template bool inside_subarray( + const uint8_t* coords, + const uint8_t* subarray, + int dim_num); +template bool inside_subarray( + const int16_t* coords, + const int16_t* subarray, + int dim_num); +template bool inside_subarray( + const uint16_t* coords, + const uint16_t* subarray, + int dim_num); +template bool inside_subarray( + const uint32_t* coords, + const uint32_t* subarray, + int dim_num); +template bool inside_subarray( + const uint64_t* coords, + const uint64_t* subarray, + int dim_num); template bool intersect( const std::vector& v1, @@ -2147,9 +2437,39 @@ template bool is_contained( const double* range_A, const double* range_B, int dim_num); +template bool is_contained( + const int8_t* range_A, + const int8_t* range_B, + int dim_num); +template bool is_contained( + const uint8_t* range_A, + const uint8_t* range_B, + int dim_num); +template bool is_contained( + const int16_t* range_A, + const int16_t* range_B, + int dim_num); +template bool is_contained( + const uint16_t* range_A, + const uint16_t* range_B, + int dim_num); +template bool is_contained( + const uint32_t* range_A, + const uint32_t* range_B, + int dim_num); +template bool is_contained( + const uint64_t* range_A, + const uint64_t* range_B, + int dim_num); template bool is_unary_subarray(const int* subarray, int dim_num); template bool is_unary_subarray(const int64_t* subarray, int dim_num); template bool is_unary_subarray(const float* subarray, int dim_num); template bool is_unary_subarray(const double* subarray, int dim_num); +template bool is_unary_subarray(const int8_t* subarray, int dim_num); +template bool is_unary_subarray(const uint8_t* subarray, int dim_num); +template bool is_unary_subarray(const int16_t* subarray, int dim_num); +template bool is_unary_subarray(const uint16_t* subarray, int dim_num); +template bool is_unary_subarray(const uint32_t* subarray, int dim_num); +template bool is_unary_subarray(const uint64_t* subarray, int dim_num); diff --git a/core/src/storage_manager/storage_manager.cc b/core/src/storage_manager/storage_manager.cc index 9f154998dde..5fb38bff13c 100755 --- a/core/src/storage_manager/storage_manager.cc +++ b/core/src/storage_manager/storage_manager.cc @@ -166,6 +166,23 @@ int StorageManager::workspace_create(const std::string& workspace) { return TILEDB_SM_OK; } +int StorageManager::dir_type(const char* dir) { + // Get real path + std::string dir_real = real_dir(dir); + + // Return type + if(is_workspace(dir_real)) + return TILEDB_WORKSPACE; + else if(is_group(dir_real)) + return TILEDB_GROUP; + else if(is_array(dir_real)) + return TILEDB_ARRAY; + else if(is_metadata(dir_real)) + return TILEDB_METADATA; + else + return -1; +} + int StorageManager::ls_workspaces( char** workspaces, int& workspace_num) { diff --git a/examples/src/tiledb_directory_type.cc b/examples/src/tiledb_directory_type.cc new file mode 100644 index 00000000000..433444273c5 --- /dev/null +++ b/examples/src/tiledb_directory_type.cc @@ -0,0 +1,74 @@ +/** + * @file tiledb_directory_type.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2016 MIT and Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Checks the type of a given directory. + */ + +#include "tiledb.h" +#include + +void print_dir_type(int type) { + if(type == TILEDB_WORKSPACE) + std::cout << "Workspace\n"; + else if(type == TILEDB_GROUP) + std::cout << "Group\n"; + else if(type == TILEDB_ARRAY) + std::cout << "Array\n"; + else if(type == TILEDB_METADATA) + std::cout << "Metadata\n"; + else if(type == -1) + std::cout << "Not a TileDB object\n"; + else + std::cout << "Unknown directory type\n"; +} + +int main() { + // Initialize context with the default configuration parameters + TileDB_CTX* tiledb_ctx; + tiledb_ctx_init(&tiledb_ctx, NULL); + + // Create a workspace + tiledb_workspace_create(tiledb_ctx, "my_workspace"); + + // Create a group in the worskpace + tiledb_group_create(tiledb_ctx, "my_workspace/dense_arrays"); + + // Create two groups in the worskpace + tiledb_group_create(tiledb_ctx, "my_workspace/sparse_arrays"); + + // Check types + print_dir_type(tiledb_dir_type(tiledb_ctx, "my_workspace")); + print_dir_type(tiledb_dir_type(tiledb_ctx, "my_workspace/dense_arrays")); + print_dir_type(tiledb_dir_type(tiledb_ctx, "my_workspace/array")); + + // Finalize context + tiledb_ctx_finalize(tiledb_ctx); + + return 0; +}