Merge pull request #230 from TileDB-Inc/walk

Adding a tiledb_walk function and fixing minor URI bug.

core/include/c_api/tiledb.h

@@ -135,6 +135,13 @@ typedef enum {
 #undef TILEDB_COMPRESSOR_ENUM
 } tiledb_compressor_t;
 
+/** Walk traversal order. */
+typedef enum {
+#define TILEDB_WALK_ORDER_ENUM(id) TILEDB_##id
+#include "tiledb_enum.inc"
+#undef TILEDB_WALK_ORDER_ENUM
+} tiledb_walk_order_t;
+
 /* ****************************** */
 /*            VERSION             */
 /* ****************************** */
@@ -1066,6 +1073,34 @@ TILEDB_EXPORT int tiledb_delete(tiledb_ctx_t* ctx, const char* path);
 TILEDB_EXPORT int tiledb_move(
     tiledb_ctx_t* ctx, const char* old_path, const char* new_path, bool force);
 
+/**
+ * Walks (iterates) over the TileDB objects contained in *path*. The traversal
+ * is done recursively in the order defined by the user. The user provides
+ * a callback function which is applied on each of the visited TileDB objects.
+ * The iteration continues for as long the callback returns non-zero, and stops
+ * when the callback returns 0. Note that this function ignores any object
+ * (e.g., file or directory) that is not TileDB-related.
+ *
+ * @param ctx The TileDB context.
+ * @param path The path in which the traversal will occur.
+ * @param order The order of the recursive traversal (e.g., pre-order or
+ *     post-order.
+ * @param callback The callback function to be applied on every visited object.
+ *     The callback should return 0 if the iteration must stop, and 1
+ *     if the iteration must continue. It takes as input the currently visited
+ *     path, the type of that path (e.g., array or group), and the data
+ *     provided by the user for the callback. The callback returns -1 upon
+ *     error. Note that `path` in the callback will be an **absolute** path.
+ * @param data The data passed in the callback as the last argument.
+ * @return TILEDB_OK for success and TILEDB_ERR for error.
+ */
+TILEDB_EXPORT int tiledb_walk(
+    tiledb_ctx_t* ctx,
+    const char* path,
+    tiledb_walk_order_t order,
+    int (*callback)(const char*, tiledb_object_t, void*),
+    void* data);
+
 #undef TILEDB_EXPORT
 #ifdef __cplusplus
 }

core/include/c_api/tiledb_enum.inc

@@ -100,4 +100,10 @@ TILEDB_QUERY_STATUS_ENUM(FAILED) = -1,
 TILEDB_QUERY_STATUS_ENUM(COMPLETED) = 0,
 TILEDB_QUERY_STATUS_ENUM(INPROGRESS) = 1,
 TILEDB_QUERY_STATUS_ENUM(INCOMPLETE) = 2,
+#endif
+
+/** TileDB walk traversal order */
+#ifdef TILEDB_WALK_ORDER_ENUM
+TILEDB_WALK_ORDER_ENUM(PREORDER),
+TILEDB_WALK_ORDER_ENUM(POSTORDER),
 #endif

core/include/enums/walk_order.h

@@ -0,0 +1,47 @@
+/**
+ * @file walk_order.h
+ *
+ * @section LICENSE
+ *
+ * The MIT License
+ *
+ * @copyright Copyright (c) 2017 TileDB, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ * @section DESCRIPTION
+ *
+ * This file defines the tiledb WalkOrder enum that maps to the
+ * tiledb_walk_order_t C-api enum
+ */
+
+#ifndef TILEDB_WALK_ORDER_H
+#define TILEDB_WALK_ORDER_H
+
+namespace tiledb {
+
+enum class WalkOrder : char {
+#define TILEDB_WALK_ORDER_ENUM(id) id
+#include "tiledb_enum.inc"
+#undef TILEDB_WALK_ORDER_ENUM
+};
+
+}  // namespace tiledb
+
+#endif  // TILEDB_WALK_ORDER_H

core/include/storage_manager/storage_manager.h

@@ -35,6 +35,7 @@
 #define TILEDB_STORAGE_MANAGER_H
 
 #include <condition_variable>
+#include <list>
 #include <map>
 #include <mutex>
 #include <queue>
@@ -50,12 +51,34 @@
 #include "status.h"
 #include "uri.h"
 #include "vfs.h"
+#include "walk_order.h"
 
 namespace tiledb {
 
 /** The storage manager that manages pretty much everything in TileDB. */
 class StorageManager {
  public:
+  /* ********************************* */
+  /*          TYPE DEFINITIONS         */
+  /* ********************************* */
+
+  class ObjectIter {
+   public:
+    /**
+     * There is a one-to-one correspondence between `expanded_` and `objs_`.
+     * An `expanded_` value is `true` if the corresponding `objs_` path
+     * has been expanded to the paths it contains in a post ored traversal.
+     * This is not used in a preorder traversal.
+     */
+    std::list<bool> expanded_;
+    /** The next URI in string format. */
+    std::string next_;
+    /** The next objects to be visited. */
+    std::list<URI> objs_;
+    /** The traversal order of the iterator. */
+    WalkOrder order_;
+  };
+
   /* ********************************* */
   /*     CONSTRUCTORS & DESTRUCTORS    */
   /* ********************************* */
@@ -187,6 +210,73 @@ class StorageManager {
    */
   Status move_path(const URI& old_uri, const URI& new_uri, bool force = false);
 
+  /**
+   * Creates a new object iterator for the input path.
+   *
+   * @param obj_iter The object iterator to be created (memory is allocated for
+   *     it by the function).
+   * @param path The path the iterator will target at.
+   * @param order The traversal order of the iterator.
+   * @return Status
+   */
+  Status object_iter_begin(
+      ObjectIter** obj_iter, const char* path, WalkOrder order);
+
+  /** Frees the object iterator. */
+  void object_iter_free(ObjectIter* obj_iter);
+
+  /**
+   * Retrieves the next object path and type.
+   *
+   * @param obj_iter The object iterator.
+   * @param path The object path that is retrieved.
+   * @param type The object type that is retrieved.
+   * @param has_next True if an object path was retrieved and false otherwise.
+   * @return Status
+   */
+  Status object_iter_next(
+      ObjectIter* obj_iter,
+      const char** path,
+      ObjectType* type,
+      bool* has_next);
+
+  /**
+   * Retrieves the next object in the post-order traversal.
+   *
+   * @param obj_iter The object iterator.
+   * @param path The object path that is retrieved.
+   * @param type The object type that is retrieved.
+   * @param has_next True if an object path was retrieved and false otherwise.
+   * @return Status
+   */
+  Status object_iter_next_postorder(
+      ObjectIter* obj_iter,
+      const char** path,
+      ObjectType* type,
+      bool* has_next);
+
+  /**
+   * Retrieves the next object in the post-order traversal.
+   *
+   * @param obj_iter The object iterator.
+   * @param path The object path that is retrieved.
+   * @param type The object type that is retrieved.
+   * @param has_next True if an object path was retrieved and false otherwise.
+   * @return Status
+   */
+  Status object_iter_next_preorder(
+      ObjectIter* obj_iter,
+      const char** path,
+      ObjectType* type,
+      bool* has_next);
+
+  /**
+   * Returns the tiledb object type
+   * @param uri Path to tiledb object resource
+   * @return ObjectType
+   */
+  ObjectType object_type(const URI& uri) const;
+
   /** Finalizes a query. */
   Status query_finalize(Query* query);
 
@@ -278,13 +368,6 @@ class StorageManager {
    */
   Status write_to_file(const URI& uri, Buffer* buffer) const;
 
-  /**
-   * Returns the tiledb object type
-   * @param uri Path to tiledb object resource
-   * @return ObjectType
-   */
-  ObjectType object_type(const URI& uri) const;
-
  private:
   /* ********************************* */
   /*        PRIVATE ATTRIBUTES         */

core/src/c_api/tiledb.cc

@@ -85,8 +85,8 @@ struct tiledb_array_metadata_t {
 struct tiledb_attribute_iter_t {
   const tiledb_array_metadata_t* array_metadata_;
   tiledb_attribute_t* attr_;
-  int attr_num_;
-  int current_attr_;
+  unsigned int attr_num_;
+  unsigned int current_attr_;
 };
 
 struct tiledb_dimension_t {
@@ -96,8 +96,8 @@ struct tiledb_dimension_t {
 struct tiledb_dimension_iter_t {
   const tiledb_domain_t* domain_;
   tiledb_dimension_t* dim_;
-  int dim_num_;
-  int current_dim_;
+  unsigned int dim_num_;
+  unsigned int current_dim_;
 };
 
 struct tiledb_domain_t {
@@ -608,7 +608,7 @@ int tiledb_dimension_iter_create(
   (*dim_it)->domain_ = domain;
   (*dim_it)->dim_num_ = domain->domain_->dim_num();
   (*dim_it)->current_dim_ = 0;
-  if ((*dim_it)->dim_num_ <= 0) {
+  if ((*dim_it)->dim_num_ == 0) {
     (*dim_it)->dim_ = nullptr;
   } else {
     // Create a dimension struct inside the iterator struct
@@ -681,7 +681,7 @@ int tiledb_dimension_iter_next(
   if (dim_it->dim_ != nullptr) {
     delete dim_it->dim_->dim_;
 
-    if (dim_it->current_dim_ >= 0 && dim_it->current_dim_ < dim_it->dim_num_)
+    if (dim_it->current_dim_ < dim_it->dim_num_)
       dim_it->dim_->dim_ = new tiledb::Dimension(
           dim_it->domain_->domain_->dimension(dim_it->current_dim_));
     else
@@ -1053,7 +1053,7 @@ int tiledb_attribute_iter_create(
 
   // Initialize the rest members of the iterator
   (*attr_it)->current_attr_ = 0;
-  if ((*attr_it)->attr_num_ <= 0) {
+  if ((*attr_it)->attr_num_ == 0) {
     (*attr_it)->attr_ = nullptr;
   } else {
     // Create an attribute struct inside the iterator struct
@@ -1124,8 +1124,7 @@ int tiledb_attribute_iter_next(
   ++(attr_it->current_attr_);
   if (attr_it->attr_ != nullptr) {
     delete attr_it->attr_->attr_;
-    if (attr_it->current_attr_ >= 0 &&
-        attr_it->current_attr_ < attr_it->attr_num_) {
+    if (attr_it->current_attr_ < attr_it->attr_num_) {
       attr_it->attr_->attr_ = new tiledb::Attribute(
           attr_it->array_metadata_->array_metadata_->attribute(
               attr_it->current_attr_));
@@ -1375,15 +1374,15 @@ int tiledb_array_consolidate(tiledb_ctx_t* ctx, const char* array_name) {
 
 int tiledb_object_type(
     tiledb_ctx_t* ctx, const char* path, tiledb_object_t* type) {
-  if (sanity_check(ctx))
+  if (sanity_check(ctx) == TILEDB_ERR)
     return TILEDB_ERR;
   auto uri = tiledb::URI(path);
   *type = static_cast<tiledb_object_t>(ctx->storage_manager_->object_type(uri));
   return TILEDB_OK;
 }
 
 int tiledb_delete(tiledb_ctx_t* ctx, const char* path) {
-  if (sanity_check(ctx))
+  if (sanity_check(ctx) == TILEDB_ERR)
     return TILEDB_ERR;
   auto uri = tiledb::URI(path);
   if (save_error(ctx, ctx->storage_manager_->remove_path(uri)))
@@ -1393,11 +1392,63 @@ int tiledb_delete(tiledb_ctx_t* ctx, const char* path) {
 
 int tiledb_move(
     tiledb_ctx_t* ctx, const char* old_path, const char* new_path, bool force) {
-  if (sanity_check(ctx))
+  if (sanity_check(ctx) == TILEDB_ERR)
     return TILEDB_ERR;
   auto old_uri = tiledb::URI(old_path);
   auto new_uri = tiledb::URI(new_path);
   if (save_error(ctx, ctx->storage_manager_->move(old_uri, new_uri, force)))
     return TILEDB_ERR;
   return TILEDB_OK;
 }
+
+int tiledb_walk(
+    tiledb_ctx_t* ctx,
+    const char* path,
+    tiledb_walk_order_t order,
+    int (*callback)(const char*, tiledb_object_t, void*),
+    void* data) {
+  // Sanity checks
+  if (sanity_check(ctx))
+    return TILEDB_ERR;
+  if (callback == nullptr) {
+    save_error(
+        ctx,
+        tiledb::Status::Error(
+            "Cannot initiate walk; Invalid callback function"));
+    return TILEDB_ERR;
+  }
+
+  // Create an object iterator
+  tiledb::StorageManager::ObjectIter* obj_iter;
+  if (save_error(
+          ctx,
+          ctx->storage_manager_->object_iter_begin(
+              &obj_iter, path, (tiledb::WalkOrder)order)))
+    return TILEDB_ERR;
+
+  // For as long as there is another object and the callback indicates to
+  // continue, walk over the TileDB objects in the path
+  const char* obj_name;
+  tiledb::ObjectType obj_type;
+  bool has_next;
+  int rc = 0;
+  do {
+    if (save_error(
+            ctx,
+            ctx->storage_manager_->object_iter_next(
+                obj_iter, &obj_name, &obj_type, &has_next))) {
+      ctx->storage_manager_->object_iter_free(obj_iter);
+      return TILEDB_ERR;
+    }
+    if (!has_next)
+      break;
+    rc = callback(obj_name, tiledb_object_t(obj_type), data);
+  } while (rc == 1);
+
+  // Clean up
+  ctx->storage_manager_->object_iter_free(obj_iter);
+
+  if (rc == -1)
+    return TILEDB_ERR;
+  return TILEDB_OK;
+}

core/src/filesystem/hdfs_filesystem.cc

@@ -296,7 +296,7 @@ Status ls(hdfsFS fs, const URI& uri, std::vector<std::string>* paths) {
   }
   for (int i = 0; i < numEntries; ++i) {
     auto path = std::string(fileList[i].mName);
-    if (!utils::starts_with("hdfs://", path)) {
+    if (!utils::starts_with(path, "hdfs://")) {
       path = std::string("hdfs://") + path;
     }
     paths->push_back(path);

core/src/filesystem/posix_filesystem.cc

@@ -236,6 +236,8 @@ Status ls(const std::string& path, std::vector<std::string>* paths) {
     return Status::Ok();
   }
   while ((next_path = readdir(dir)) != 0) {
+    if (!strcmp(next_path->d_name, ".") || !strcmp(next_path->d_name, ".."))
+      continue;
     auto abspath = path + "/" + next_path->d_name;
     paths->push_back(abspath);
   }