Some modifications for vineyard (#8)

* Add template specialization to std::string_view.
Add inline keyword to get_available_memory.
Fix bug of inline failed when call _mm_popcnt_u64.

Signed-off-by: vegetableysm <[email protected]>

Author: vegetableysm

grape/graph/hashmap_indexer_impl.h

@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <vector>
 
+#include "grape/communication/sync_comm.h"
 #include "grape/utils/ref_vector.h"
 #include "grape/utils/string_view_vector.h"
 
@@ -43,6 +44,11 @@ inline int8_t log2(size_t value) {
   return table[((value - (value >> 1)) * 0x07EDD5E59A4E28C2) >> 58];
 }
 
+template <typename T>
+size_t vec_dump_bytes(T const& vec) {
+  return vec.size() * sizeof(vec.front()) + sizeof(typename T::size_type);
+}
+
 template <typename T>
 struct KeyBuffer {
  public:
@@ -96,6 +102,8 @@ struct KeyBuffer {
     dumper.dump_vec(inner_);
   }
 
+  size_t dump_size() { return vec_dump_bytes(inner_); }
+
  private:
   std::vector<T, Allocator<T>> inner_;
 };
@@ -144,9 +152,61 @@ struct KeyBuffer<nonstd::string_view> {
     dumper.dump_vec(inner_.offset_buffer());
   }
 
+  size_t dump_size() {
+    return vec_dump_bytes(inner_.content_buffer()) +
+           vec_dump_bytes(inner_.offset_buffer());
+  }
+
+ private:
+  StringViewVector inner_;
+};
+
+#if __cplusplus >= 201703L
+template <>
+struct KeyBuffer<std::string_view> {
+  KeyBuffer() = default;
+  ~KeyBuffer() = default;
+
+  std::string_view get(size_t idx) const {
+    std::string_view view(inner_[idx].data(), inner_[idx].size());
+    return view;
+  }
+
+  void push_back(const std::string_view& val) {
+    nonstd::string_view view(val.data(), val.size());
+    inner_.push_back(view);
+  }
+
+  size_t size() const { return inner_.size(); }
+
+  StringViewVector& buffer() { return inner_; }
+  const StringViewVector& buffer() const { return inner_; }
+
+  void swap(KeyBuffer& rhs) { inner_.swap(rhs.inner_); }
+
+  void clear() { inner_.clear(); }
+
+  template <typename Loader>
+  void load(Loader& loader) {
+    loader.load_vec(inner_.content_buffer());
+    loader.load_vec(inner_.offset_buffer());
+  }
+
+  template <typename Dumper>
+  void dump(Dumper& dumper) const {
+    dumper.dump_vec(inner_.content_buffer());
+    dumper.dump_vec(inner_.offset_buffer());
+  }
+
+  size_t dump_size() {
+    return vec_dump_bytes(inner_.content_buffer()) +
+           vec_dump_bytes(inner_.offset_buffer());
+  }
+
  private:
   StringViewVector inner_;
 };
+#endif
 
 template <typename T>
 struct KeyBufferView {

grape/graph/perfect_hash_indexer.h

@@ -17,7 +17,9 @@
 #define GRAPE_GRAPH_PERFECT_HASH_INDEXER_H_
 
 #include "grape/graph/hashmap_indexer_impl.h"
+#include "grape/util.h"
 #include "grape/utils/pthash_utils/single_phf_view.h"
+#include "grape/utils/string_view_vector.h"
 
 namespace grape {
 
@@ -69,6 +71,10 @@ class ImmPHIdxer {
     idxer_.init(buffer_.data(), buffer_.size());
   }
 
+  void Init(const char *buf, size_t size) {
+    idxer_.init(buf, size);
+  }
+
   size_t entry_num() const { return idxer_.entry_num(); }
 
   bool empty() const { return idxer_.empty(); }
@@ -119,33 +125,34 @@ class PHIdxerViewBuilder {
 
   void add(KEY_T&& oid) { keys_.push_back(std::move(oid)); }
 
-  ImmPHIdxer<KEY_T, INDEX_T> finish() {
-    mem_dumper dumper;
-    {
-      SinglePHFView<murmurhasher>::build(keys_.begin(), keys_.size(), dumper,
-                                         1);
-      mem_loader loader(dumper.buffer().data(), dumper.buffer().size());
-      SinglePHFView<murmurhasher> phf;
-      phf.load(loader);
-      hashmap_indexer_impl::KeyBuffer<KEY_T> key_buffer;
-
-      std::vector<KEY_T> ordered_keys(keys_.size());
-      for (auto& key : keys_) {
-        size_t idx = phf(key);
-        ordered_keys[idx] = key;
-      }
-      for (auto& key : ordered_keys) {
-        key_buffer.push_back(key);
-      }
-      key_buffer.dump(dumper);
+  void buildPhf() {
+    SinglePHFView<murmurhasher>::build(keys_.begin(),
+                                                     keys_.size(), phf, 1);
+    std::vector<KEY_T> ordered_keys(keys_.size());
+    for (auto& key : keys_) {
+      size_t idx = phf(key);
+      ordered_keys[idx] = key;
     }
-    ImmPHIdxer<KEY_T, INDEX_T> idxer;
-    idxer.Init(std::move(dumper.buffer()));
-    return idxer;
+    for (auto& key : ordered_keys) {
+      key_buffer.push_back(key);
+    }
+  }
+
+  void finish(void *buffer, size_t size, ImmPHIdxer<KEY_T, INDEX_T> &idxer) {
+    external_mem_dumper dumper(buffer, size);
+    phf.dump(dumper);
+    key_buffer.dump(dumper);
+    idxer.Init(static_cast<const char*>(dumper.buffer()), dumper.size());
+  }
+
+  size_t getSerializeSize() {
+    return phf.num_bits() / 8 + key_buffer.dump_size();
   }
 
  private:
   std::vector<KEY_T> keys_;
+  hashmap_indexer_impl::KeyBuffer<KEY_T> key_buffer;
+  pthash::single_phf<murmurhasher, pthash::dictionary_dictionary, true> phf;
 };
 
 }  // namespace grape

grape/types.h

@@ -156,6 +156,12 @@ struct murmurhasher {
   static inline hash_type hash(const nonstd::string_view& val, uint64_t seed) {
     return pthash::MurmurHash2_64(val.data(), val.size(), seed);
   }
+
+#if __cplusplus >= 201703L
+  static inline hash_type hash(std::string_view const& val, uint64_t seed) {
+    return pthash::MurmurHash2_64(val.data(), val.size(), seed);
+  }
+#endif
 };
 
 }  // namespace grape

grape/util.h

@@ -190,7 +190,7 @@ inline std::map<std::string, size_t> parse_meminfo() {
   return ret;
 }
 
-size_t get_available_memory() {
+inline size_t get_available_memory() {
   auto meminfo = parse_meminfo();
 #ifdef USE_HUGEPAGES
   return meminfo.at("HugePages_Free") * meminfo.at("Hugepagesize");

grape/utils/pthash_utils/single_phf_view.h

@@ -55,6 +55,46 @@ struct mem_dumper {
   std::vector<char> buf_;
 };
 
+struct external_mem_dumper {
+ public:
+  external_mem_dumper(void* buf, size_t size) : buf_(buf), size_(size) {}
+
+  ~external_mem_dumper() = default;
+
+  template <typename T>
+  void dump(const T& val) {
+    static_assert(std::is_pod<T>::value);
+    const char* ptr = reinterpret_cast<const char*>(&val);
+    if (pos_ + sizeof(T) > size_) {
+      return;
+    }
+    memcpy(reinterpret_cast<char*>(buf_) + pos_, ptr, sizeof(T));
+    pos_ += sizeof(T);
+  }
+
+  template <typename T, typename ALLOC_T>
+  void dump_vec(const std::vector<T, ALLOC_T>& vec) {
+    static_assert(std::is_pod<T>::value);
+    size_t n = vec.size();
+    if (pos_ + sizeof(T) * n + sizeof(size_t) > size_) {
+      return;
+    }
+    dump(n);
+    const char* ptr = reinterpret_cast<const char*>(vec.data());
+    memcpy(reinterpret_cast<char*>(buf_) + pos_, ptr, sizeof(T) * n);
+    pos_ += sizeof(T) * n;
+  }
+
+  const void* buffer() const { return buf_; }
+
+  size_t size() const { return size_; }
+
+ private:
+  void* buf_ = nullptr;
+  size_t pos_ = 0;
+  size_t size_ = 0;
+};
+
 struct mem_loader {
  public:
   mem_loader(const char* buf, size_t size)
@@ -147,6 +187,22 @@ struct SinglePHFView {
     phf.dump(dumper);
   }
 
+  template <typename Iterator>
+  static void build(
+      Iterator keys, uint64_t n,
+      pthash::single_phf<murmurhasher, pthash::dictionary_dictionary, true>&
+          phf,
+      int thread_num) {
+    pthash::build_configuration config;
+    config.c = 7.0;
+    config.alpha = 0.94;
+    config.num_threads = thread_num;
+    config.minimal_output = true;
+    config.verbose_output = false;
+
+    phf.build_in_internal_memory(keys, n, config);
+  }
+
  private:
   uint64_t m_seed;
   uint64_t m_num_keys;

grape/utils/string_view_vector.h

@@ -161,6 +161,50 @@ struct ref_vector<nonstd::string_view> {
   ref_vector<size_t> offsets_;
 };
 
+#if __cplusplus >= 201703L
+template <>
+struct ref_vector<std::string_view> {
+  ref_vector() {}
+  ~ref_vector() {}
+
+  size_t init(const void* buffer, size_t size) {
+    size_t buffer_size = buffer_.init(buffer, size);
+    const void* ptr = reinterpret_cast<const char*>(buffer) + buffer_size;
+    size_t offset_size = offsets_.init(ptr, size - buffer_size);
+    return buffer_size + offset_size;
+  }
+
+  ref_vector<char>& buffer() { return buffer_; }
+  ref_vector<size_t>& offsets() { return offsets_; }
+
+  const ref_vector<char>& buffer() const { return buffer_; }
+  const ref_vector<size_t>& offsets() const { return offsets_; }
+
+  size_t size() const {
+    if (offsets_.size() == 0) {
+      return 0;
+    }
+    return offsets_.size() - 1;
+  }
+
+  std::string_view get(size_t idx) const {
+    size_t from = offsets_.get(idx);
+    size_t to = offsets_.get(idx + 1);
+    return std::string_view(buffer_.data() + from, to - from);
+  }
+
+  template <typename Loader>
+  void load(Loader& loader) {
+    loader.load_ref_vec(buffer_);
+    loader.load_ref_vec(offsets_);
+  }
+
+ private:
+  ref_vector<char> buffer_;
+  ref_vector<size_t> offsets_;
+};
+#endif
+
 }  // namespace grape
 
 #endif  // GRAPE_UTILS_STRING_VIEW_VECTOR_H_

thirdparty/pthash/encoders/util.hpp

@@ -21,14 +21,21 @@
 
 #pragma once
 
+#include <cstdint>
+#include <cassert>
+
+#if defined(__x86_64__) && __SSE4_2__
 #include <immintrin.h>
+#endif
 
 namespace pthash::util {
 
+#if defined(__x86_64__) && __SSE4_2__
 template <typename T>
 inline void prefetch(T const* ptr) {
   _mm_prefetch(reinterpret_cast<const char*>(ptr), _MM_HINT_T0);
 }
+#endif
 
 inline uint8_t msb(uint64_t x) {
   assert(x);
@@ -66,7 +73,13 @@ inline uint8_t lsb(uint64_t x) {
 }
 
 inline uint64_t popcount(uint64_t x) {
-  return static_cast<uint64_t>(_mm_popcnt_u64(x));
+#ifdef __SSE4_2__
+    return static_cast<uint64_t>(_mm_popcnt_u64(x));
+#elif __cplusplus >= 202002L
+    return std::popcount(x);
+#else
+    return static_cast<uint64_t>(__builtin_popcountll(x));
+#endif
 }
 
 inline uint64_t select64_pdep_tzcnt(uint64_t x, const uint64_t k) {