Skip to content

Commit be4a814

Browse files
slashdottedslashdotted
committed
Initial commit
1 parent c4e89f3 commit be4a814

14 files changed

+1528
-0
lines changed

‎.gitmodules

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[submodule "vcpkg"]
2+
path = vcpkg
3+
url = https://github.com/microsoft/vcpkg.git

‎CMakeLists.txt

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
cmake_minimum_required(VERSION 3.5)
2+
3+
project(mementohash LANGUAGES CXX)
4+
5+
set(CMAKE_CXX_STANDARD 20)
6+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
7+
8+
set(BUILD_SHARED_LIBS OFF)
9+
set(CMAKE_EXE_LINKER_FLAGS "-static")
10+
11+
option(WITH_PCG32 "Use PCG32 random number generator" OFF)
12+
option(WITH_HEAPSTATS "Enable heap allocation statistics" ON)
13+
14+
find_package(Boost REQUIRED)
15+
find_package(xxHash REQUIRED)
16+
find_package(fmt REQUIRED)
17+
find_package(cxxopts REQUIRED)
18+
19+
if(WITH_PCG32)
20+
find_path(PCG_INCLUDE_DIRS "pcg_extras.hpp")
21+
add_definitions(-DUSE_PCG32)
22+
endif()
23+
24+
if(WITH_HEAPSTATS)
25+
add_definitions(-DUSE_HEAPSTATS)
26+
endif()
27+
28+
add_executable(speed_test speed_test.cpp
29+
vcpkg.json
30+
memento/memento.h
31+
memento/mementoengine.h
32+
anchor/AnchorHashQre.cpp anchor/AnchorHashQre.hpp anchor/misc/crc32c_sse42_u64.h
33+
anchor/anchorengine.h
34+
memento/mashtable.h
35+
)
36+
37+
add_executable(balance balance.cpp
38+
vcpkg.json
39+
memento/memento.h
40+
memento/mementoengine.h
41+
anchor/AnchorHashQre.cpp anchor/AnchorHashQre.hpp anchor/misc/crc32c_sse42_u64.h
42+
anchor/anchorengine.h
43+
memento/mashtable.h
44+
)
45+
46+
add_executable(mashtable_test mashtable_test.cpp memento/mashtable.h)
47+
48+
target_include_directories(speed_test PRIVATE ${PCG_INCLUDE_DIRS})
49+
target_include_directories(balance PRIVATE ${PCG_INCLUDE_DIRS})
50+
target_link_libraries(speed_test PRIVATE xxHash::xxhash fmt::fmt cxxopts::cxxopts)
51+
target_link_libraries(balance PRIVATE xxHash::xxhash fmt::fmt cxxopts::cxxopts)
52+
53+
include(GNUInstallDirs)
54+
install(TARGETS speed_test
55+
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
56+
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
57+
)
58+
59+
install(TARGETS balance
60+
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
61+
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
62+
)

‎anchor/AnchorHashQre.cpp

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
// MIT License
2+
3+
// Copyright (c) 2020 anchorhash
4+
5+
// Permission is hereby granted, free of charge, to any person obtaining a copy
6+
// of this software and associated documentation files (the "Software"), to deal
7+
// in the Software without restriction, including without limitation the rights
8+
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
// copies of the Software, and to permit persons to whom the Software is
10+
// furnished to do so, subject to the following conditions:
11+
12+
// The above copyright notice and this permission notice shall be included in all
13+
// copies or substantial portions of the Software.
14+
15+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
// SOFTWARE.
22+
#include "AnchorHashQre.hpp"
23+
#include "./misc/crc32c_sse42_u64.h"
24+
25+
using namespace std;
26+
27+
/** Constructor */
28+
AnchorHashQre::AnchorHashQre (uint32_t a, uint32_t w) {
29+
30+
// Allocate the anchor array
31+
A = new uint32_t [a]();
32+
33+
// Allocate the working array
34+
W = new uint32_t [a]();
35+
36+
// Allocate the last apperance array
37+
L = new uint32_t [a]();
38+
39+
// Allocate the "map diagonal"
40+
K = new uint32_t [a]();
41+
42+
// Initialize "swap" arrays
43+
for(uint32_t i = 0; i < a; ++i) {
44+
L[i] = i;
45+
W[i] = i;
46+
K[i] = i;
47+
}
48+
49+
// We treat initial removals as ordered removals
50+
for(uint32_t i = a - 1; i >= w; --i) {
51+
A[i] = i;
52+
r.push(i);
53+
}
54+
55+
// Set initial set sizes
56+
M = a;
57+
N = w;
58+
59+
}
60+
61+
/** Destructor */
62+
AnchorHashQre::~AnchorHashQre () {
63+
64+
delete [] A;
65+
delete [] W;
66+
delete [] L;
67+
delete [] K;
68+
69+
}
70+
71+
uint32_t AnchorHashQre::ComputeTranslation(uint32_t i , uint32_t j) {
72+
73+
if (i == j) return K[i];
74+
75+
uint32_t b = j;
76+
77+
while (A[i] <= A[b]) {
78+
b = K[b];
79+
}
80+
81+
return b;
82+
83+
}
84+
85+
uint32_t AnchorHashQre::ComputeBucket(uint64_t key1 , uint64_t key2) {
86+
87+
// First hash is uniform on the anchor set
88+
uint32_t bs = crc32c_sse42_u64(key1, key2);
89+
uint32_t b = bs % M;
90+
91+
// Loop until hitting a working bucket
92+
while (A[b] != 0) {
93+
94+
// New candidate (bs - for better balance - avoid patterns)
95+
bs = crc32c_sse42_u64(key1 - bs, key2 + bs);
96+
uint32_t h = bs % A[b];
97+
98+
// h is working or observed by bucket
99+
if ((A[h] == 0) || (A[h] < A[b])) {
100+
b = h;
101+
}
102+
103+
// need translation for (bucket, h)
104+
else {
105+
b = ComputeTranslation(b,h);
106+
}
107+
108+
}
109+
110+
return b;
111+
112+
}
113+
114+
uint32_t AnchorHashQre::UpdateRemoval(uint32_t b) {
115+
116+
// update reserved stack
117+
r.push(b);
118+
119+
// update live set size
120+
N--;
121+
122+
// who is the replacement
123+
W[L[b]] = W[N];
124+
L[W[N]] = L[b];
125+
126+
// Update map diagonal
127+
K[b] = W[N];
128+
129+
// Update removal
130+
A[b] = N;
131+
132+
return 0;
133+
134+
}
135+
136+
uint32_t AnchorHashQre::UpdateNewBucket() {
137+
138+
// Who was removed last?
139+
uint32_t b = r.top();
140+
r.pop();
141+
142+
// Restore in observed_set
143+
L[W[N]] = N;
144+
W[L[b]] = b;
145+
146+
// update live set size
147+
N++;
148+
149+
// Ressurect
150+
A[b] = 0;
151+
152+
// Restore in diagonal
153+
K[b] = b;
154+
155+
return b;
156+
157+
}

‎anchor/AnchorHashQre.hpp

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// MIT License
2+
3+
// Copyright (c) 2020 anchorhash
4+
5+
// Permission is hereby granted, free of charge, to any person obtaining a copy
6+
// of this software and associated documentation files (the "Software"), to deal
7+
// in the Software without restriction, including without limitation the rights
8+
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
// copies of the Software, and to permit persons to whom the Software is
10+
// furnished to do so, subject to the following conditions:
11+
12+
// The above copyright notice and this permission notice shall be included in all
13+
// copies or substantial portions of the Software.
14+
15+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
// SOFTWARE.
22+
#include <iostream>
23+
#include <stack>
24+
#include <stdint.h>
25+
26+
/** Class declaration */
27+
class AnchorHashQre {
28+
29+
private:
30+
31+
// Anchor
32+
uint32_t *A;
33+
34+
// Working
35+
uint32_t *W;
36+
37+
// Last appearance
38+
uint32_t *L;
39+
40+
// "Map diagonal"
41+
uint32_t *K;
42+
43+
// Size of the anchor
44+
uint32_t M;
45+
46+
// Size of the working
47+
uint32_t N;
48+
49+
// Removed buckets
50+
std::stack<uint32_t> r;
51+
52+
// Translation oracle
53+
uint32_t ComputeTranslation(uint32_t i , uint32_t j);
54+
55+
public:
56+
57+
AnchorHashQre (uint32_t, uint32_t);
58+
59+
~AnchorHashQre();
60+
61+
uint32_t ComputeBucket(uint64_t, uint64_t);
62+
63+
uint32_t UpdateRemoval(uint32_t);
64+
65+
uint32_t UpdateNewBucket();
66+
67+
};

‎anchor/anchorengine.h

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Copyright (c) 2023 Amos Brocco.
3+
*
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation, version 3.
7+
*
8+
* This program is distributed in the hope that it will be useful, but
9+
* WITHOUT ANY WARRANTY; without even the implied warranty of
10+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11+
* General Public License for more details.
12+
*
13+
* You should have received a copy of the GNU General Public License
14+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
15+
*/
16+
#ifndef ANCHORENGINE_H
17+
#define ANCHORENGINE_H
18+
#include "AnchorHashQre.hpp"
19+
20+
class AnchorEngine final {
21+
public:
22+
AnchorEngine(uint32_t anchor_set, uint32_t working_set)
23+
: m_anchor{anchor_set, working_set}
24+
{}
25+
26+
/**
27+
* Returns the bucket where the given key should be mapped.
28+
* This version uses the same hash function as Anchor
29+
*
30+
* @param key the key to map
31+
* @param key the initial seed for CRC32c
32+
* @return the related bucket
33+
*/
34+
uint32_t getBucketCRC32c(uint64_t key, uint64_t seed) noexcept
35+
{
36+
return m_anchor.ComputeBucket(key, seed);
37+
}
38+
39+
/**
40+
* Adds a new bucket to the engine.
41+
*
42+
* @return the added bucket
43+
*/
44+
uint32_t addBucket() noexcept { return m_anchor.UpdateNewBucket(); }
45+
46+
/**
47+
* Removes the given bucket from the engine.
48+
*
49+
* @param bucket the bucket to remove
50+
* @return the removed bucket
51+
*/
52+
uint32_t removeBucket(uint32_t bucket) noexcept
53+
{
54+
m_anchor.UpdateRemoval(bucket);
55+
return bucket;
56+
}
57+
58+
private:
59+
AnchorHashQre m_anchor;
60+
};
61+
62+
#endif // ANCHORENGINE_H

‎anchor/misc/crc32c_sse42_u64.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
// Harware based crc calculation
2+
3+
static inline uint32_t crc32c_sse42_u64(uint64_t key, uint64_t seed) {
4+
__asm__ volatile(
5+
"crc32q %[key], %[seed];"
6+
: [seed] "+r" (seed)
7+
: [key] "rm" (key));
8+
return seed;
9+
}
10+

0 commit comments

Comments
 (0)