Skip to content

Commit a0eec37

Browse files
ethanglaserethanglaser
authored
Merge branch 'main' into dev/eglaser/api-fallback2
2 parents edbec56 + c39a2ab commit a0eec37

File tree

24 files changed

+291
-42
lines changed

24 files changed

+291
-42
lines changed

‎.github/workflows/build-linux-arm.yml

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
# Copyright 2025 Intel Corporation
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: Linux Build and Test on ARM
16+
17+
on:
18+
push:
19+
branches:
20+
- main
21+
pull_request:
22+
23+
permissions:
24+
contents: read
25+
26+
# This allows a subsequently queued workflow run to interrupt previous runs
27+
concurrency:
28+
group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
29+
cancel-in-progress: true
30+
31+
jobs:
32+
build:
33+
name: ${{ matrix.cxx }}, ${{ matrix.build_type }}
34+
runs-on: ubuntu-22.04-arm
35+
strategy:
36+
matrix:
37+
build_type: [RelWithDebugInfo]
38+
cxx: [g++-11, g++-12, clang++-15]
39+
include:
40+
- cxx: g++-11
41+
cc: gcc-11
42+
- cxx: g++-12
43+
cc: gcc-12
44+
- cxx: clang++-15
45+
cc: clang-15
46+
47+
steps:
48+
- uses: actions/checkout@v4
49+
50+
- name: Configure build
51+
working-directory: ${{ runner.temp }}
52+
env:
53+
CXX: ${{ matrix.cxx }}
54+
CC: ${{ matrix.cc }}
55+
TEMP_WORKSPACE: ${{ runner.temp }}
56+
run: |
57+
cmake -B${TEMP_WORKSPACE}/build -S${GITHUB_WORKSPACE} \
58+
-DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
59+
-DSVS_BUILD_BINARIES=YES \
60+
-DSVS_BUILD_TESTS=YES \
61+
-DSVS_BUILD_EXAMPLES=YES \
62+
-DSVS_EXPERIMENTAL_LEANVEC=YES
63+
64+
- name: Build Tests and Utilities
65+
working-directory: ${{ runner.temp }}/build
66+
run: make -j$(nproc)
67+
68+
- name: Run tests
69+
env:
70+
CTEST_OUTPUT_ON_FAILURE: 1
71+
working-directory: ${{ runner.temp }}/build/tests
72+
run: ctest -C ${{ matrix.build_type }}

‎.github/workflows/build-macos.yaml

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# Copyright 2025 Intel Corporation
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: MacOS Build and Test
16+
17+
on:
18+
push:
19+
branches:
20+
- main
21+
pull_request:
22+
23+
permissions:
24+
contents: read
25+
26+
# This allows a subsequently queued workflow run to interrupt previous runs
27+
concurrency:
28+
group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
29+
cancel-in-progress: true
30+
31+
jobs:
32+
build:
33+
name: ${{ matrix.cxx }}, ${{ matrix.build_type }}
34+
runs-on: macos-latest
35+
strategy:
36+
matrix:
37+
build_type: [RelWithDebugInfo]
38+
cxx: [clang++-15]
39+
include:
40+
- cxx: clang++-15
41+
package: llvm@15
42+
cc_name: clang
43+
cxx_name: clang++
44+
needs_prefix: true
45+
46+
steps:
47+
- uses: actions/checkout@v4
48+
49+
- name: Install Compiler
50+
run: |
51+
echo "Installing ${{ matrix.package }}..."
52+
brew install ${{ matrix.package }}
53+
54+
- name: Configure build
55+
working-directory: ${{ runner.temp }}
56+
env:
57+
TEMP_WORKSPACE: ${{ runner.temp }}
58+
run: |
59+
if [[ "${{ matrix.needs_prefix }}" == "true" ]]; then
60+
# For non-default packages like llvm@15, get the install prefix
61+
COMPILER_PREFIX=$(brew --prefix ${{ matrix.package }})
62+
export CC="${COMPILER_PREFIX}/bin/${{ matrix.cc_name }}"
63+
export CXX="${COMPILER_PREFIX}/bin/${{ matrix.cxx_name }}"
64+
else
65+
# For versioned GCC installs, the name is usually directly available
66+
export CC="${{ matrix.cc_name }}"
67+
export CXX="${{ matrix.cxx_name }}"
68+
fi
69+
70+
cmake -B${TEMP_WORKSPACE}/build -S${GITHUB_WORKSPACE} \
71+
-DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
72+
-DSVS_BUILD_BINARIES=YES \
73+
-DSVS_BUILD_TESTS=YES \
74+
-DSVS_BUILD_EXAMPLES=YES \
75+
-DSVS_EXPERIMENTAL_LEANVEC=YES
76+
77+
- name: Build Tests and Utilities
78+
working-directory: ${{ runner.temp }}/build
79+
run: make -j$(sysctl -n hw.ncpu)
80+
81+
- name: Run tests
82+
env:
83+
CTEST_OUTPUT_ON_FAILURE: 1
84+
working-directory: ${{ runner.temp }}/build/tests
85+
run: ctest -C ${{ matrix.build_type }}

‎.github/workflows/cibuildwheel.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,3 +60,9 @@ jobs:
6060
working-directory: ${{ runner.temp }}
6161
run: python -m unittest discover -s ${GITHUB_WORKSPACE}/bindings/python
6262

63+
- name: Run examples
64+
env:
65+
PYTHONPATH: ${{ runner.temp }}/usr
66+
CTEST_OUTPUT_ON_FAILURE: 1
67+
working-directory: ${{ runner.temp }}
68+
run: python -m unittest discover -p "example*.py" -s ${GITHUB_WORKSPACE}/examples/python

‎benchmark/include/svs-benchmark/benchmark.h

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,19 @@ class SaveDirectoryChecker {
115115
~SaveDirectoryChecker() = default;
116116

117117
private:
118-
std::unordered_set<std::filesystem::path> directories_{};
118+
#if defined(__APPLE__)
119+
// Custom hash for macOS
120+
struct PathHash {
121+
std::size_t operator()(const std::filesystem::path& p) const {
122+
return std::hash<std::string>{}(p.string());
123+
}
124+
};
125+
using PathSet = std::unordered_set<std::filesystem::path, PathHash>;
126+
#else
127+
using PathSet = std::unordered_set<std::filesystem::path>;
128+
#endif // __APPLE__
129+
130+
PathSet directories_{};
119131
};
120132

121133
// Place-holder to indicate no extra arguments need forwarding to inner calls.

‎examples/python/example_vamana.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,12 @@
2121
# [imports]
2222

2323
DEBUG_MODE = False
24-
def assert_equal(lhs, rhs, message: str = ""):
24+
def assert_equal(lhs, rhs, message: str = "", epsilon = 0.05):
2525
if DEBUG_MODE:
2626
print(f"{message}: {lhs} == {rhs}")
2727
else:
28-
assert lhs == rhs, message
28+
assert lhs < rhs + epsilon, message
29+
assert lhs > rhs - epsilon, message
2930

3031
def run_test_float(index, queries, groundtruth):
3132
expected = {
@@ -79,7 +80,6 @@ def run_test_build_two_level4_8(index, queries, groundtruth):
7980
test_data_dir = None
8081

8182
def run():
82-
8383
# ###
8484
# Generating test data
8585
# ###
@@ -159,7 +159,7 @@ def run():
159159
# Compare with the groundtruth.
160160
recall = svs.k_recall_at(groundtruth, I, 10, 10)
161161
print(f"Recall = {recall}")
162-
assert(recall == 0.8288)
162+
assert_equal(recall, 0.8288)
163163
# [perform-queries]
164164

165165
# [search-window-size]
@@ -213,7 +213,7 @@ def run():
213213
# Compare with the groundtruth.
214214
recall = svs.k_recall_at(groundtruth, I, 10, 10)
215215
print(f"Recall = {recall}")
216-
assert(recall == 0.8288)
216+
assert_equal(recall, 0.8288)
217217
# [loading]
218218

219219
##### Begin Test

‎examples/python/example_vamana_dynamic.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,12 @@
2222
# [imports]
2323

2424
DEBUG_MODE = False
25-
def assert_equal(lhs, rhs, message: str = ""):
25+
def assert_equal(lhs, rhs, message: str = "", epsilon = 0.05):
2626
if DEBUG_MODE:
2727
print(f"{message}: {lhs} == {rhs}")
2828
else:
29-
assert lhs == rhs, message
29+
assert lhs < rhs + epsilon, message
30+
assert lhs > rhs - epsilon, message
3031

3132
def run_test_float(index, queries, groundtruth):
3233
expected = {
@@ -118,7 +119,7 @@ def run():
118119
# Compare with the groundtruth.
119120
recall = svs.k_recall_at(groundtruth, I, 10, 10)
120121
print(f"Recall = {recall}")
121-
assert(recall == 0.8202)
122+
assert_equal(recall, 0.8202)
122123
# [perform-queries]
123124

124125
##### Begin Test
@@ -158,8 +159,7 @@ def run():
158159
# Compare with the groundtruth.
159160
recall = svs.k_recall_at(groundtruth, I, 10, 10)
160161
print(f"Recall = {recall}")
161-
assert(recall == 0.8202)
162-
162+
assert_equal(recall, 0.8202)
163163

164164
##### Begin Test
165165
run_test_float(index, queries, groundtruth)

‎include/svs/core/allocator.h

Lines changed: 41 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,16 @@
4141
#include <array>
4242
#include <fcntl.h>
4343
#include <filesystem>
44+
45+
// <linux/mman.h> provides some linux-specific flags like
46+
// MAP_POPULATE, MAP_NORESERVE, MAP_HUGETLB.
47+
// They can be used only with linux, and are not availiable
48+
// for MacOS
49+
50+
#if defined(__linux__)
4451
#include <linux/mman.h>
52+
#endif // __linux__
53+
4554
#include <memory>
4655
#include <optional>
4756
#include <string>
@@ -78,11 +87,19 @@ struct HugepageX86Parameters {
7887
};
7988

8089
// Hugepage Allocation will happen in the order given below.
90+
//
91+
// MAP_HUGETLB, MAP_HUGE_1GB, MAP_HUGE_2MB are linux-specific
92+
//
93+
#if defined(__linux__)
8194
static constexpr std::array<HugepageX86Parameters, 3> hugepage_x86_options{
8295
HugepageX86Parameters{1 << 30, MAP_HUGETLB | MAP_HUGE_1GB},
8396
HugepageX86Parameters{1 << 21, MAP_HUGETLB | MAP_HUGE_2MB},
8497
HugepageX86Parameters{1 << 12, 0},
8598
};
99+
#else
100+
static constexpr std::array<HugepageX86Parameters, 1> hugepage_x86_options{
101+
HugepageX86Parameters{1 << 12, 0}};
102+
#endif // __linux__
86103

87104
namespace detail {
88105

@@ -104,14 +121,21 @@ struct HugepageAllocation {
104121
auto pagesize = params.pagesize;
105122
auto flags = params.mmap_flags;
106123
sz = lib::round_up_to_multiple_of(bytes, pagesize);
107-
ptr = mmap(
108-
nullptr,
109-
sz,
110-
PROT_READ | PROT_WRITE,
111-
MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE | flags,
112-
-1,
113-
0
114-
);
124+
125+
int mmap_flags = MAP_PRIVATE;
126+
#if defined(MAP_ANONYMOUS)
127+
mmap_flags |= MAP_ANONYMOUS;
128+
#elif defined(MAP_ANON)
129+
mmap_flags |= MAP_ANON;
130+
#else
131+
#error "System does not support anonymous mmap (no MAP_ANONYMOUS or MAP_ANON)"
132+
#endif
133+
// Add Linux-specific flags
134+
#ifdef __linux__
135+
mmap_flags |= MAP_POPULATE;
136+
#endif // __linux__
137+
138+
ptr = mmap(nullptr, sz, PROT_READ | PROT_WRITE, mmap_flags | flags, -1, 0);
115139

116140
if (ptr != MAP_FAILED) {
117141
break;
@@ -489,15 +513,16 @@ class MemoryMapper {
489513
}
490514
}
491515
lseek(fd, 0, SEEK_SET);
516+
517+
int mmap_flags = MAP_SHARED; // Accessible from all processes
518+
// Add Linux-specific flags
519+
#ifdef __linux__
520+
mmap_flags |= MAP_NORESERVE; // Don't reserve space in DRAM for this until used
521+
mmap_flags |= MAP_POPULATE; // Populate page table entries in the DRAM
522+
#endif // __linux__
523+
492524
void* base = ::mmap(
493-
nullptr,
494-
bytes.value(),
495-
mmap_permissions(permission_),
496-
MAP_NORESERVE // Don't reserve space in DRAM for this until used
497-
| MAP_SHARED // Accessible from all processes
498-
| MAP_POPULATE, // Populate page table entries in the DRAM
499-
fd,
500-
0
525+
nullptr, bytes.value(), mmap_permissions(permission_), mmap_flags, fd, 0
501526
);
502527
close(fd);
503528

‎include/svs/core/distance/cosine.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
#include <iostream>
3030
#include <memory>
3131
#include <type_traits>
32-
#include <x86intrin.h>
3332

3433
namespace svs::distance {
3534
// Forward declare implementation to allow entry point to be near the top.

‎include/svs/core/distance/euclidean.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
#include <memory>
3232
#include <span>
3333
#include <type_traits>
34-
#include <x86intrin.h>
3534

3635
// Implementation Notes regarding Intel(R) AVX Extentions
3736
// Top most entry in the bulleted list underneath each type pair <T,U> is the preferred

‎include/svs/core/distance/inner_product.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
#include <iostream>
3030
#include <memory>
3131
#include <type_traits>
32-
#include <x86intrin.h>
3332

3433
namespace svs::distance {
3534
// Forward declare implementation to allow entry point to be near the top.

0 commit comments

Comments
 (0)