Skip to content

Commit 9ed0fab

Browse files
luoxiaojianluoxiaojian
committed
tracking memory as option
1 parent 87f395a commit 9ed0fab

8 files changed

+31
-26
lines changed

‎CMakeLists.txt

+6
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ option(USE_JEMALLOC "Whether to use jemalloc." OFF)
1414
option(USE_HUGEPAGES "Whether to use hugepages." OFF)
1515
option(BUILD_SHARED_LIBS "Whether to build libgrape-lite as shared library" ON)
1616
option(PROFILING "Whether to enable profiling" OFF)
17+
option(TRACKING_MEMORY "Whether to enable memory tracking" OFF)
1718
option(WITH_ASAN "Build with Address Sanitizer" OFF)
1819
option(BUILD_LIBGRAPELITE_DOCS "Build libgrape-lite documentation" ON)
1920
option(BUILD_LIBGRAPELITE_TESTS "Build libgrape-lite test cases" ON)
@@ -38,6 +39,11 @@ if (PROFILING)
3839
add_definitions(-DPROFILING)
3940
endif ()
4041

42+
if (TRACKING_MEMORY)
43+
message(STATUS "Enable memory tracking")
44+
add_definitions(-DTRACKING_MEMORY)
45+
endif ()
46+
4147
if (WCC_USE_GID)
4248
add_definitions(-DWCC_USE_GID)
4349
endif ()

‎grape/fragment/basic_vc_ds_fragment_loader.h

+8-8
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ class BasicVCDSFragmentLoader {
140140
edge_num += partition_bucket_edge_num_[comm_spec_.fid()][k];
141141
}
142142
edges_.resize(edge_num);
143-
#ifdef TRACKING_MEMORY_ALLOCATIONS
143+
#ifdef TRACKING_MEMORY
144144
// allocate memory for edges
145145
MemoryTracker::GetInstance().allocate((sizeof(edge_t)) * edge_num);
146146
#endif
@@ -156,7 +156,7 @@ class BasicVCDSFragmentLoader {
156156
edges_to_frag_[fid].DisableComm();
157157
}
158158
}
159-
#ifdef TRACKING_MEMORY_ALLOCATIONS
159+
#ifdef TRACKING_MEMORY
160160
// allocate memory for edge shuffle-out buffer
161161
MemoryTracker::GetInstance().allocate(
162162
(sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
@@ -167,7 +167,7 @@ class BasicVCDSFragmentLoader {
167167
edge_move_threads_.emplace_back([this] {
168168
ShuffleBufferTuple<oid_t, oid_t, edata_t> cur;
169169
std::vector<std::vector<edge_t>> edges_cache(bucket_num_ * bucket_num_);
170-
#ifdef TRACKING_MEMORY_ALLOCATIONS
170+
#ifdef TRACKING_MEMORY
171171
// allocate memory for thread local edge cache
172172
MemoryTracker::GetInstance().allocate(
173173
(sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) *
@@ -187,7 +187,7 @@ class BasicVCDSFragmentLoader {
187187
edges_cache[bucket_id].clear();
188188
}
189189
});
190-
#ifdef TRACKING_MEMORY_ALLOCATIONS
190+
#ifdef TRACKING_MEMORY
191191
// deallocate edge shuffle-in buffer
192192
MemoryTracker::GetInstance().deallocate(
193193
(sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * cur_size);
@@ -200,7 +200,7 @@ class BasicVCDSFragmentLoader {
200200
edges_.begin() + cursor);
201201
}
202202
}
203-
#ifdef TRACKING_MEMORY_ALLOCATIONS
203+
#ifdef TRACKING_MEMORY
204204
// deallocate memory for thread local edge cache
205205
MemoryTracker::GetInstance().deallocate(
206206
(sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) *
@@ -218,7 +218,7 @@ class BasicVCDSFragmentLoader {
218218
edges_to_frag_[fid].Emplace(src, dst, data);
219219
if (fid == comm_spec_.fid() &&
220220
edges_to_frag_[fid].buffers().size() >= shuffle_out_size) {
221-
#ifdef TRACKING_MEMORY_ALLOCATIONS
221+
#ifdef TRACKING_MEMORY
222222
// allocate memory for edge shuffle-out(to self) buffer
223223
MemoryTracker::GetInstance().allocate(
224224
(sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) *
@@ -237,7 +237,7 @@ class BasicVCDSFragmentLoader {
237237
edge_recv_thread_.join();
238238
recv_thread_running_ = false;
239239
edges_to_frag_.clear();
240-
#ifdef TRACKING_MEMORY_ALLOCATIONS
240+
#ifdef TRACKING_MEMORY
241241
// deallocate memory for edge shuffle-out buffer
242242
MemoryTracker::GetInstance().deallocate(
243243
(sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
@@ -275,7 +275,7 @@ class BasicVCDSFragmentLoader {
275275
break;
276276
}
277277
CHECK_EQ(dst_fid, comm_spec_.fid());
278-
#ifdef TRACKING_MEMORY_ALLOCATIONS
278+
#ifdef TRACKING_MEMORY
279279
// allocate memory for edge shuffle-in buffer
280280
MemoryTracker::GetInstance().allocate(
281281
(sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) *

‎grape/fragment/basic_vc_fragment_loader.h

+7-7
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ class BasicVCFragmentLoader {
5050
}
5151
}
5252

53-
#ifdef TRACKING_MEMORY_ALLOCATIONS
53+
#ifdef TRACKING_MEMORY
5454
// allocate shuffle out buffers
5555
MemoryTracker::GetInstance().allocate(
5656
(sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
@@ -99,13 +99,13 @@ class BasicVCFragmentLoader {
9999
got_edges_memory_usage +=
100100
buf.size() * (sizeof(oid_t) * 2 + sizeof(edata_t));
101101
}
102-
#ifdef TRACKING_MEMORY_ALLOCATIONS
102+
#ifdef TRACKING_MEMORY
103103
// allocate edges recv buffer
104104
MemoryTracker::GetInstance().allocate(got_edges_memory_usage);
105105
#endif
106106

107107
edges_to_frag_[comm_spec_.fid()].Clear();
108-
#ifdef TRACKING_MEMORY_ALLOCATIONS
108+
#ifdef TRACKING_MEMORY
109109
// deallocate shuffle out buffers
110110
MemoryTracker::GetInstance().deallocate(
111111
(sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
@@ -163,14 +163,14 @@ class BasicVCFragmentLoader {
163163

164164
std::vector<edge_t> edges;
165165
edges.resize(edge_num);
166-
#ifdef TRACKING_MEMORY_ALLOCATIONS
166+
#ifdef TRACKING_MEMORY
167167
// allocate edges buffer
168168
MemoryTracker::GetInstance().allocate(sizeof(edge_t) * edge_num);
169169
#endif
170170

171171
{
172172
static constexpr size_t thread_local_cache_size = 128;
173-
#ifdef TRACKING_MEMORY_ALLOCATIONS
173+
#ifdef TRACKING_MEMORY
174174
// allocate thread local cache
175175
MemoryTracker::GetInstance().allocate(
176176
sizeof(edge_t) * bucket_num_ * bucket_num_ * thread_local_cache_size *
@@ -223,14 +223,14 @@ class BasicVCFragmentLoader {
223223
for (auto& thrd : insert_threads) {
224224
thrd.join();
225225
}
226-
#ifdef TRACKING_MEMORY_ALLOCATIONS
226+
#ifdef TRACKING_MEMORY
227227
// deallocate thread local cache
228228
MemoryTracker::GetInstance().deallocate(
229229
sizeof(edge_t) * bucket_num_ * bucket_num_ * thread_local_cache_size *
230230
load_concurrency_);
231231
#endif
232232
got_edges_.clear();
233-
#ifdef TRACKING_MEMORY_ALLOCATIONS
233+
#ifdef TRACKING_MEMORY
234234
// deallocate edges recv buffer
235235
MemoryTracker::GetInstance().deallocate(got_edges_memory_usage);
236236
#endif

‎grape/fragment/immutable_vertexcut_fragment.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
100100

101101
#ifdef USE_EDGE_ARRAY
102102
edges_.resize(edges.size());
103-
#ifdef TRACKING_MEMORY_ALLOCATIONS
103+
#ifdef TRACKING_MEMORY
104104
// reallocate memory for edges
105105
MemoryTracker::GetInstance().allocate(sizeof(edge_t) * edges_.size());
106106
MemoryTracker::GetInstance().deallocate(sizeof(edge_t) * edges_.size());
@@ -212,7 +212,7 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
212212

213213
if (std::is_pod<edata_t>::value && std::is_pod<oid_t>::value) {
214214
edges_.resize(edge_num);
215-
#ifdef TRACKING_MEMORY_ALLOCATIONS
215+
#ifdef TRACKING_MEMORY
216216
MemoryTracker::GetInstance().allocate(sizeof(edge_t) * edges_.size());
217217
#endif
218218
if (edge_num > 0) {

‎grape/fragment/vc_fragment_loader.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ class VCFragmentLoader {
119119
if (comm_spec_.worker_id() == 0) {
120120
VLOG(1) << "finished constructing fragment, time: " << t2 << " s";
121121
}
122-
#ifdef TRACKING_MEMORY_ALLOCATIONS
122+
#ifdef TRACKING_MEMORY
123123
VLOG(1) << "[frag-" << comm_spec_.fid() << "] after constructing fragment: "
124124
<< MemoryTracker::GetInstance().GetMemoryUsageInfo();
125125
#endif
@@ -223,7 +223,7 @@ class VCFragmentLoader {
223223
if (comm_spec_.worker_id() == 0) {
224224
VLOG(1) << "finished constructing fragment, time: " << t2 << " s";
225225
}
226-
#ifdef TRACKING_MEMORY_ALLOCATIONS
226+
#ifdef TRACKING_MEMORY
227227
VLOG(1) << "[frag-" << comm_spec_.fid() << "] after constructing fragment: "
228228
<< MemoryTracker::GetInstance().GetMemoryUsageInfo();
229229
#endif

‎grape/parallel/gather_scatter_message_manager.h

+4-4
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ class GatherScatterMessageManager : public MessageManagerBase {
107107
fid_t src_fid = (fid_ + fnum_ - i) % fnum_;
108108
auto src_vertices = frag.GetPartitioner().get_src_vertices(src_fid);
109109
if (output_range.IsSubsetOf(src_vertices)) {
110-
#ifdef TRACKING_MEMORY_ALLOCATIONS
110+
#ifdef TRACKING_MEMORY
111111
// allocate memory for received messages
112112
MemoryTracker::GetInstance().allocate(output_size);
113113
#endif
@@ -119,7 +119,7 @@ class GatherScatterMessageManager : public MessageManagerBase {
119119
}
120120
auto dst_vertices = frag.GetPartitioner().get_dst_vertices(src_fid);
121121
if (output_range.IsSubsetOf(dst_vertices)) {
122-
#ifdef TRACKING_MEMORY_ALLOCATIONS
122+
#ifdef TRACKING_MEMORY
123123
// allocate memory for received messages
124124
MemoryTracker::GetInstance().allocate(output_size);
125125
#endif
@@ -267,7 +267,7 @@ class GatherScatterMessageManager : public MessageManagerBase {
267267
thrd.join();
268268
}
269269

270-
#ifdef TRACKING_MEMORY_ALLOCATIONS
270+
#ifdef TRACKING_MEMORY
271271
// deallocate memory for received messages
272272
while (!from_heap.empty()) {
273273
MemoryTracker::GetInstance().deallocate(from_heap.front().size());
@@ -360,7 +360,7 @@ class GatherScatterMessageManager : public MessageManagerBase {
360360
}
361361
}
362362
}
363-
#ifdef TRACKING_MEMORY_ALLOCATIONS
363+
#ifdef TRACKING_MEMORY
364364
// allocate memory for received messages
365365
MemoryTracker::GetInstance().allocate(size);
366366
#endif

‎grape/utils/memory_tracker.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ limitations under the License.
1717

1818
namespace grape {
1919

20-
#ifdef TRACKING_MEMORY_ALLOCATIONS
20+
#ifdef TRACKING_MEMORY
2121

2222
MemoryTracker& MemoryTracker::GetInstance() {
2323
static MemoryTracker instance;

‎grape/utils/memory_tracker.h

+1-2
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@ limitations under the License.
2020

2121
namespace grape {
2222

23-
#define TRACKING_MEMORY_ALLOCATIONS
24-
#ifdef TRACKING_MEMORY_ALLOCATIONS
23+
#ifdef TRACKING_MEMORY
2524

2625
struct MemoryTracker {
2726
static MemoryTracker& GetInstance();

0 commit comments

Comments
 (0)