Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

atlas: add method to find hotspots in index #1098

Merged
merged 1 commit into from
Dec 2, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -19,12 +19,15 @@
import com.netflix.spectator.api.Registry;
import com.netflix.spectator.impl.Cache;

import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Deque;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
@@ -533,6 +536,69 @@ public void forEachMatch(Function<String, String> tags, Consumer<T> consumer) {
}
}

/**
* Find hot spots in the index where there is a large set of linear matches, e.g. a bunch
* of regex queries for a given key.
*
* @param threshold
* Threshold for the number of entries in the other checks sub-tree to be considered
* a hot spot.
* @param consumer
* Function that will be invoked with a path and set of queries for the hot spot.
*/
public void findHotSpots(int threshold, BiConsumer<List<String>, List<Query.KeyQuery>> consumer) {
Deque<String> path = new ArrayDeque<>();
findHotSpots(threshold, path, consumer);
}

private void findHotSpots(
int threshold,
Deque<String> path,
BiConsumer<List<String>, List<Query.KeyQuery>> consumer
) {
if (key != null) {
path.addLast("K=" + key);

equalChecks.forEach((v, idx) -> {
path.addLast(key + "," + v + ",:eq");
idx.findHotSpots(threshold, path, consumer);
path.removeLast();
});

path.addLast("other-checks");
if (otherChecks.size() > threshold) {
List<Query.KeyQuery> queries = new ArrayList<>(otherChecks.keySet());
consumer.accept(new ArrayList<>(path), queries);
}
otherChecks.forEach((q, idx) -> {
path.addLast(q.toString());
idx.findHotSpots(threshold, path, consumer);
path.removeLast();
});
path.removeLast();

if (hasKeyIdx != null) {
path.addLast("has");
hasKeyIdx.findHotSpots(threshold, path, consumer);
path.removeLast();
}

path.removeLast();
}

if (otherKeysIdx != null) {
path.addLast("other-keys");
otherKeysIdx.findHotSpots(threshold, path, consumer);
path.removeLast();
}

if (missingKeysIdx != null) {
path.addLast("missing-keys");
missingKeysIdx.findHotSpots(threshold, path, consumer);
path.removeLast();
}
}

@Override public String toString() {
StringBuilder builder = new StringBuilder();
buildString(builder, 0);
Original file line number Diff line number Diff line change
@@ -533,4 +533,22 @@ public void addRemoveFuzz() {
Assertions.assertTrue(idx.isEmpty());
}
}

@Test
public void findHotSpots() {
Registry registry = new NoopRegistry();
QueryIndex<Integer> idx = QueryIndex.newInstance(registry);
for (int i = 0; i < 5; ++i) {
Query q = Parser.parseQuery("name,foo,:re,i," + i + ",:re,:and");
idx.add(q, i);
}

idx.findHotSpots(4, (path, queries) -> {
Assertions.assertEquals(
"K=name > other-checks > name,foo,:re > K=i > other-checks",
String.join(" > ", path)
);
Assertions.assertEquals(5, queries.size());
});
}
}