Skip to content

Commit fe1ab9b

Browse files
marchello2000marchello2000
authored
fix(elasticsearch): Upgrading to ElasticSearch 6 for entity tags (spinnaker#4386)
This change follows up on the change spinnaker#2913. * Remove ES2.x support - now all docs using mapping type `_doc` recommended by ES6 and only one is allowed. * Change the queries to use `entityRef.entityType` instead of the document type. * Fixup ES template to not use `not_analyzed` but instead use `keyword` (with exception of `entityRef.entityType` which we wildcard query) * Adding an `id` field to the document because the built-in `_id` field is no longer wildcard queryable * Changing to use a container for the unittests
1 parent dcf1828 commit fe1ab9b

File tree

5 files changed

+142
-118
lines changed

5 files changed

+142
-118
lines changed

‎clouddriver-elasticsearch/clouddriver-elasticsearch.gradle

+3-2
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,16 @@ dependencies {
77
implementation "com.netflix.spinnaker.kork:kork-security"
88
implementation "com.squareup.retrofit:retrofit"
99
implementation "org.codehaus.groovy:groovy-all"
10-
implementation("org.elasticsearch:elasticsearch:2.4.1") {
10+
implementation("org.elasticsearch:elasticsearch:6.8.6") {
1111
force = true
1212
}
1313

14-
implementation("io.searchbox:jest:2.0.3") {
14+
implementation("io.searchbox:jest:6.3.1") {
1515
force = true
1616
}
1717
implementation "org.springframework.boot:spring-boot-starter-web"
1818

19+
testCompile "org.testcontainers:elasticsearch:1.12.5"
1920
testImplementation "cglib:cglib-nodep"
2021
testImplementation "org.objenesis:objenesis"
2122
testImplementation "org.spockframework:spock-core"

‎clouddriver-elasticsearch/elasticsearch_index_template.json

+19-25
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
{
22
"order": 0,
3-
"template": "tags_v*",
3+
"index_patterns": [
4+
"tags_v*"
5+
],
46
"settings": {
57
"index": {
68
"number_of_shards": "6",
@@ -9,7 +11,7 @@
911
}
1012
},
1113
"mappings": {
12-
"_default_": {
14+
"_doc": {
1315
"dynamic": "false",
1416
"dynamic_templates": [
1517
{
@@ -24,62 +26,54 @@
2426
"entityRef_template": {
2527
"path_match": "entityRef.*",
2628
"mapping": {
27-
"index": "not_analyzed"
29+
"index": "keyword"
2830
}
2931
}
3032
}
3133
],
3234
"properties": {
35+
"id": {
36+
"type": "text"
37+
},
3338
"entityRef": {
3439
"properties": {
3540
"accountId": {
36-
"index": "not_analyzed",
37-
"type": "string"
41+
"type": "keyword"
3842
},
3943
"application": {
40-
"index": "not_analyzed",
41-
"type": "string"
44+
"type": "keyword"
4245
},
4346
"entityType": {
44-
"index": "not_analyzed",
45-
"type": "string"
47+
"type": "text"
4648
},
4749
"cloudProvider": {
48-
"index": "not_analyzed",
49-
"type": "string"
50+
"type": "keyword"
5051
},
5152
"entityId": {
52-
"index": "not_analyzed",
53-
"type": "string"
53+
"type": "keyword"
5454
},
5555
"region": {
56-
"index": "not_analyzed",
57-
"type": "string"
56+
"type": "keyword"
5857
},
5958
"account": {
60-
"index": "not_analyzed",
61-
"type": "string"
59+
"type": "keyword"
6260
}
6361
}
6462
},
6563
"tags": {
6664
"type": "nested",
6765
"properties": {
6866
"valueType": {
69-
"index": "not_analyzed",
70-
"type": "string"
67+
"type": "keyword"
7168
},
7269
"name": {
73-
"index": "not_analyzed",
74-
"type": "string"
70+
"type": "keyword"
7571
},
7672
"namespace": {
77-
"index": "not_analyzed",
78-
"type": "string"
73+
"type": "keyword"
7974
},
8075
"value": {
81-
"index": "not_analyzed",
82-
"type": "string"
76+
"type": "keyword"
8377
}
8478
}
8579
}

‎clouddriver-elasticsearch/src/main/java/com/netflix/spinnaker/clouddriver/elasticsearch/model/ElasticSearchEntityTagsProvider.java

+29-45
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import java.io.IOException;
4343
import java.util.*;
4444
import java.util.stream.Collectors;
45+
import org.apache.lucene.search.join.ScoreMode;
4546
import org.elasticsearch.index.query.BoolQueryBuilder;
4647
import org.elasticsearch.index.query.QueryBuilder;
4748
import org.elasticsearch.index.query.QueryBuilders;
@@ -64,7 +65,6 @@ public class ElasticSearchEntityTagsProvider implements EntityTagsProvider {
6465

6566
private final String activeElasticSearchIndex;
6667

67-
private final boolean singleMappingType;
6868
private final String mappingTypeName;
6969

7070
@Autowired
@@ -81,7 +81,6 @@ public ElasticSearchEntityTagsProvider(
8181
this.front50Service = front50Service;
8282
this.jestClient = jestClient;
8383
this.activeElasticSearchIndex = elasticSearchConfigProperties.getActiveIndex();
84-
this.singleMappingType = elasticSearchConfigProperties.isSingleMappingType();
8584
this.mappingTypeName = elasticSearchConfigProperties.getMappingTypeName();
8685
}
8786

@@ -131,6 +130,11 @@ public Collection<EntityTags> getAll(
131130
queryBuilder = queryBuilder.must(QueryBuilders.wildcardQuery("id", idPrefix));
132131
}
133132

133+
if (entityType != null) {
134+
queryBuilder =
135+
queryBuilder.must(QueryBuilders.wildcardQuery("entityRef.entityType", entityType));
136+
}
137+
134138
if (tags != null) {
135139
for (Map.Entry<String, Object> entry : tags.entrySet()) {
136140
// each key/value pair maps to a distinct nested `tags` object and must be a unique query
@@ -150,7 +154,7 @@ public Collection<EntityTags> getAll(
150154
queryBuilder = queryBuilder.must(applyTagsToBuilder(namespace, Collections.emptyMap()));
151155
}
152156

153-
return search(entityType, queryBuilder, maxResults);
157+
return search(queryBuilder, maxResults);
154158
}
155159

156160
@Override
@@ -173,7 +177,7 @@ public Optional<EntityTags> get(String id, Map<String, Object> tags) {
173177
}
174178
}
175179

176-
List<EntityTags> entityTags = search(null, queryBuilder, 1);
180+
List<EntityTags> entityTags = search(queryBuilder, 1);
177181
return entityTags.isEmpty() ? Optional.empty() : Optional.of(entityTags.get(0));
178182
}
179183

@@ -184,7 +188,7 @@ public void index(EntityTags entityTags) {
184188
new Index.Builder(
185189
objectMapper.convertValue(prepareForWrite(objectMapper, entityTags), Map.class))
186190
.index(activeElasticSearchIndex)
187-
.type(getDocumentType(entityTags))
191+
.type(mappingTypeName)
188192
.id(entityTags.getId())
189193
.build();
190194

@@ -209,13 +213,13 @@ public void bulkIndex(Collection<EntityTags> multipleEntityTags) {
209213
Bulk.Builder builder = new Bulk.Builder().defaultIndex(activeElasticSearchIndex);
210214

211215
for (EntityTags entityTags : tags) {
216+
Map tag =
217+
objectMapper.convertValue(prepareForWrite(objectMapper, entityTags), Map.class);
212218
builder =
213219
builder.addAction(
214-
new Index.Builder(
215-
objectMapper.convertValue(
216-
prepareForWrite(objectMapper, entityTags), Map.class))
220+
new Index.Builder(tag)
217221
.index(activeElasticSearchIndex)
218-
.type(getDocumentType(entityTags))
222+
.type(mappingTypeName)
219223
.id(entityTags.getId())
220224
.build());
221225
}
@@ -255,10 +259,7 @@ public void delete(String id) {
255259
}
256260

257261
Delete action =
258-
new Delete.Builder(id)
259-
.index(activeElasticSearchIndex)
260-
.type(getDocumentType(entityTags))
261-
.build();
262+
new Delete.Builder(id).index(activeElasticSearchIndex).type(mappingTypeName).build();
262263

263264
JestResult jestResult = jestClient.execute(action);
264265
if (!jestResult.isSucceeded()) {
@@ -281,9 +282,7 @@ public void bulkDelete(Collection<EntityTags> multipleEntityTags) {
281282
for (EntityTags entityTags : tags) {
282283
builder =
283284
builder.addAction(
284-
new Delete.Builder(entityTags.getId())
285-
.type(getDocumentType(entityTags))
286-
.build());
285+
new Delete.Builder(entityTags.getId()).type(mappingTypeName).build());
287286
}
288287

289288
Bulk bulk = builder.build();
@@ -352,9 +351,14 @@ public Map delta() {
352351
entityTagsByEntityTypeFront50
353352
.keySet()
354353
.forEach(
355-
entityType ->
356-
entityTagsByEntityTypeElasticsearch.put(
357-
entityType, fetchAll(QueryBuilders.matchAllQuery(), entityType, 5000, "2m")));
354+
entityType -> {
355+
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
356+
queryBuilder =
357+
queryBuilder.must(QueryBuilders.termQuery("entityRef.entityType", entityType));
358+
359+
entityTagsByEntityTypeElasticsearch.put(
360+
entityType, fetchAll(queryBuilder, 5000, "2m"));
361+
});
358362

359363
Map<String, Map> metadata = new HashMap<>();
360364

@@ -566,7 +570,7 @@ private List<EntityTags> getAllMatchingEntityTags(String namespace, String tag)
566570
QueryBuilders.boolQuery()
567571
.must(applyTagsToBuilder(null, Collections.singletonMap(tag, "*")));
568572

569-
fetchAll(queryBuilder, null, 5000, "2m")
573+
fetchAll(queryBuilder, 5000, "2m")
570574
.forEach(
571575
entityTags -> {
572576
if (!entityTagsIdentifiers.contains(entityTags.getId())) {
@@ -580,7 +584,7 @@ private List<EntityTags> getAllMatchingEntityTags(String namespace, String tag)
580584
BoolQueryBuilder queryBuilder =
581585
QueryBuilders.boolQuery().must(applyTagsToBuilder(namespace, Collections.emptyMap()));
582586

583-
fetchAll(queryBuilder, null, 5000, "2m")
587+
fetchAll(queryBuilder, 5000, "2m")
584588
.forEach(
585589
entityTags -> {
586590
if (!entityTagsIdentifiers.contains(entityTags.getId())) {
@@ -610,7 +614,7 @@ private QueryBuilder applyTagsToBuilder(String namespace, Map<String, Object> ta
610614
boolQueryBuilder.must(QueryBuilders.termQuery("tags.namespace", namespace));
611615
}
612616

613-
return QueryBuilders.nestedQuery("tags", boolQueryBuilder);
617+
return QueryBuilders.nestedQuery("tags", boolQueryBuilder, ScoreMode.Avg);
614618
}
615619

616620
/** Elasticsearch requires that all search criteria be flattened (vs. nested) */
@@ -628,17 +632,13 @@ private Map<String, Object> flatten(
628632
return accumulator;
629633
}
630634

631-
private List<EntityTags> search(String type, QueryBuilder queryBuilder, int maxResults) {
635+
private List<EntityTags> search(QueryBuilder queryBuilder, int maxResults) {
632636
SearchSourceBuilder searchSourceBuilder =
633637
new SearchSourceBuilder().query(queryBuilder).size(maxResults);
634638
String searchQuery = searchSourceBuilder.toString();
635639

636640
Search.Builder searchBuilder =
637641
new Search.Builder(searchQuery).addIndex(activeElasticSearchIndex);
638-
if (type != null) {
639-
// restrict to a specific index type (optional)
640-
searchBuilder.addType(type.toLowerCase());
641-
}
642642

643643
try {
644644
SearchResult searchResult = jestClient.execute(searchBuilder.build());
@@ -651,18 +651,13 @@ private List<EntityTags> search(String type, QueryBuilder queryBuilder, int maxR
651651
}
652652
}
653653

654-
private List<EntityTags> fetchAll(
655-
QueryBuilder queryBuilder, String type, int scrollSize, String scrollTime) {
654+
private List<EntityTags> fetchAll(QueryBuilder queryBuilder, int scrollSize, String scrollTime) {
656655
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
657656
searchSourceBuilder.query(queryBuilder);
658657

659658
Search.Builder builder =
660659
new Search.Builder(searchSourceBuilder.toString()).addIndex(activeElasticSearchIndex);
661660

662-
if (type != null) {
663-
builder.addType(type);
664-
}
665-
666661
Search search =
667662
builder
668663
.setParameter(Parameters.SIZE, scrollSize)
@@ -684,10 +679,7 @@ private List<EntityTags> fetchAll(
684679

685680
try {
686681
while (entityTags.size() > 0) {
687-
SearchScroll scroll =
688-
new SearchScroll.Builder(scrollId, scrollTime)
689-
.setParameter(Parameters.SIZE, scrollSize)
690-
.build();
682+
SearchScroll scroll = new SearchScroll.Builder(scrollId, scrollTime).build();
691683

692684
try {
693685
result = jestClient.execute(scroll);
@@ -748,12 +740,4 @@ private static EntityTags prepareForRead(ObjectMapper objectMapper, Map indexedE
748740

749741
return entityTags;
750742
}
751-
752-
private String getDocumentType(EntityTags entityTags) {
753-
if (singleMappingType) {
754-
return mappingTypeName;
755-
} else {
756-
return entityTags.getEntityRef().getEntityType();
757-
}
758-
}
759743
}

‎clouddriver-elasticsearch/src/main/java/com/netflix/spinnaker/config/ElasticSearchConfigProperties.java

-9
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ public class ElasticSearchConfigProperties {
3434
// The name of the unique mapping type is configurable as mappingTypeName, but is defaulted to
3535
// "_doc", which is
3636
// recommended for forward compatibility with Elasticsearch 7.0.
37-
private boolean singleMappingType = false;
3837
private String mappingTypeName = "_doc";
3938

4039
public String getActiveIndex() {
@@ -69,14 +68,6 @@ public void setConnectionTimeout(int connectionTimeout) {
6968
this.connectionTimeout = connectionTimeout;
7069
}
7170

72-
public void setSingleMappingType(boolean singleMappingType) {
73-
this.singleMappingType = singleMappingType;
74-
}
75-
76-
public boolean isSingleMappingType() {
77-
return singleMappingType;
78-
}
79-
8071
public void setMappingTypeName(String mappingTypeName) {
8172
this.mappingTypeName = mappingTypeName;
8273
}

0 commit comments

Comments
 (0)