es: use ignore_above setting for flattened mappings

datadope-io · Mar 30, 2021 · e6368cd · e6368cd
1 parent 88812a7
commit e6368cd
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 1 deletion.
diff --git a/analyzer/storage.go b/analyzer/storage.go
@@ -81,7 +81,8 @@ func newGraphBackendFromConfig(etcdClient *etcd.Client) (graph.PersistentBackend
 		var mapping map[string]interface{}
 		if useFlattened {
 			mapping = map[string]interface{}{
-				"type": "flattened",
+				"type":         "flattened",
+				"ignore_above": config.GetInt(configPath + ".flattened_ignore_above"),
 			}
 		} else {
 			mapping = map[string]interface{}{

diff --git a/config/config.go b/config/config.go
@@ -210,6 +210,7 @@ func init() {
 	cfg.SetDefault("storage.elasticsearch.indices_to_keep", 0)
 	cfg.SetDefault("storage.elasticsearch.index_prefix", "skydive_")
 	cfg.SetDefault("storage.elasticsearch.use_flattened", true)
+	cfg.SetDefault("storage.elasticsearch.flattened_ignore_above", 32768)
 	cfg.SetDefault("storage.elasticsearch.exclude_from_mapping", []string{
 		"Metadata.*.Extra",
 		"Metadata.Container.Labels",

diff --git a/etc/skydive.yml.default b/etc/skydive.yml.default
@@ -431,6 +431,12 @@ storage:
     #   - Metadata.OVN.Options
     #   - Metadata.OVN.IPv6RAConfigs
 
+    # Use flatten mapping type for fields specified by 'exclude_from_mapping'
+    # use_flattened: true
+
+    # Ignore flattened fields whose length is above the specified value
+    # flattened_ignore_above: 32768
+
     # Snif Nodes Info API to get all the nodes in the cluster
     # See https://pkg.go.dev/gopkg.in/olivere/elastic.v2?tab=doc#NewClient
     # Default: false