Skip to content

Commit

Permalink
Standardize container metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
lebauce committed Dec 23, 2020
1 parent 6dde778 commit a30d0e1
Show file tree
Hide file tree
Showing 23 changed files with 477 additions and 991 deletions.
6 changes: 3 additions & 3 deletions agent/topology_probes.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,12 @@ import (
"github.com/skydive-project/skydive/graffiti/logging"
"github.com/skydive-project/skydive/plugin"
"github.com/skydive-project/skydive/probe"
"github.com/skydive-project/skydive/topology"
tp "github.com/skydive-project/skydive/topology/probes"
"github.com/skydive-project/skydive/topology/probes/bess"
"github.com/skydive-project/skydive/topology/probes/blockdev"
"github.com/skydive-project/skydive/topology/probes/docker"
"github.com/skydive-project/skydive/topology/probes/hardware"
"github.com/skydive-project/skydive/topology/probes/libvirt"
"github.com/skydive-project/skydive/topology/probes/lldp"
"github.com/skydive-project/skydive/topology/probes/lxd"
Expand All @@ -42,19 +44,17 @@ import (
"github.com/skydive-project/skydive/topology/probes/runc"
"github.com/skydive-project/skydive/topology/probes/socketinfo"
"github.com/skydive-project/skydive/topology/probes/vpp"
"github.com/skydive-project/skydive/topology/probes/hardware"
)

func registerStaticProbes() {
blockdev.Register()
netlink.Register()
docker.Register()
topology.RegisterContainer()
lldp.Register()
lxd.Register()
neutron.Register()
opencontrail.Register()
ovsdb.Register()
runc.Register()
libvirt.Register()
ovn.Register()
hardware.Register()
Expand Down
6 changes: 2 additions & 4 deletions analyzer/probes.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ import (
"github.com/skydive-project/skydive/plugin"
"github.com/skydive-project/skydive/probe"
"github.com/skydive-project/skydive/sflow"
"github.com/skydive-project/skydive/topology"
"github.com/skydive-project/skydive/topology/probes/blockdev"
"github.com/skydive-project/skydive/topology/probes/docker"
"github.com/skydive-project/skydive/topology/probes/fabric"
"github.com/skydive-project/skydive/topology/probes/istio"
"github.com/skydive-project/skydive/topology/probes/k8s"
Expand All @@ -41,19 +41,17 @@ import (
"github.com/skydive-project/skydive/topology/probes/ovn"
"github.com/skydive-project/skydive/topology/probes/ovsdb"
"github.com/skydive-project/skydive/topology/probes/peering"
"github.com/skydive-project/skydive/topology/probes/runc"
)

func registerStaticProbes() {
netlink.Register()
blockdev.Register()
docker.Register()
topology.RegisterContainer()
lldp.Register()
lxd.Register()
neutron.Register()
opencontrail.Register()
ovsdb.Register()
runc.Register()
libvirt.Register()
ovn.Register()
}
Expand Down
3 changes: 2 additions & 1 deletion config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,8 @@ func init() {
cfg.SetDefault("storage.elasticsearch.use_flattened", true)
cfg.SetDefault("storage.elasticsearch.exclude_from_mapping", []string{
"Metadata.*.Extra",
"Metadata.Docker.Labels",
"Metadata.Container.Labels",
"Metadata.Container.Hosts.ByIP",
"Metadata.K8s.Labels",
"Metadata.Actions",
"Metadata.Filters",
Expand Down
3 changes: 2 additions & 1 deletion etc/skydive.yml.default
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,8 @@ storage:
# Fields to exclude to avoid mapping explosion.
# exclude_from_mapping:
# - Metadata.*.Extra
# - Metadata.Docker.Labels
# - Metadata.Container.Labels
# - Metadata.Container.Hosts.ByIP
# - Metadata.K8s.Labels
# - Metadata.Actions
# - Metadata.Filters
Expand Down
10 changes: 5 additions & 5 deletions graffiti/graph/indexer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,18 +81,18 @@ func TestMetadataIndexer(t *testing.T) {
filter := filters.NewAndFilter(
filters.NewTermStringFilter("Manager", "docker"),
filters.NewTermStringFilter("Type", "container"),
filters.NewNotNullFilter("Docker.Labels.io.kubernetes.pod.name"),
filters.NewNotNullFilter("Docker.Labels.io.kubernetes.pod.namespace"),
filters.NewNotNullFilter("Docker.Labels.io.kubernetes.container.name"))
filters.NewNotNullFilter("Container.Labels.io.kubernetes.pod.name"),
filters.NewNotNullFilter("Container.Labels.io.kubernetes.pod.namespace"),
filters.NewNotNullFilter("Container.Labels.io.kubernetes.container.name"))
m := NewElementFilter(filter)

dockerCache := NewMetadataIndexer(g, g, m, "Docker.Labels.io.kubernetes.pod.namespace", "Docker.Labels.io.kubernetes.pod.name", "Docker.Labels.io.kubernetes.container.name")
dockerCache := NewMetadataIndexer(g, g, m, "Container.Labels.io.kubernetes.pod.namespace", "Container.Labels.io.kubernetes.pod.name", "Container.Labels.io.kubernetes.container.name")
dockerCache.Start()

m5 := Metadata{
"Type": "container",
"Manager": "docker",
"Docker": map[string]interface{}{
"Container": map[string]interface{}{
"Labels": map[string]interface{}{
"io": map[string]interface{}{
"kubernetes": map[string]interface{}{
Expand Down
4 changes: 2 additions & 2 deletions statics/bindata.go

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions statics/css/skydive.css
Original file line number Diff line number Diff line change
Expand Up @@ -671,6 +671,11 @@ ul.dropdown-menu .fa.pull-right {
margin-bottom: 5px;
}

.object-key-value.container {
padding-left: 0px;
width: 100%;
}

.object-sub-detail .object-detail {
margin-left: 20px;
}
Expand Down
14 changes: 7 additions & 7 deletions statics/js/components/topology.js
Original file line number Diff line number Diff line change
Expand Up @@ -211,9 +211,9 @@ var TopologyComponent = {
title="Metadata" :description="\'ID: \'+currentEdge.id">\
<object-detail :object="currentEdgeMetadata"></object-detail>\
</panel>\
<panel id="docker-metadata" v-if="currentNodeDocker"\
title="Docker">\
<object-detail :object="currentNodeDocker"></object-detail>\
<panel id="container-metadata" v-if="currentNodeContainer"\
title="Container">\
<object-detail :object="currentNodeContainer"></object-detail>\
</panel>\
<panel id="edge-src-metadata" v-if="currentEdgeSrc"\
title="Source">\
Expand Down Expand Up @@ -468,7 +468,7 @@ var TopologyComponent = {
currentNodeMetadata: function() {
if (!this.currentNode) return null;
return this.extractMetadata(this.currentNode.metadata,
['LastUpdateMetric', 'Metric', 'Blockdev', 'Ovs.Metric', 'Ovs.LastUpdateMetric', 'SFlow.Metric', 'SFlow.LastUpdateMetric', 'RoutingTables', 'Features', 'K8s.Extra', 'Docker']);
['LastUpdateMetric', 'Metric', 'Blockdev', 'Ovs.Metric', 'Ovs.LastUpdateMetric', 'SFlow.Metric', 'SFlow.LastUpdateMetric', 'RoutingTables', 'Features', 'K8s.Extra', 'Container']);
},

currentNodeFlowsQuery: function() {
Expand All @@ -477,9 +477,9 @@ var TopologyComponent = {
return null;
},

currentNodeDocker: function() {
if (!this.currentNodeMetadata || !this.currentNode.metadata.Docker) return null;
return this.currentNode.metadata.Docker;
currentNodeContainer: function() {
if (!this.currentNodeMetadata || !this.currentNode.metadata.Container) return null;
return this.currentNode.metadata.Container;
},

currentEdgeSrc: function() {
Expand Down
10 changes: 5 additions & 5 deletions tests/docker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func TestDockerSimple(t *testing.T) {

checks: []CheckFunction{func(c *CheckContext) error {
gremlin := c.gremlin.V().Has("Type", "netns", "Manager", "docker")
gremlin = gremlin.Out("Type", "container", "Docker.ContainerName", "test-skydive-docker-simple")
gremlin = gremlin.Out("Type", "container", "Name", "test-skydive-docker-simple")

nodes, err := c.gh.GetNodes(gremlin)
if err != nil {
Expand Down Expand Up @@ -74,7 +74,7 @@ func TestDockerShareNamespace(t *testing.T) {

checks: []CheckFunction{func(c *CheckContext) error {
gremlin := c.gremlin.V().Has("Type", "netns", "Manager", "docker")
gremlin = gremlin.Out().Has("Type", "container", "Docker.ContainerName", g.Within("test-skydive-docker-share-ns", "test-skydive-docker-share-ns2"))
gremlin = gremlin.Out().Has("Type", "container", "Name", g.Within("test-skydive-docker-share-ns", "test-skydive-docker-share-ns2"))
nodes, err := c.gh.GetNodes(gremlin)
if err != nil {
return err
Expand Down Expand Up @@ -104,7 +104,7 @@ func TestDockerNetHost(t *testing.T) {
mode: Replay,

checks: []CheckFunction{func(c *CheckContext) error {
gremlin := c.gremlin.V().Has("Docker.ContainerName", "test-skydive-docker-net-host", "Type", "container")
gremlin := c.gremlin.V().Has("Name", "test-skydive-docker-net-host", "Type", "container")
nodes, err := c.gh.GetNodes(gremlin)
if err != nil {
return err
Expand Down Expand Up @@ -159,11 +159,11 @@ func TestDockerLabels(t *testing.T) {
mode: Replay,

checks: []CheckFunction{func(c *CheckContext) error {
gremlin := c.gremlin.V().Has("Docker.ContainerName", "test-skydive-docker-strange-labels", "Type", "container", "Docker.Labels.a.b.c", "123", "Docker.Labels.a~b/c@d", "456")
gremlin := c.gremlin.V().Has("Name", "test-skydive-docker-strange-labels", "Type", "container", "Container.Labels.a.b.c", "123", "Container.Labels.a~b/c@d", "456")
_, err := c.gh.GetNode(gremlin)
return err
}, func(c *CheckContext) error {
gremlin := c.gremlin.V().Has("Docker.ContainerName", "test-skydive-docker-many-labels", "Type", "container", "Docker.Labels.label999", "1")
gremlin := c.gremlin.V().Has("Name", "test-skydive-docker-many-labels", "Type", "container", "Container.Labels.label999", "1")
_, err := c.gh.GetNode(gremlin)
return err
}},
Expand Down
2 changes: 1 addition & 1 deletion tests/runc_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func TestRuncSimple(t *testing.T) {

// Newer version of runc switched to bolt to store labels so
// we disable the test on labels for now
gremlin := c.gremlin.V().Has("Type", "container", "Manager", "runc", "Runc.ContainerID", containerID) // , "Runc.CreateConfig.Labels.Name", "test")
gremlin := c.gremlin.V().Has("Type", "container", "Manager", "runc", "Container.ID", containerID) // , "Container.Labels.Name", "test")
gremlin = gremlin.In("Type", "netns", "Manager", "runc")

nodes, err := c.gh.GetNodes(gremlin)
Expand Down
123 changes: 123 additions & 0 deletions topology/container.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
//go:generate go run github.com/skydive-project/skydive/graffiti/gendecoder -package github.com/skydive-project/skydive/topology
//go:generate go run github.com/mailru/easyjson/easyjson $GOFILE

/*
* Copyright (C) 2019 Red Hat, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy ofthe License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specificlanguage governing permissions and
* limitations under the License.
*
*/

package topology

import (
"bufio"
"encoding/json"
"fmt"
"net"
"os"
"strings"

"github.com/skydive-project/skydive/graffiti/getter"
"github.com/skydive-project/skydive/graffiti/graph"
)

// ContainerMetadata describe the metadata of a docker container
// easyjson:json
// gendecoder
type ContainerMetadata struct {
ID string
Image string `json:",omitempty"`
ImageID string `json:",omitempty"`
Runtime string
Status string
InitProcessPID int64
Hosts Hosts `json:",omitempty"`
Labels graph.Metadata `json:",omitempty" field:"Metadata"`
}

// Hosts describes a /etc/hosts file
// easyjson:json
// gendecoder
type Hosts struct {
IP string `json:",omitempty"`
Hostname string `json:",omitempty"`
ByIP graph.Metadata `json:",omitempty" field:"Metadata"`
}

func newHosts() *Hosts {
return &Hosts{ByIP: graph.Metadata{}}
}

// ReadHosts parses a 'hosts' file such as /etc/hosts
func ReadHosts(path string) (*Hosts, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()

hosts := newHosts()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()

if err := scanner.Err(); err != nil {
return nil, err
}

if i := strings.IndexByte(line, '#'); i >= 0 {
// Discard comment.
line = line[0:i]
}

f := strings.Fields(line)
if len(f) < 2 {
continue
}

ip := net.ParseIP(f[0])
if ip == nil {
continue
}

if ip.IsLoopback() {
continue
}

hosts.IP = ip.String()
ips := make([]string, len(f))
for i := 1; i < len(f); i++ {
hosts.Hostname = strings.ToLower(f[i])
ips[i-1] = hosts.Hostname
}
hosts.ByIP[hosts.IP] = ips
}

return hosts, nil
}

// ContainerMetadataDecoder implements a json message raw decoder
func ContainerMetadataDecoder(raw json.RawMessage) (getter.Getter, error) {
var m ContainerMetadata
if err := json.Unmarshal(raw, &m); err != nil {
return nil, fmt.Errorf("unable to unmarshal container metadata %s: %s", string(raw), err)
}

return &m, nil
}

// RegisterContainer registers container metadata decoder
func RegisterContainer() {
graph.NodeMetadataDecoders["Container"] = ContainerMetadataDecoder
}
Loading

0 comments on commit a30d0e1

Please sign in to comment.