Skip to content

Commit d3aeda7

Browse files
authored
feat: [NPM] Restructure code and add deploy manifests (#1203)
1 parent 580c3e4 commit d3aeda7

34 files changed

+1201
-202
lines changed

Tiltfile

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
allow_k8s_contexts(k8s_context())
2+
default_registry('ttl.sh/nitishm-12390')
3+
docker_build('azure-npm', '.', dockerfile='npm/Dockerfile', build_args = {
4+
"VERSION": "v1.4.14-101-gf900e319-dirty",
5+
"NPM_AI_PATH": "github.com/Azure/azure-container-networking/npm.aiMetadata",
6+
"NPM_AI_ID": "014c22bd-4107-459e-8475-67909e96edcb"
7+
})
8+
# watch_file('npm')
9+
k8s_yaml('npm/deploy/manifests/controller/azure-npm.yaml')
10+
k8s_yaml('npm/deploy/manifests/daemon/azure-npm.yaml', allow_duplicates=True)
11+

docs/npm.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ Azure-NPM serves as a distributed firewall for the Kubernetes cluster, and it ca
1212

1313
Running the command below will bring up one azure-npm instance on each Kubernetes node.
1414
```
15-
kubectl apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/azure-npm.yaml
15+
kubectl apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/deploy/npm/azure-npm.yaml
1616
```
1717
Now you can secure your Kubernetes cluster with Azure-NPM by applying Kubernetes network policies.
1818

npm/azure-npm.yaml

+3-3
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,10 @@ spec:
9797
- name: NPM_CONFIG
9898
value: /etc/azure-npm/azure-npm.json
9999
volumeMounts:
100-
- name: xtables-lock
101-
mountPath: /run/xtables.lock
102100
- name: log
103101
mountPath: /var/log
102+
- name: xtables-lock
103+
mountPath: /run/xtables.lock
104104
- name: protocols
105105
mountPath: /etc/protocols
106106
- name: azure-npm-config
@@ -156,4 +156,4 @@ data:
156156
"EnableV2NPM": false,
157157
"PlaceAzureChainFirst": false
158158
}
159-
}
159+
}

npm/cmd/root.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,13 @@ func NewRootCmd() *cobra.Command {
1414
},
1515
}
1616

17-
rootCmd.AddCommand(newStartNPMCmd())
17+
startCmd := newStartNPMCmd()
18+
19+
startCmd.AddCommand(newStartNPMControlplaneCmd())
20+
startCmd.AddCommand(newStartNPMDaemonCmd())
21+
22+
rootCmd.AddCommand(startCmd)
23+
1824
rootCmd.AddCommand(newDebugCmd())
1925

2026
return rootCmd

npm/cmd/start.go

+2-6
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"github.com/Azure/azure-container-networking/npm/pkg/dataplane"
1919
"github.com/Azure/azure-container-networking/npm/pkg/dataplane/ipsets"
2020
"github.com/Azure/azure-container-networking/npm/pkg/dataplane/policies"
21+
"github.com/Azure/azure-container-networking/npm/pkg/models"
2122
"github.com/Azure/azure-container-networking/npm/util"
2223
"github.com/spf13/cobra"
2324
"github.com/spf13/viper"
@@ -85,11 +86,6 @@ func newStartNPMCmd() *cobra.Command {
8586

8687
startNPMCmd.Flags().String(flagKubeConfigPath, flagDefaults[flagKubeConfigPath], "path to kubeconfig")
8788

88-
// The controlplane subcommand starts the NPM controller's controlplane component in the decomposed mode
89-
startNPMCmd.AddCommand(newStartNPMControlplaneCmd())
90-
// The daemon subcommand starts the NPM controller's datapath component in the daemon mode
91-
startNPMCmd.AddCommand(newStartNPMDaemonCmd())
92-
9389
return startNPMCmd
9490
}
9591

@@ -149,7 +145,7 @@ func start(config npmconfig.Config, flags npmconfig.Flags) error {
149145
} else {
150146
npmV2DataplaneCfg.IPSetMode = ipsets.ApplyAllIPSets
151147
}
152-
dp, err = dataplane.NewDataPlane(npm.GetNodeName(), common.NewIOShim(), npmV2DataplaneCfg, stopChannel)
148+
dp, err = dataplane.NewDataPlane(models.GetNodeName(), common.NewIOShim(), npmV2DataplaneCfg, stopChannel)
153149
if err != nil {
154150
return fmt.Errorf("failed to create dataplane with error %w", err)
155151
}

npm/cmd/start_daemon.go

+5-4
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,11 @@ import (
77
"strconv"
88

99
"github.com/Azure/azure-container-networking/common"
10-
"github.com/Azure/azure-container-networking/npm"
1110
npmconfig "github.com/Azure/azure-container-networking/npm/config"
11+
"github.com/Azure/azure-container-networking/npm/daemon"
1212
"github.com/Azure/azure-container-networking/npm/pkg/controlplane/goalstateprocessor"
1313
"github.com/Azure/azure-container-networking/npm/pkg/dataplane"
14+
"github.com/Azure/azure-container-networking/npm/pkg/models"
1415
"github.com/Azure/azure-container-networking/npm/pkg/transport"
1516
"github.com/spf13/cobra"
1617
"github.com/spf13/viper"
@@ -48,7 +49,7 @@ func startDaemon(config npmconfig.Config) error {
4849
pod := os.Getenv(podNameEnv)
4950
node := os.Getenv(nodeNameEnv)
5051

51-
addr := config.Transport.Address + ":" + strconv.Itoa(config.Transport.Port)
52+
addr := config.Transport.Address + ":" + strconv.Itoa(config.Transport.ServicePort)
5253
ctx := context.Background()
5354
err := initLogging()
5455
if err != nil {
@@ -58,7 +59,7 @@ func startDaemon(config npmconfig.Config) error {
5859

5960
var dp dataplane.GenericDataplane
6061

61-
dp, err = dataplane.NewDataPlane(npm.GetNodeName(), common.NewIOShim(), npmV2DataplaneCfg, wait.NeverStop)
62+
dp, err = dataplane.NewDataPlane(models.GetNodeName(), common.NewIOShim(), npmV2DataplaneCfg, wait.NeverStop)
6263
if err != nil {
6364
klog.Errorf("failed to create dataplane: %v", err)
6465
return fmt.Errorf("failed to create dataplane with error %w", err)
@@ -76,7 +77,7 @@ func startDaemon(config npmconfig.Config) error {
7677
return fmt.Errorf("failed to create goalstate processor: %w", err)
7778
}
7879

79-
n, err := npm.NewNetworkPolicyDaemon(ctx, config, dp, gsp, client, version)
80+
n, err := daemon.NewNetworkPolicyDaemon(ctx, config, dp, gsp, client, version)
8081
if err != nil {
8182
klog.Errorf("failed to create dataplane : %v", err)
8283
return fmt.Errorf("failed to create dataplane: %w", err)

npm/cmd/start_server.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"github.com/Azure/azure-container-networking/npm"
1010
npmconfig "github.com/Azure/azure-container-networking/npm/config"
11+
"github.com/Azure/azure-container-networking/npm/controller"
1112
restserver "github.com/Azure/azure-container-networking/npm/http/server"
1213
"github.com/Azure/azure-container-networking/npm/metrics"
1314
"github.com/Azure/azure-container-networking/npm/pkg/dataplane"
@@ -105,7 +106,7 @@ func startControlplane(config npmconfig.Config, flags npmconfig.Flags) error {
105106
return fmt.Errorf("failed to create dataplane with error: %w", err)
106107
}
107108

108-
npMgr, err := npm.NewNetworkPolicyServer(config, factory, mgr, dp, version, k8sServerVersion)
109+
npMgr, err := controller.NewNetworkPolicyServer(config, factory, mgr, dp, version, k8sServerVersion)
109110
if err != nil {
110111
klog.Errorf("failed to create NPM controlplane manager with error: %v", err)
111112
return fmt.Errorf("failed to create NPM controlplane manager: %w", err)

npm/config/config.go

+9-5
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
package npmconfig
22

33
const (
4-
defaultResyncPeriod = 15
5-
defaultListeningPort = 10091
6-
defaultGrpcPort = 10092
4+
defaultResyncPeriod = 15
5+
defaultListeningPort = 10091
6+
defaultGrpcPort = 10092
7+
defaultGrpcServicePort = 9002
78
// ConfigEnvPath is what's used by viper to load config path
89
ConfigEnvPath = "NPM_CONFIG"
910
)
@@ -16,8 +17,9 @@ var DefaultConfig = Config{
1617
ListeningAddress: "0.0.0.0",
1718

1819
Transport: GrpcServerConfig{
19-
Address: "0.0.0.0",
20-
Port: defaultGrpcPort,
20+
Address: "0.0.0.0",
21+
Port: defaultGrpcPort,
22+
ServicePort: defaultGrpcServicePort,
2123
},
2224

2325
Toggles: Toggles{
@@ -35,6 +37,8 @@ type GrpcServerConfig struct {
3537
Address string `json:"Address,omitempty"`
3638
// Port is the port on which the gRPC server will listen
3739
Port int `json:"Port,omitempty"`
40+
// ServicePort is the service port for the client to connect to the gRPC server
41+
ServicePort int `json:"ServicePort,omitempty"`
3842
}
3943

4044
type Config struct {

npm/server.go renamed to npm/controller/server.go

+46-43
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// Copyright 2018 Microsoft. All rights reserved.
22
// MIT License
3-
package npm
3+
package controller
44

55
import (
66
"encoding/json"
@@ -9,6 +9,7 @@ import (
99
npmconfig "github.com/Azure/azure-container-networking/npm/config"
1010
controllersv2 "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/v2"
1111
"github.com/Azure/azure-container-networking/npm/pkg/dataplane"
12+
"github.com/Azure/azure-container-networking/npm/pkg/models"
1213
"github.com/Azure/azure-container-networking/npm/pkg/transport"
1314
"github.com/pkg/errors"
1415
"k8s.io/apimachinery/pkg/version"
@@ -17,6 +18,8 @@ import (
1718
"k8s.io/klog"
1819
)
1920

21+
var aiMetadata string //nolint // aiMetadata is set in Makefile
22+
2023
type NetworkPolicyServer struct {
2124
config npmconfig.Config
2225

@@ -25,20 +28,20 @@ type NetworkPolicyServer struct {
2528

2629
// Informers are the Kubernetes Informer
2730
// https://pkg.go.dev/k8s.io/client-go/informers
28-
Informers
31+
models.Informers
2932

3033
// Controllers for handling Kubernetes resource watcher events
31-
K8SControllersV2
34+
models.K8SControllersV2
3235

3336
// Azure-specific variables
34-
AzureConfig
37+
models.AzureConfig
3538
}
3639

3740
var (
38-
ErrInformerFactoryNil = errors.New("informer factory is nil")
39-
ErrTransportManagerNil = errors.New("transport manager is nil")
40-
ErrK8SServerVersionNil = errors.New("k8s server version is nil")
41-
ErrInformerSyncFailure = errors.New("informer sync failure")
41+
ErrInformerFactoryNil = errors.New("informer factory is nil")
42+
ErrTransportManagerNil = errors.New("transport manager is nil")
43+
ErrK8SServerVersionNil = errors.New("k8s server version is nil")
44+
ErrDataplaneNotInitialized = errors.New("dataplane is not initialized")
4245
)
4346

4447
func NewNetworkPolicyServer(
@@ -70,87 +73,87 @@ func NewNetworkPolicyServer(
7073
n := &NetworkPolicyServer{
7174
config: config,
7275
tm: mgr,
73-
Informers: Informers{
74-
informerFactory: informerFactory,
75-
podInformer: informerFactory.Core().V1().Pods(),
76-
nsInformer: informerFactory.Core().V1().Namespaces(),
77-
npInformer: informerFactory.Networking().V1().NetworkPolicies(),
76+
Informers: models.Informers{
77+
InformerFactory: informerFactory,
78+
PodInformer: informerFactory.Core().V1().Pods(),
79+
NsInformer: informerFactory.Core().V1().Namespaces(),
80+
NpInformer: informerFactory.Networking().V1().NetworkPolicies(),
7881
},
79-
AzureConfig: AzureConfig{
80-
k8sServerVersion: k8sServerVersion,
81-
NodeName: GetNodeName(),
82-
version: npmVersion,
82+
AzureConfig: models.AzureConfig{
83+
K8sServerVersion: k8sServerVersion,
84+
NodeName: models.GetNodeName(),
85+
Version: npmVersion,
8386
TelemetryEnabled: true,
8487
},
8588
}
8689

87-
n.npmNamespaceCacheV2 = &controllersv2.NpmNamespaceCache{NsMap: make(map[string]*controllersv2.Namespace)}
88-
n.podControllerV2 = controllersv2.NewPodController(n.podInformer, dp, n.npmNamespaceCacheV2)
89-
n.namespaceControllerV2 = controllersv2.NewNamespaceController(n.nsInformer, dp, n.npmNamespaceCacheV2)
90-
n.netPolControllerV2 = controllersv2.NewNetworkPolicyController(n.npInformer, dp)
90+
n.NpmNamespaceCacheV2 = &controllersv2.NpmNamespaceCache{NsMap: make(map[string]*controllersv2.Namespace)}
91+
n.PodControllerV2 = controllersv2.NewPodController(n.PodInformer, dp, n.NpmNamespaceCacheV2)
92+
n.NamespaceControllerV2 = controllersv2.NewNamespaceController(n.NsInformer, dp, n.NpmNamespaceCacheV2)
93+
n.NetPolControllerV2 = controllersv2.NewNetworkPolicyController(n.NpInformer, dp)
9194

9295
return n, nil
9396
}
9497

9598
func (n *NetworkPolicyServer) MarshalJSON() ([]byte, error) {
96-
m := map[CacheKey]json.RawMessage{}
99+
m := map[models.CacheKey]json.RawMessage{}
97100

98101
var npmNamespaceCacheRaw []byte
99102
var err error
100-
npmNamespaceCacheRaw, err = json.Marshal(n.npmNamespaceCacheV2)
103+
npmNamespaceCacheRaw, err = json.Marshal(n.NpmNamespaceCacheV2)
101104

102105
if err != nil {
103-
return nil, errors.Errorf("%s: %v", errMarshalNPMCache, err)
106+
return nil, errors.Errorf("%s: %v", models.ErrMarshalNPMCache, err)
104107
}
105-
m[NsMap] = npmNamespaceCacheRaw
108+
m[models.NsMap] = npmNamespaceCacheRaw
106109

107110
var podControllerRaw []byte
108-
podControllerRaw, err = json.Marshal(n.podControllerV2)
111+
podControllerRaw, err = json.Marshal(n.PodControllerV2)
109112

110113
if err != nil {
111-
return nil, errors.Errorf("%s: %v", errMarshalNPMCache, err)
114+
return nil, errors.Errorf("%s: %v", models.ErrMarshalNPMCache, err)
112115
}
113-
m[PodMap] = podControllerRaw
116+
m[models.PodMap] = podControllerRaw
114117

115118
nodeNameRaw, err := json.Marshal(n.NodeName)
116119
if err != nil {
117-
return nil, errors.Errorf("%s: %v", errMarshalNPMCache, err)
120+
return nil, errors.Errorf("%s: %v", models.ErrMarshalNPMCache, err)
118121
}
119-
m[NodeName] = nodeNameRaw
122+
m[models.NodeName] = nodeNameRaw
120123

121124
npmCacheRaw, err := json.Marshal(m)
122125
if err != nil {
123-
return nil, errors.Errorf("%s: %v", errMarshalNPMCache, err)
126+
return nil, errors.Errorf("%s: %v", models.ErrMarshalNPMCache, err)
124127
}
125128

126129
return npmCacheRaw, nil
127130
}
128131

129132
func (n *NetworkPolicyServer) GetAppVersion() string {
130-
return n.version
133+
return n.Version
131134
}
132135

133136
func (n *NetworkPolicyServer) Start(config npmconfig.Config, stopCh <-chan struct{}) error {
134-
// Starts all informers manufactured by n's informerFactory.
135-
n.informerFactory.Start(stopCh)
137+
// Starts all informers manufactured by n's InformerFactory.
138+
n.InformerFactory.Start(stopCh)
136139

137140
// Wait for the initial sync of local cache.
138-
if !cache.WaitForCacheSync(stopCh, n.podInformer.Informer().HasSynced) {
139-
return fmt.Errorf("Pod informer error: %w", ErrInformerSyncFailure)
141+
if !cache.WaitForCacheSync(stopCh, n.PodInformer.Informer().HasSynced) {
142+
return fmt.Errorf("Pod informer error: %w", models.ErrInformerSyncFailure)
140143
}
141144

142-
if !cache.WaitForCacheSync(stopCh, n.nsInformer.Informer().HasSynced) {
143-
return fmt.Errorf("Namespace informer error: %w", ErrInformerSyncFailure)
145+
if !cache.WaitForCacheSync(stopCh, n.NsInformer.Informer().HasSynced) {
146+
return fmt.Errorf("Namespace informer error: %w", models.ErrInformerSyncFailure)
144147
}
145148

146-
if !cache.WaitForCacheSync(stopCh, n.npInformer.Informer().HasSynced) {
147-
return fmt.Errorf("NetworkPolicy informer error: %w", ErrInformerSyncFailure)
149+
if !cache.WaitForCacheSync(stopCh, n.NpInformer.Informer().HasSynced) {
150+
return fmt.Errorf("NetworkPolicy informer error: %w", models.ErrInformerSyncFailure)
148151
}
149152

150153
// start v2 NPM controllers after synced
151-
go n.podControllerV2.Run(stopCh)
152-
go n.namespaceControllerV2.Run(stopCh)
153-
go n.netPolControllerV2.Run(stopCh)
154+
go n.PodControllerV2.Run(stopCh)
155+
go n.NamespaceControllerV2.Run(stopCh)
156+
go n.NetPolControllerV2.Run(stopCh)
154157

155158
// start the transport layer (gRPC) server
156159
// We block the main thread here until the server is stopped.

npm/daemon.go renamed to npm/daemon/daemon.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
// Copyright 2018 Microsoft. All rights reserved.
22
// MIT License
3-
package npm
3+
package daemon
44

55
import (
66
"context"
7+
"errors"
78
"fmt"
89

910
npmconfig "github.com/Azure/azure-container-networking/npm/config"
@@ -12,6 +13,10 @@ import (
1213
"github.com/Azure/azure-container-networking/npm/pkg/transport"
1314
)
1415

16+
var aiMetadata string //nolint // aiMetadata is set in Makefile
17+
18+
var ErrDataplaneNotInitialized = errors.New("dataplane is not initialized")
19+
1520
type NetworkPolicyDaemon struct {
1621
ctx context.Context
1722
config npmconfig.Config

0 commit comments

Comments
 (0)