Skip to content
This repository was archived by the owner on Feb 8, 2021. It is now read-only.

Commit 5453b70

Browse files
committed
re-enable the in sandbox portmapping with exec in sandbox
make it more flexible, update without modify hyperstart protocol Signed-off-by: Wang Xu <[email protected]>
1 parent a530ca1 commit 5453b70

File tree

7 files changed

+249
-26
lines changed

7 files changed

+249
-26
lines changed

daemon/pod/pod.go

+2
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ type XPod struct {
6363
labels map[string]string
6464
resourceLock *sync.Mutex
6565

66+
prestartExecs [][]string
67+
6668
sandbox *hypervisor.Vm
6769
factory *PodFactory
6870

daemon/pod/portmappings.go

+49-4
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,20 @@ func (p *XPod) initPortMapping() error {
3131
hlog.Log(ERROR, err)
3232
return err
3333
}
34-
err = portmapping.SetupPortMaps(p.containerIP, pms)
34+
var extPrefix []string
35+
if p.globalSpec.PortmappingWhiteLists != nil &&
36+
len(p.globalSpec.PortmappingWhiteLists.InternalNetworks) > 0 &&
37+
len(p.globalSpec.PortmappingWhiteLists.ExternalNetworks) > 0 {
38+
extPrefix = p.globalSpec.PortmappingWhiteLists.ExternalNetworks
39+
}
40+
preExec, err := portmapping.SetupPortMaps(p.containerIP, extPrefix, pms)
3541
if err != nil {
3642
p.Log(ERROR, "failed to setup port mappings: %v", err)
3743
return err
3844
}
45+
if len(preExec) > 0 {
46+
p.prestartExecs = append(p.prestartExecs, preExec...)
47+
}
3948
}
4049
return nil
4150
}
@@ -47,7 +56,7 @@ func (p *XPod) flushPortMapping() error {
4756
hlog.Log(ERROR, err)
4857
return err
4958
}
50-
err = portmapping.ReleasePortMaps(p.containerIP, pms)
59+
_, err = portmapping.ReleasePortMaps(p.containerIP, nil, pms)
5160
if err != nil {
5261
p.Log(ERROR, "release port mappings failed: %v", err)
5362
return err
@@ -75,11 +84,29 @@ func (p *XPod) AddPortMapping(spec []*apitypes.PortMapping) error {
7584
p.Log(ERROR, "failed to generate port mapping rules: %v", err)
7685
return err
7786
}
78-
err = portmapping.SetupPortMaps(p.containerIP, pms)
87+
var extPrefix []string
88+
if p.globalSpec.PortmappingWhiteLists != nil &&
89+
len(p.globalSpec.PortmappingWhiteLists.InternalNetworks) > 0 &&
90+
len(p.globalSpec.PortmappingWhiteLists.ExternalNetworks) > 0 {
91+
extPrefix = p.globalSpec.PortmappingWhiteLists.ExternalNetworks
92+
}
93+
preExec, err := portmapping.SetupPortMaps(p.containerIP, extPrefix, pms)
7994
if err != nil {
8095
p.Log(ERROR, "failed to apply port mapping rules: %v", err)
8196
return err
8297
}
98+
if len(preExec) > 0 {
99+
p.prestartExecs = append(p.prestartExecs, preExec...)
100+
if p.sandbox != nil {
101+
for _, ex := range preExec {
102+
_, stderr, err := p.sandbox.HyperstartExecSync(ex, nil)
103+
if err != nil {
104+
p.Log(ERROR, "failed to setup inSandbox mapping: %v [ %s", err, string(stderr))
105+
return err
106+
}
107+
}
108+
}
109+
}
83110

84111
all := make([]*apitypes.PortMapping, len(p.portMappings)+len(spec))
85112
copy(all, spec)
@@ -134,11 +161,29 @@ func (p *XPod) removePortMapping(tbr []*apitypes.PortMapping, eq portMappingComp
134161
return err
135162
}
136163

137-
err = portmapping.ReleasePortMaps(p.containerIP, act)
164+
var extPrefix []string
165+
if p.globalSpec.PortmappingWhiteLists != nil &&
166+
len(p.globalSpec.PortmappingWhiteLists.InternalNetworks) > 0 &&
167+
len(p.globalSpec.PortmappingWhiteLists.ExternalNetworks) > 0 {
168+
extPrefix = p.globalSpec.PortmappingWhiteLists.ExternalNetworks
169+
}
170+
postExec, err := portmapping.ReleasePortMaps(p.containerIP, extPrefix, act)
138171
if err != nil {
139172
p.Log(ERROR, "failed to clean up rules: %v", err)
140173
return err
141174
}
175+
if len(postExec) > 0 {
176+
// don't need to release prestartExec here, it is not persistent
177+
if p.sandbox != nil {
178+
for _, ex := range postExec {
179+
_, stderr, err := p.sandbox.HyperstartExecSync(ex, nil)
180+
if err != nil {
181+
p.Log(ERROR, "failed to setup inSandbox mapping: %v [ %s", err, string(stderr))
182+
return err
183+
}
184+
}
185+
}
186+
}
142187

143188
p.portMappings = other
144189
err = p.savePortMapping()

daemon/pod/provision.go

+23-13
Original file line numberDiff line numberDiff line change
@@ -86,19 +86,20 @@ func newXPod(factory *PodFactory, spec *apitypes.UserPod) (*XPod, error) {
8686
factory.hosts = HostsCreator(spec.Id)
8787
factory.logCreator = initLogCreator(factory, spec)
8888
p := &XPod{
89-
name: spec.Id,
90-
logPrefix: fmt.Sprintf("Pod[%s] ", spec.Id),
91-
globalSpec: spec.CloneGlobalPart(),
92-
containers: make(map[string]*Container),
93-
volumes: make(map[string]*Volume),
94-
interfaces: make(map[string]*Interface),
95-
portMappings: spec.Portmappings,
96-
labels: spec.Labels,
97-
execs: make(map[string]*Exec),
98-
resourceLock: &sync.Mutex{},
99-
statusLock: &sync.RWMutex{},
100-
stoppedChan: make(chan bool, 1),
101-
factory: factory,
89+
name: spec.Id,
90+
logPrefix: fmt.Sprintf("Pod[%s] ", spec.Id),
91+
globalSpec: spec.CloneGlobalPart(),
92+
containers: make(map[string]*Container),
93+
volumes: make(map[string]*Volume),
94+
interfaces: make(map[string]*Interface),
95+
portMappings: spec.Portmappings,
96+
labels: spec.Labels,
97+
prestartExecs: [][]string{},
98+
execs: make(map[string]*Exec),
99+
resourceLock: &sync.Mutex{},
100+
statusLock: &sync.RWMutex{},
101+
stoppedChan: make(chan bool, 1),
102+
factory: factory,
102103
}
103104
p.initCond = sync.NewCond(p.statusLock.RLocker())
104105
return p, nil
@@ -484,6 +485,15 @@ func (p *XPod) startAll() error {
484485
p.Log(INFO, "start all containers")
485486
future := utils.NewFutureSet()
486487

488+
for _, pre := range p.prestartExecs {
489+
p.Log(DEBUG, "run prestart exec %v", pre)
490+
_, stderr, err := p.sandbox.HyperstartExecSync(pre, nil)
491+
if err != nil {
492+
p.Log(ERROR, "failed to execute prestart command %v: %v [ %s", pre, err, string(stderr))
493+
return err
494+
}
495+
}
496+
487497
for ic, c := range p.containers {
488498
future.Add(ic, c.start)
489499
}

examples/port-mapping-in-sandbox.pod

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"id": "port-mapping-test",
3+
"containers" : [{
4+
"name": "pmtest",
5+
"image": "busybox:latest",
6+
"command": ["/bin/nc", "-l", "-p", "1300"]
7+
}],
8+
"resource": {
9+
"vcpu": 1,
10+
"memory": 128
11+
},
12+
"portmappingWhiteLists":{
13+
"internalNetworks": ["127.0.0.1/32", "192.168.123.0/24"],
14+
"externalNetworks": ["0.0.0.0/0"]
15+
},
16+
"portmappings": [{
17+
"containerport": "1300-1310",
18+
"hostport": "3000-3010",
19+
"protocol": "tcp"
20+
}]
21+
}
22+

networking/portmapping/host_portmapping_linux.go

+2-9
Original file line numberDiff line numberDiff line change
@@ -130,11 +130,7 @@ func parseRawResultOnHyper(output []byte, err error) error {
130130
return nil
131131
}
132132

133-
func SetupPortMaps(containerip string, maps []*PortMapping) error {
134-
if disableIptables || len(maps) == 0 {
135-
return nil
136-
}
137-
133+
func setupIptablesPortMaps(containerip string, maps []*PortMapping) error {
138134
var (
139135
revert bool
140136
revertRules = [][]string{}
@@ -209,10 +205,7 @@ func SetupPortMaps(containerip string, maps []*PortMapping) error {
209205
return nil
210206
}
211207

212-
func ReleasePortMaps(containerip string, maps []*PortMapping) error {
213-
if disableIptables || len(maps) == 0 {
214-
return nil
215-
}
208+
func releaseIptablesPortMaps(containerip string, maps []*PortMapping) error {
216209

217210
release_loop:
218211
for _, m := range maps {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package portmapping
2+
3+
func SetupPortMaps(containerip string, externalPrefix []string, maps []*PortMapping) (preExec [][]string, err error) {
4+
if len(maps) == 0 {
5+
return [][]string{}, nil
6+
}
7+
if len(externalPrefix) > 0 {
8+
preExec, err = setupInSandboxMappings(externalPrefix, maps)
9+
if err != nil {
10+
return [][]string{}, err
11+
}
12+
defer func() {
13+
if err != nil {
14+
preExec, _ = releaseInSandboxMappings(externalPrefix, maps)
15+
}
16+
}()
17+
}
18+
if !disableIptables {
19+
err = setupIptablesPortMaps(containerip, maps)
20+
if err != nil {
21+
return [][]string{}, err
22+
}
23+
}
24+
return preExec, nil
25+
}
26+
27+
func ReleasePortMaps(containerip string, externalPrefix []string, maps []*PortMapping) (postExec [][]string, err error) {
28+
if len(maps) == 0 {
29+
return [][]string{}, nil
30+
}
31+
if len(externalPrefix) > 0 {
32+
postExec, err = releaseInSandboxMappings(externalPrefix, maps)
33+
if err != nil {
34+
return [][]string{}, err
35+
}
36+
}
37+
if !disableIptables {
38+
err = releaseIptablesPortMaps(containerip, maps)
39+
if err != nil {
40+
return [][]string{}, err
41+
}
42+
}
43+
return postExec, nil
44+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package portmapping
2+
3+
import (
4+
"fmt"
5+
"strconv"
6+
"strings"
7+
)
8+
9+
/* Inside hyperstart, there are iptables init job should be done before configure in sandbox rules.
10+
11+
// iptables -t filter -N hyperstart-INPUT
12+
// iptables -t nat -N hyperstart-PREROUTING
13+
// iptables -t filter -I INPUT -j hyperstart-INPUT
14+
// iptables -t nat -I PREROUTING -j hyperstart-PREROUTING
15+
// iptables -t filter -A hyperstart-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
16+
// iptables -t filter -A hyperstart-INPUT -p icmp -j ACCEPT
17+
// iptables -t filter -A hyperstart-INPUT -i lo -j ACCEPT
18+
// iptables -t filter -A hyperstart-INPUT -j DROP
19+
// iptables -t nat -A hyperstart-PREROUTING -j RETURN
20+
// sh -c "echo 10485760 > /proc/sys/net/nf_conntrack_max"
21+
// sh -c "echo 300 > /proc/sys/net/netfilter/nf_conntrack_tcp_timeout_established"
22+
23+
// lan
24+
// iptables -t filter -I hyperstart-INPUT -s %s -j ACCEPT
25+
26+
These job has been done by hyperstart during initSandbox.
27+
*/
28+
29+
func generateRedirectArgs(prefix string, m *PortMapping, insert bool) ([]string, []string, error) {
30+
// outside
31+
//iptables -t nat -I hyperstart-PREROUTING -s %s -p %s -m %s --dport %d -j REDIRECT --to-ports %d"
32+
//iptables -t filter -I hyperstart-INPUT -s %s -p %s -m %s --dport %d -j ACCEPT
33+
var (
34+
action = "-I"
35+
proto string
36+
dest string
37+
to string
38+
)
39+
40+
if !insert {
41+
action = "-D"
42+
}
43+
44+
if strings.EqualFold(m.Protocol, "udp") {
45+
proto = "udp"
46+
} else {
47+
proto = "tcp"
48+
}
49+
50+
if m.FromPorts.End == 0 || m.FromPorts.End == m.FromPorts.Begin {
51+
dest = strconv.Itoa(m.FromPorts.Begin)
52+
m.FromPorts.End = m.FromPorts.Begin
53+
} else if m.FromPorts.End > m.FromPorts.Begin {
54+
dest = fmt.Sprintf("%d:%d", m.FromPorts.Begin, m.FromPorts.End)
55+
} else {
56+
return []string{}, []string{}, fmt.Errorf("invalid from port range %d-%d", m.FromPorts.Begin, m.FromPorts.End)
57+
}
58+
59+
if m.ToPorts.End == 0 || m.ToPorts.End == m.ToPorts.Begin {
60+
to = strconv.Itoa(m.ToPorts.Begin)
61+
m.ToPorts.End = m.ToPorts.Begin
62+
} else if m.ToPorts.End > m.ToPorts.Begin {
63+
to = fmt.Sprintf("%d-%d", m.ToPorts.Begin, m.ToPorts.End)
64+
} else {
65+
return []string{}, []string{}, fmt.Errorf("invalid to port range %d-%d", m.ToPorts.Begin, m.ToPorts.End)
66+
}
67+
68+
//we may map ports 1:N or N:N, but not M:N (M!=1, M!=N)
69+
hostRange := m.FromPorts.End - m.FromPorts.Begin
70+
containerRange := m.ToPorts.End - m.ToPorts.Begin
71+
if hostRange != 0 && hostRange != containerRange {
72+
return []string{}, []string{}, fmt.Errorf("range mismatch, cannot map ports %s to %s", dest, to)
73+
}
74+
75+
filterArgs := []string{"iptables", "-t", "filter", action, "hyperstart-INPUT", "-s", prefix, "-p", proto, "-m", proto, "--dport", dest, "-j", "ACCEPT"}
76+
redirectArgs := []string{"iptables", "-t", "nat", action, "hyperstart-PREROUTING", "-s", prefix, "-p", proto, "-m", proto, "--dport", dest, "-j", "REDIRECT", "--to-port", to}
77+
78+
return redirectArgs, filterArgs, nil
79+
}
80+
81+
func setupInSandboxMappings(extPrefix []string, maps []*PortMapping) ([][]string, error) {
82+
res := [][]string{}
83+
for _, prefix := range extPrefix {
84+
for _, m := range maps {
85+
redirect, filter, err := generateRedirectArgs(prefix, m, true)
86+
if err != nil {
87+
return nil, err
88+
}
89+
res = append(res, redirect, filter)
90+
}
91+
}
92+
return res, nil
93+
}
94+
95+
func releaseInSandboxMappings(extPrefix []string, maps []*PortMapping) ([][]string, error) {
96+
res := [][]string{}
97+
for _, prefix := range extPrefix {
98+
for _, m := range maps {
99+
redirect, filter, err := generateRedirectArgs(prefix, m, false)
100+
if err != nil {
101+
return nil, err
102+
}
103+
res = append(res, redirect, filter)
104+
}
105+
}
106+
return res, nil
107+
}

0 commit comments

Comments
 (0)