Skip to content

Commit 0cc2f98

Browse files
dioss-Machieldioss-Machiel
committed
Update config parser to allow setting a weight per gateway
1 parent 1eaabb6 commit 0cc2f98

File tree

4 files changed

+165
-68
lines changed

4 files changed

+165
-68
lines changed

‎examples/config.yml

+22-1
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,28 @@ tun:
234234

235235
# Unsafe routes allows you to route traffic over nebula to non-nebula nodes
236236
# Unsafe routes should be avoided unless you have hosts/services that cannot run nebula
237-
# NOTE: The nebula certificate of the "via" node *MUST* have the "route" defined as a subnet in its certificate
237+
# Supports weighted ECMP if you define a list of gateways, this can be used for load balancing or redundancy to hosts outside of nebula
238+
# NOTES:
239+
# * You will only see a single gateway in the routing table if you are not on linux
240+
# * If a gateway is not reachable through the overlay another gateway will be selected to send the traffic through, ignoring weights
241+
#
242+
# unsafe_routes:
243+
# # Multiple gateways without defining a weight defaults to a weight of 1, this will balance traffic equally between the three gateways
244+
# - route: 192.168.87.0/24
245+
# via:
246+
# - gateway: 10.0.0.1
247+
# - gateway: 10.0.0.2
248+
# - gateway: 10.0.0.3
249+
# # Multiple gateways with a weight, this will balance traffic accordingly
250+
# - route: 192.168.87.0/24
251+
# via:
252+
# - gateway: 10.0.0.1
253+
# weight: 10
254+
# - gateway: 10.0.0.2
255+
# weight: 5
256+
#
257+
# NOTE: The nebula certificate of the "via" node(s) *MUST* have the "route" defined as a subnet in its certificate
258+
# `via`: single node or list of gateways to use for this route
238259
# `mtu`: will default to tun mtu if this option is not specified
239260
# `metric`: will default to 0 if this option is not specified
240261
# `install`: will default to true, controls whether this route is installed in the systems routing table.

‎overlay/route.go

+54-27
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ type Route struct {
1818
MTU int
1919
Metric int
2020
Cidr netip.Prefix
21-
Via []netip.Addr
21+
Via []routing.Gateway
2222
Install bool
2323
}
2424

@@ -55,14 +55,7 @@ func makeRouteTree(l *logrus.Logger, routes []Route, allowMTU bool) (*bart.Table
5555
l.WithField("route", r).Warnf("route MTU is not supported in %s", runtime.GOOS)
5656
}
5757

58-
gateways := []routing.Gateway{}
59-
60-
for _, via := range r.Via {
61-
if via.IsValid() {
62-
gateways = append(gateways, routing.NewGateway(via, 1))
63-
}
64-
}
65-
58+
gateways := r.Via
6659
if len(gateways) > 0 {
6760
routing.RebalanceGateways(gateways)
6861
routeTree.Insert(r.Cidr, gateways)
@@ -211,29 +204,63 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
211204
return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes is not present", i+1)
212205
}
213206

214-
var viasInConfig = []string{}
215-
via, isSingleVia := rVia.(string)
207+
var gateways []routing.Gateway
216208

217-
if !isSingleVia {
218-
multiVia, isMultiVia := rVia.([]string)
219-
220-
if !isMultiVia {
221-
return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes is not a string or array of string: found %T", i+1, rVia)
209+
switch via := rVia.(type) {
210+
case string:
211+
viaIp, err := netip.ParseAddr(via)
212+
if err != nil {
213+
return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes failed to parse address: %v", i+1, err)
222214
}
223215

224-
viasInConfig = append(viasInConfig, multiVia...)
225-
} else {
226-
viasInConfig = append(viasInConfig, via)
227-
}
216+
gateways = []routing.Gateway{routing.NewGateway(viaIp, 1)}
217+
218+
case []interface{}:
219+
gateways = make([]routing.Gateway, len(via))
220+
for ig, v := range via {
221+
gatewayMap, ok := v.(map[interface{}]interface{})
222+
if !ok {
223+
return nil, fmt.Errorf("entry %v in tun.unsafe_routes[%v].via is invalid", i+1, ig+1)
224+
}
225+
226+
rGateway, ok := gatewayMap["gateway"]
227+
if !ok {
228+
return nil, fmt.Errorf("entry .gateway in tun.unsafe_routes[%v].via[%v] is not present", i+1, ig+1)
229+
}
230+
231+
parsedGateway, ok := rGateway.(string)
232+
if !ok {
233+
return nil, fmt.Errorf("entry .gateway in tun.unsafe_routes[%v].via[%v] is not a string", i+1, ig+1)
234+
}
235+
236+
gatewayIp, err := netip.ParseAddr(parsedGateway)
237+
if err != nil {
238+
return nil, fmt.Errorf("entry .gateway in tun.unsafe_routes[%v].via[%v] failed to parse address: %v", i+1, ig+1, err)
239+
}
240+
241+
rGatewayWeight, ok := gatewayMap["weight"]
242+
if !ok {
243+
rGatewayWeight = 1
244+
}
245+
246+
gatewayWeight, ok := rGatewayWeight.(int)
247+
if !ok {
248+
_, err = strconv.ParseInt(rGatewayWeight.(string), 10, 32)
249+
if err != nil {
250+
return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1)
251+
}
252+
}
253+
254+
if gatewayWeight < 1 || gatewayWeight > math.MaxInt32 {
255+
return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not in range (1-%d) : %v", i+1, ig+1, math.MaxInt32, metric)
256+
}
257+
258+
gateways[ig] = routing.NewGateway(gatewayIp, gatewayWeight)
228259

229-
var parsedVias = []netip.Addr{}
230-
for _, viaString := range viasInConfig {
231-
viaVpnIp, err := netip.ParseAddr(viaString)
232-
if err != nil {
233-
return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes failed to parse address: %v", i+1, err)
234260
}
235261

236-
parsedVias = append(parsedVias, viaVpnIp)
262+
default:
263+
return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes is not a string or list of gateways: found %T", i+1, rVia)
237264
}
238265

239266
rRoute, ok := m["route"]
@@ -251,7 +278,7 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
251278
}
252279

253280
r := Route{
254-
Via: parsedVias,
281+
Via: gateways,
255282
MTU: mtu,
256283
Metric: metric,
257284
Install: install,

‎overlay/route_test.go

+87-38
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package overlay
33
import (
44
"fmt"
55
"net/netip"
6-
"reflect"
76
"testing"
87

98
"github.com/slackhq/nebula/config"
@@ -155,26 +154,44 @@ func Test_parseUnsafeRoutes(t *testing.T) {
155154

156155
// invalid via
157156
for _, invalidValue := range []interface{}{
158-
127, false, nil, 1.0,
157+
127, false, nil, 1.0, []string{"1", "2"},
159158
} {
160159
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": invalidValue}}}
161160
routes, err = parseUnsafeRoutes(c, []netip.Prefix{n})
162161
assert.Nil(t, routes)
163-
require.EqualError(t, err, fmt.Sprintf("entry 1.via in tun.unsafe_routes is not a string or array of string: found %T", invalidValue))
162+
require.EqualError(t, err, fmt.Sprintf("entry 1.via in tun.unsafe_routes is not a string or list of gateways: found %T", invalidValue))
164163
}
165164

166165
// Unparsable list of via
167166
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": []string{"1", "2"}}}}
168167
routes, err = parseUnsafeRoutes(c, []netip.Prefix{n})
169168
assert.Nil(t, routes)
170-
assert.EqualError(t, err, fmt.Sprintf("entry 1.via in tun.unsafe_routes failed to parse address: ParseAddr(\"1\"): unable to parse IP"))
169+
require.EqualError(t, err, "entry 1.via in tun.unsafe_routes is not a string or list of gateways: found []string")
171170

172171
// unparsable via
173172
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"mtu": "500", "via": "nope"}}}
174173
routes, err = parseUnsafeRoutes(c, []netip.Prefix{n})
175174
assert.Nil(t, routes)
176175
require.EqualError(t, err, "entry 1.via in tun.unsafe_routes failed to parse address: ParseAddr(\"nope\"): unable to parse IP")
177176

177+
// unparsable gateway
178+
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"mtu": "500", "via": []interface{}{map[interface{}]interface{}{"gateway": "1"}}}}}
179+
routes, err = parseUnsafeRoutes(c, []netip.Prefix{n})
180+
assert.Nil(t, routes)
181+
require.EqualError(t, err, "entry .gateway in tun.unsafe_routes[1].via[1] failed to parse address: ParseAddr(\"1\"): unable to parse IP")
182+
183+
// missing gateway element
184+
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"mtu": "500", "via": []interface{}{map[interface{}]interface{}{"weight": "1"}}}}}
185+
routes, err = parseUnsafeRoutes(c, []netip.Prefix{n})
186+
assert.Nil(t, routes)
187+
require.EqualError(t, err, "entry .gateway in tun.unsafe_routes[1].via[1] is not present")
188+
189+
// unparsable weight element
190+
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"mtu": "500", "via": []interface{}{map[interface{}]interface{}{"gateway": "10.0.0.1", "weight": "a"}}}}}
191+
routes, err = parseUnsafeRoutes(c, []netip.Prefix{n})
192+
assert.Nil(t, routes)
193+
require.EqualError(t, err, "entry .weight in tun.unsafe_routes[1].via[1] is not an integer")
194+
178195
// missing route
179196
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{map[interface{}]interface{}{"via": "127.0.0.1", "mtu": "500"}}}
180197
routes, err = parseUnsafeRoutes(c, []netip.Prefix{n})
@@ -305,51 +322,83 @@ func Test_makeRouteTree(t *testing.T) {
305322
assert.False(t, ok)
306323
}
307324

308-
func sameElements[T comparable](arr1 []T, arr2 []T) bool {
309-
if len(arr1) != len(arr2) {
310-
return false
311-
}
312-
313-
counts1 := make(map[T]int)
314-
counts2 := make(map[T]int)
315-
316-
for _, v := range arr1 {
317-
counts1[v]++
318-
}
319-
for _, v := range arr2 {
320-
counts2[v]++
321-
}
322-
323-
return reflect.DeepEqual(counts1, counts2)
324-
}
325-
326-
func Test_multipath(t *testing.T) {
325+
func Test_makeMultipathUnsafeRouteTree(t *testing.T) {
327326
l := test.NewLogger()
328327
c := config.NewC(l)
329328
n, err := netip.ParsePrefix("10.0.0.0/24")
330-
assert.NoError(t, err)
329+
require.NoError(t, err)
331330

332-
c.Settings["tun"] = map[interface{}]interface{}{"unsafe_routes": []interface{}{
333-
map[interface{}]interface{}{"via": []string{"192.168.0.1", "192.168.0.2", "192.168.0.3"}, "route": "1.0.0.0/16"},
334-
}}
331+
c.Settings["tun"] = map[interface{}]interface{}{
332+
"unsafe_routes": []interface{}{
333+
map[interface{}]interface{}{
334+
"route": "192.168.86.0/24",
335+
"via": "192.168.100.10",
336+
},
337+
map[interface{}]interface{}{
338+
"route": "192.168.87.0/24",
339+
"via": []interface{}{
340+
map[interface{}]interface{}{
341+
"gateway": "10.0.0.1",
342+
},
343+
map[interface{}]interface{}{
344+
"gateway": "10.0.0.2",
345+
},
346+
map[interface{}]interface{}{
347+
"gateway": "10.0.0.3",
348+
},
349+
},
350+
},
351+
map[interface{}]interface{}{
352+
"route": "192.168.89.0/24",
353+
"via": []interface{}{
354+
map[interface{}]interface{}{
355+
"gateway": "10.0.0.1",
356+
"weight": 10,
357+
},
358+
map[interface{}]interface{}{
359+
"gateway": "10.0.0.2",
360+
"weight": 5,
361+
},
362+
},
363+
},
364+
},
365+
}
335366

336367
routes, err := parseUnsafeRoutes(c, []netip.Prefix{n})
337-
assert.NoError(t, err)
338-
assert.Len(t, routes, 1)
368+
require.NoError(t, err)
369+
assert.Len(t, routes, 3)
339370
routeTree, err := makeRouteTree(l, routes, true)
340-
assert.NoError(t, err)
371+
require.NoError(t, err)
341372

342-
ip, err := netip.ParseAddr("1.0.0.2")
343-
assert.NoError(t, err)
373+
ip, err := netip.ParseAddr("192.168.86.1")
374+
require.NoError(t, err)
344375
r, ok := routeTree.Lookup(ip)
345376
assert.True(t, ok)
346377

347-
nips := []routing.Gateway{}
348-
nips = append(nips, routing.NewGateway(netip.MustParseAddr("192.168.0.1"), 1))
349-
nips = append(nips, routing.NewGateway(netip.MustParseAddr("192.168.0.2"), 1))
350-
nips = append(nips, routing.NewGateway(netip.MustParseAddr("192.168.0.3"), 1))
378+
nip, err := netip.ParseAddr("192.168.100.10")
379+
require.NoError(t, err)
380+
assert.Equal(t, nip, r[0].Ip())
381+
382+
ip, err = netip.ParseAddr("192.168.87.1")
383+
require.NoError(t, err)
384+
r, ok = routeTree.Lookup(ip)
385+
assert.True(t, ok)
386+
387+
expectedGateways := []routing.Gateway{routing.NewGateway(netip.MustParseAddr("10.0.0.1"), 1),
388+
routing.NewGateway(netip.MustParseAddr("10.0.0.2"), 1),
389+
routing.NewGateway(netip.MustParseAddr("10.0.0.3"), 1)}
390+
391+
routing.RebalanceGateways(expectedGateways)
392+
assert.ElementsMatch(t, expectedGateways, r)
393+
394+
ip, err = netip.ParseAddr("192.168.89.1")
395+
require.NoError(t, err)
396+
r, ok = routeTree.Lookup(ip)
397+
assert.True(t, ok)
351398

352-
routing.RebalanceGateways(nips)
399+
expectedGateways = []routing.Gateway{routing.NewGateway(netip.MustParseAddr("10.0.0.1"), 10),
400+
routing.NewGateway(netip.MustParseAddr("10.0.0.2"), 5)}
353401

354-
assert.ElementsMatch(t, nips, r)
402+
routing.RebalanceGateways(expectedGateways)
403+
assert.ElementsMatch(t, expectedGateways, r)
355404
}

‎overlay/tun_windows.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ func (t *winTun) addRoutes(logErrors bool) error {
157157
// Windows does not support multipath routes natively, so we install only a single route.
158158
// This is not a problem as traffic will always be sent to Nebula which handles the multipath routing internally.
159159
// In effect this provides multipath routing support to windows supporting loadbalancing and redundancy.
160-
err := luid.AddRoute(r.Cidr, r.Via[0], uint32(r.Metric))
160+
err := luid.AddRoute(r.Cidr, r.Via[0].Ip(), uint32(r.Metric))
161161
if err != nil {
162162
retErr := util.NewContextualError("Failed to add route", map[string]interface{}{"route": r}, err)
163163
if logErrors {
@@ -203,7 +203,7 @@ func (t *winTun) removeRoutes(routes []Route) error {
203203
}
204204

205205
// See comment on luid.AddRoute
206-
err := luid.DeleteRoute(r.Cidr, r.Via[0])
206+
err := luid.DeleteRoute(r.Cidr, r.Via[0].Ip())
207207
if err != nil {
208208
t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
209209
} else {

0 commit comments

Comments
 (0)