m/node: enlarge K8s networks
For bigger clusters, the current 10.0.0.0/16 subnet is far too small.
Switch to 10.192.0.0/11 which should be out of the way of most of our
test infra and is large enough for 8192 nodes with 253 pods which is
big enough for the time being. Also migrate the service network
to 10.224.0.0/16 and make it much bigger. It does not need to be in the
pod CIDR, so move it out of there.
But for large clusters this will continue to be a problem until we have
a better allocation algorithm or switch to IPv6 with 464xlat (which
is not supported on Linux currently however).
Change-Id: Ib3a019fffacec2172721f04c01133b44bffba73b
Reviewed-on: https://review.monogon.dev/c/monogon/+/1848
Reviewed-by: Leopold Schabel <leo@monogon.tech>
Tested-by: Jenkins CI
diff --git a/metropolis/node/core/curator/impl_leader_cluster_networking.go b/metropolis/node/core/curator/impl_leader_cluster_networking.go
index 80da0d3..1427eb9 100644
--- a/metropolis/node/core/curator/impl_leader_cluster_networking.go
+++ b/metropolis/node/core/curator/impl_leader_cluster_networking.go
@@ -96,7 +96,7 @@
}
// TODO(q3k): unhardcode this and synchronize with Kubernetes code.
- clusterNet := netip.MustParsePrefix("10.0.0.0/16")
+ clusterNet := netip.MustParsePrefix("10.192.0.0/11")
// Retrieve node ...
node, err := nodeLoad(ctx, l.leadership, id)
diff --git a/metropolis/node/core/curator/impl_leader_test.go b/metropolis/node/core/curator/impl_leader_test.go
index a87c45e..5490b5b 100644
--- a/metropolis/node/core/curator/impl_leader_test.go
+++ b/metropolis/node/core/curator/impl_leader_test.go
@@ -1416,7 +1416,7 @@
Clusternet: &cpb.NodeClusterNetworking{
WireguardPubkey: "w9RbFvF14pytyraq16IEuMov032XXrPBOQUr59kcxHg=",
Prefixes: []*cpb.NodeClusterNetworking_Prefix{
- {Cidr: "10.0.0.128/16"},
+ {Cidr: "10.192.0.128/16"},
},
},
}, "must be in canonical format"},
@@ -1425,7 +1425,7 @@
WireguardPubkey: "w9RbFvF14pytyraq16IEuMov032XXrPBOQUr59kcxHg=",
Prefixes: []*cpb.NodeClusterNetworking_Prefix{
// Prefix outside of cluster net should not be allowed.
- {Cidr: "10.0.0.0/15"},
+ {Cidr: "10.192.0.0/10"},
},
},
}, "must be fully contained"},
@@ -1442,10 +1442,10 @@
Clusternet: &cpb.NodeClusterNetworking{
WireguardPubkey: "GaNXuc/yl8IaXduX6PQ+ZxIG4HtBACubHrRI7rqfA20=",
Prefixes: []*cpb.NodeClusterNetworking_Prefix{
- {Cidr: "10.0.0.0/24"},
+ {Cidr: "10.192.0.0/24"},
// Yes, this is allowed.
- {Cidr: "10.0.0.0/16"},
- {Cidr: "10.0.12.23/32"},
+ {Cidr: "10.192.0.0/11"},
+ {Cidr: "10.195.12.23/32"},
// External address should be allowed.
{Cidr: "203.0.113.43/32"},
},
@@ -1486,7 +1486,7 @@
if want, got := 4, len(cn.Prefixes); want != got {
t.Errorf("Wanted %d prefixes, got %d", want, got)
} else {
- for i, want := range []string{"10.0.0.0/24", "10.0.0.0/16", "10.0.12.23/32", "203.0.113.43/32"} {
+ for i, want := range []string{"10.192.0.0/24", "10.192.0.0/11", "10.195.12.23/32", "203.0.113.43/32"} {
if got := cn.Prefixes[i].Cidr; want != got {
t.Errorf("Prefix %d should be %q, got %q", i, want, got)
}
diff --git a/metropolis/node/core/roleserve/worker_clusternet.go b/metropolis/node/core/roleserve/worker_clusternet.go
index 2e56098..5ae6877 100644
--- a/metropolis/node/core/roleserve/worker_clusternet.go
+++ b/metropolis/node/core/roleserve/worker_clusternet.go
@@ -37,8 +37,8 @@
svc := clusternet.Service{
Curator: cur,
ClusterNet: net.IPNet{
- IP: []byte{10, 0, 0, 0},
- Mask: net.IPMask{255, 255, 0, 0},
+ IP: []byte{10, 192, 0, 0},
+ Mask: net.IPMask{255, 224, 0, 0},
},
DataDirectory: &s.storageRoot.Data.Kubernetes.ClusterNetworking,
LocalKubernetesPodNetwork: s.podNetwork,
diff --git a/metropolis/node/core/roleserve/worker_kubernetes.go b/metropolis/node/core/roleserve/worker_kubernetes.go
index 14781e3..03ded17 100644
--- a/metropolis/node/core/roleserve/worker_kubernetes.go
+++ b/metropolis/node/core/roleserve/worker_kubernetes.go
@@ -113,14 +113,14 @@
// TODO(q3k): make these configurable.
clusterIPRange := net.IPNet{
- IP: net.IP{10, 0, 0, 0},
- // That's a /16.
- Mask: net.IPMask{0xff, 0xff, 0x00, 0x00},
+ IP: net.IP{10, 192, 0, 0},
+ // That's a /11.
+ Mask: net.IPMask{0xff, 0xe0, 0x00, 0x00},
}
serviceIPRange := net.IPNet{
- IP: net.IP{10, 0, 255, 1},
- // That's a /24.
- Mask: net.IPMask{0xff, 0xff, 0xff, 0x00},
+ IP: net.IP{10, 224, 0, 1},
+ // That's a /16.
+ Mask: net.IPMask{0xff, 0xff, 0x00, 0x00},
}
// TODO(q3k): remove this once the controller also uses curator-emitted PKI.