Add Kubernetes DNS with CoreDNS
This adds Kubernetes DNS with a CoreDNS instance running on the host. This has some distinct advantages over
running it inside a container, like a simplified lifecycle (no state reconciliation) and the possibility of redirecting
all host DNS requests over this instance for observability or central DNSSEC enforcement.
Test Plan: Manually tested (`host kubernetes` in an Alpine container), will be covered by CTS.
X-Origin-Diff: phab/D616
GitOrigin-RevId: 281f5f384f4ef7eba2c3c3190be8e6a89772295c
diff --git a/core/internal/kubernetes/BUILD.bazel b/core/internal/kubernetes/BUILD.bazel
index 6b5d652..e040a9d 100644
--- a/core/internal/kubernetes/BUILD.bazel
+++ b/core/internal/kubernetes/BUILD.bazel
@@ -17,6 +17,7 @@
"//core/internal/common:go_default_library",
"//core/internal/common/supervisor:go_default_library",
"//core/internal/kubernetes/clusternet:go_default_library",
+ "//core/internal/kubernetes/dns:go_default_library",
"//core/internal/kubernetes/nfproxy:go_default_library",
"//core/internal/kubernetes/pki:go_default_library",
"//core/internal/kubernetes/reconciler:go_default_library",
diff --git a/core/internal/kubernetes/dns/BUILD.bazel b/core/internal/kubernetes/dns/BUILD.bazel
new file mode 100644
index 0000000..173360d
--- /dev/null
+++ b/core/internal/kubernetes/dns/BUILD.bazel
@@ -0,0 +1,12 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["coredns.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/internal/kubernetes/dns",
+ visibility = ["//core:__subpackages__"],
+ deps = [
+ "//core/internal/common/supervisor:go_default_library",
+ "//core/pkg/fileargs:go_default_library",
+ ],
+)
diff --git a/core/internal/kubernetes/dns/coredns.go b/core/internal/kubernetes/dns/coredns.go
new file mode 100644
index 0000000..d020ec5
--- /dev/null
+++ b/core/internal/kubernetes/dns/coredns.go
@@ -0,0 +1,91 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package DNS provides a Kubernetes DNS server using CoreDNS.
+package dns
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "os/exec"
+ "text/template"
+
+ "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
+ "git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
+)
+
+type corefileSpec struct {
+ KubeconfigPath string
+ ClusterDomain string
+}
+
+var corefileTemplate = template.Must(template.New("corefile").Parse(`
+.:53 {
+ errors
+ health {
+ lameduck 5s
+ }
+ kubernetes {{.ClusterDomain}} in-addr.arpa ip6.arpa {
+ kubeconfig {{.KubeconfigPath}} default
+ pods insecure
+ fallthrough in-addr.arpa ip6.arpa
+ ttl 30
+ }
+ forward . /etc/resolv.conf
+ cache 30
+ loadbalance
+}
+`))
+
+type Service struct {
+ Output io.Writer
+ Kubeconfig []byte
+ ClusterDomain string
+}
+
+func (s *Service) Run(ctx context.Context) error {
+ args, err := fileargs.New()
+ if err != nil {
+ return fmt.Errorf("failed to create fileargs: %w", err)
+ }
+ defer args.Close()
+
+ var corefile bytes.Buffer
+ if err := corefileTemplate.Execute(&corefile, &corefileSpec{
+ KubeconfigPath: args.ArgPath("kubeconfig", s.Kubeconfig),
+ ClusterDomain: s.ClusterDomain,
+ }); err != nil {
+ return fmt.Errorf("failed to execute Corefile template: %w", err)
+ }
+
+ cmd := exec.CommandContext(ctx, "/kubernetes/bin/coredns",
+ args.FileOpt("-conf", "Corefile", corefile.Bytes()),
+ )
+
+ if args.Error() != nil {
+ return fmt.Errorf("failed to use fileargs: %w", err)
+ }
+
+ cmd.Stdout = s.Output
+ cmd.Stderr = s.Output
+
+ supervisor.Signal(ctx, supervisor.SignalHealthy)
+ err = cmd.Run()
+ fmt.Fprintf(s.Output, "coredns stopped: %v\n", err)
+ return err
+}
diff --git a/core/internal/kubernetes/service.go b/core/internal/kubernetes/service.go
index a22b6b9..55c20bf 100644
--- a/core/internal/kubernetes/service.go
+++ b/core/internal/kubernetes/service.go
@@ -33,6 +33,7 @@
"git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/clusternet"
+ "git.monogon.dev/source/nexantic.git/core/internal/kubernetes/dns"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/nfproxy"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/pki"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/reconciler"
@@ -55,6 +56,7 @@
controllerManagerLogs *logbuffer.LogBuffer
schedulerLogs *logbuffer.LogBuffer
kubeletLogs *logbuffer.LogBuffer
+ corednsLogs *logbuffer.LogBuffer
}
type Service struct {
@@ -82,6 +84,7 @@
controllerManagerLogs: logbuffer.New(5000, 16384),
schedulerLogs: logbuffer.New(5000, 16384),
kubeletLogs: logbuffer.New(5000, 16384),
+ corednsLogs: logbuffer.New(5000, 16384),
}
s.stateMu.Lock()
s.state = st
@@ -120,6 +123,8 @@
return fmt.Errorf("failed to get hostname: %w", err)
}
+ dnsHostIP := s.c.AdvertiseAddress // TODO: Which IP to use
+
apiserver := &apiserverService{
KPKI: s.c.KPKI,
AdvertiseAddress: s.c.AdvertiseAddress,
@@ -130,7 +135,7 @@
kubelet := kubeletService{
NodeName: hostname,
- ClusterDNS: nil,
+ ClusterDNS: []net.IP{dnsHostIP},
KubeletDirectory: &s.c.Root.Data.Kubernetes.Kubelet,
EphemeralDirectory: &s.c.Root.Ephemeral,
Output: st.kubeletLogs,
@@ -162,6 +167,12 @@
ClientSet: clientSet,
}
+ dns := dns.Service{
+ Kubeconfig: masterKubeconfig,
+ Output: s.state.corednsLogs,
+ ClusterDomain: "cluster.local", // Hardcode this here until we make this configurable
+ }
+
for _, sub := range []struct {
name string
runnable supervisor.Runnable
@@ -175,6 +186,7 @@
{"csi-provisioner", csiProvisioner.Run},
{"clusternet", clusternet.Run},
{"nfproxy", nfproxy.Run},
+ {"dns", dns.Run},
} {
err := supervisor.Run(ctx, sub.name, sub.runnable)
if err != nil {
@@ -204,6 +216,8 @@
return s.state.schedulerLogs.ReadLinesTruncated(n, "..."), nil
case "kubelet":
return s.state.kubeletLogs.ReadLinesTruncated(n, "..."), nil
+ case "coredns":
+ return s.state.corednsLogs.ReadLinesTruncated(n, "..."), nil
default:
return nil, errors.New("component not available")
}