Use CoreDNS for everything and make directives dynamic
This moves CoreDNS from Kubernetes to the network tree and uses
it for OS-side resolution too. For this to work together with Kubernetes it now
contains a dynamic directive system which allows various parts of the OS
to register and unregister directives at runtime. This system is used to hook
Kubernetes and DHCP-supplied DNS servers into the configuration.
This also enables the hosts plugin to resolve the local hostname from within
CoreDNS to avoid querying external DNS servers for that (T773).
Test Plan:
CTS covers K8s-related tests, external resolution manually tested from
a container.
Bug: T860, T773
X-Origin-Diff: phab/D628
GitOrigin-RevId: f1729237f3d17d8801506f4d299b90e7dce0893a
diff --git a/core/internal/kubernetes/BUILD.bazel b/core/internal/kubernetes/BUILD.bazel
index e040a9d..69afe18 100644
--- a/core/internal/kubernetes/BUILD.bazel
+++ b/core/internal/kubernetes/BUILD.bazel
@@ -17,12 +17,12 @@
"//core/internal/common:go_default_library",
"//core/internal/common/supervisor:go_default_library",
"//core/internal/kubernetes/clusternet:go_default_library",
- "//core/internal/kubernetes/dns:go_default_library",
"//core/internal/kubernetes/nfproxy:go_default_library",
"//core/internal/kubernetes/pki:go_default_library",
"//core/internal/kubernetes/reconciler:go_default_library",
"//core/internal/localstorage:go_default_library",
"//core/internal/localstorage/declarative:go_default_library",
+ "//core/internal/network/dns:go_default_library",
"//core/pkg/fileargs:go_default_library",
"//core/pkg/fsquota:go_default_library",
"//core/pkg/logbuffer:go_default_library",
diff --git a/core/internal/kubernetes/dns/BUILD.bazel b/core/internal/kubernetes/dns/BUILD.bazel
deleted file mode 100644
index 173360d..0000000
--- a/core/internal/kubernetes/dns/BUILD.bazel
+++ /dev/null
@@ -1,12 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
-
-go_library(
- name = "go_default_library",
- srcs = ["coredns.go"],
- importpath = "git.monogon.dev/source/nexantic.git/core/internal/kubernetes/dns",
- visibility = ["//core:__subpackages__"],
- deps = [
- "//core/internal/common/supervisor:go_default_library",
- "//core/pkg/fileargs:go_default_library",
- ],
-)
diff --git a/core/internal/kubernetes/dns/coredns.go b/core/internal/kubernetes/dns/coredns.go
deleted file mode 100644
index d020ec5..0000000
--- a/core/internal/kubernetes/dns/coredns.go
+++ /dev/null
@@ -1,91 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package DNS provides a Kubernetes DNS server using CoreDNS.
-package dns
-
-import (
- "bytes"
- "context"
- "fmt"
- "io"
- "os/exec"
- "text/template"
-
- "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
- "git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
-)
-
-type corefileSpec struct {
- KubeconfigPath string
- ClusterDomain string
-}
-
-var corefileTemplate = template.Must(template.New("corefile").Parse(`
-.:53 {
- errors
- health {
- lameduck 5s
- }
- kubernetes {{.ClusterDomain}} in-addr.arpa ip6.arpa {
- kubeconfig {{.KubeconfigPath}} default
- pods insecure
- fallthrough in-addr.arpa ip6.arpa
- ttl 30
- }
- forward . /etc/resolv.conf
- cache 30
- loadbalance
-}
-`))
-
-type Service struct {
- Output io.Writer
- Kubeconfig []byte
- ClusterDomain string
-}
-
-func (s *Service) Run(ctx context.Context) error {
- args, err := fileargs.New()
- if err != nil {
- return fmt.Errorf("failed to create fileargs: %w", err)
- }
- defer args.Close()
-
- var corefile bytes.Buffer
- if err := corefileTemplate.Execute(&corefile, &corefileSpec{
- KubeconfigPath: args.ArgPath("kubeconfig", s.Kubeconfig),
- ClusterDomain: s.ClusterDomain,
- }); err != nil {
- return fmt.Errorf("failed to execute Corefile template: %w", err)
- }
-
- cmd := exec.CommandContext(ctx, "/kubernetes/bin/coredns",
- args.FileOpt("-conf", "Corefile", corefile.Bytes()),
- )
-
- if args.Error() != nil {
- return fmt.Errorf("failed to use fileargs: %w", err)
- }
-
- cmd.Stdout = s.Output
- cmd.Stderr = s.Output
-
- supervisor.Signal(ctx, supervisor.SignalHealthy)
- err = cmd.Run()
- fmt.Fprintf(s.Output, "coredns stopped: %v\n", err)
- return err
-}
diff --git a/core/internal/kubernetes/service.go b/core/internal/kubernetes/service.go
index 55c20bf..e4391f5 100644
--- a/core/internal/kubernetes/service.go
+++ b/core/internal/kubernetes/service.go
@@ -33,11 +33,11 @@
"git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/clusternet"
- "git.monogon.dev/source/nexantic.git/core/internal/kubernetes/dns"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/nfproxy"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/pki"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/reconciler"
"git.monogon.dev/source/nexantic.git/core/internal/localstorage"
+ "git.monogon.dev/source/nexantic.git/core/internal/network/dns"
"git.monogon.dev/source/nexantic.git/core/pkg/logbuffer"
apb "git.monogon.dev/source/nexantic.git/core/proto/api"
)
@@ -47,8 +47,9 @@
ServiceIPRange net.IPNet
ClusterNet net.IPNet
- KPKI *pki.KubernetesPKI
- Root *localstorage.Root
+ KPKI *pki.KubernetesPKI
+ Root *localstorage.Root
+ CorednsRegistrationChan chan *dns.ExtraDirective
}
type state struct {
@@ -56,7 +57,6 @@
controllerManagerLogs *logbuffer.LogBuffer
schedulerLogs *logbuffer.LogBuffer
kubeletLogs *logbuffer.LogBuffer
- corednsLogs *logbuffer.LogBuffer
}
type Service struct {
@@ -84,7 +84,6 @@
controllerManagerLogs: logbuffer.New(5000, 16384),
schedulerLogs: logbuffer.New(5000, 16384),
kubeletLogs: logbuffer.New(5000, 16384),
- corednsLogs: logbuffer.New(5000, 16384),
}
s.stateMu.Lock()
s.state = st
@@ -167,12 +166,6 @@
ClientSet: clientSet,
}
- dns := dns.Service{
- Kubeconfig: masterKubeconfig,
- Output: s.state.corednsLogs,
- ClusterDomain: "cluster.local", // Hardcode this here until we make this configurable
- }
-
for _, sub := range []struct {
name string
runnable supervisor.Runnable
@@ -186,7 +179,6 @@
{"csi-provisioner", csiProvisioner.Run},
{"clusternet", clusternet.Run},
{"nfproxy", nfproxy.Run},
- {"dns", dns.Run},
} {
err := supervisor.Run(ctx, sub.name, sub.runnable)
if err != nil {
@@ -194,8 +186,13 @@
}
}
+ supervisor.Logger(ctx).Info("Registering K8s CoreDNS")
+ clusterDNSDirective := dns.NewKubernetesDirective("cluster.local", masterKubeconfig)
+ s.c.CorednsRegistrationChan <- clusterDNSDirective
+
supervisor.Signal(ctx, supervisor.SignalHealthy)
- supervisor.Signal(ctx, supervisor.SignalDone)
+ <-ctx.Done()
+ s.c.CorednsRegistrationChan <- dns.CancelDirective(clusterDNSDirective)
return nil
}
@@ -216,8 +213,6 @@
return s.state.schedulerLogs.ReadLinesTruncated(n, "..."), nil
case "kubelet":
return s.state.kubeletLogs.ReadLinesTruncated(n, "..."), nil
- case "coredns":
- return s.state.corednsLogs.ReadLinesTruncated(n, "..."), nil
default:
return nil, errors.New("component not available")
}