Use CoreDNS for everything and make directives dynamic
This moves CoreDNS from Kubernetes to the network tree and uses
it for OS-side resolution too. For this to work together with Kubernetes it now
contains a dynamic directive system which allows various parts of the OS
to register and unregister directives at runtime. This system is used to hook
Kubernetes and DHCP-supplied DNS servers into the configuration.
This also enables the hosts plugin to resolve the local hostname from within
CoreDNS to avoid querying external DNS servers for that (T773).
Test Plan:
CTS covers K8s-related tests, external resolution manually tested from
a container.
Bug: T860, T773
X-Origin-Diff: phab/D628
GitOrigin-RevId: f1729237f3d17d8801506f4d299b90e7dce0893a
diff --git a/core/internal/network/BUILD.bazel b/core/internal/network/BUILD.bazel
index ad7de74..b2b486f 100644
--- a/core/internal/network/BUILD.bazel
+++ b/core/internal/network/BUILD.bazel
@@ -8,6 +8,7 @@
deps = [
"//core/internal/common/supervisor:go_default_library",
"//core/internal/network/dhcp:go_default_library",
+ "//core/internal/network/dns:go_default_library",
"@com_github_google_nftables//:go_default_library",
"@com_github_google_nftables//expr:go_default_library",
"@com_github_vishvananda_netlink//:go_default_library",
diff --git a/core/internal/network/dns/BUILD.bazel b/core/internal/network/dns/BUILD.bazel
new file mode 100644
index 0000000..d197191
--- /dev/null
+++ b/core/internal/network/dns/BUILD.bazel
@@ -0,0 +1,17 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = [
+ "coredns.go",
+ "directives.go",
+ ],
+ importpath = "git.monogon.dev/source/nexantic.git/core/internal/network/dns",
+ visibility = ["//core:__subpackages__"],
+ deps = [
+ "//core/internal/common/supervisor:go_default_library",
+ "//core/pkg/fileargs:go_default_library",
+ "//core/pkg/logbuffer:go_default_library",
+ "@org_uber_go_zap//:go_default_library",
+ ],
+)
diff --git a/core/internal/network/dns/coredns.go b/core/internal/network/dns/coredns.go
new file mode 100644
index 0000000..8c70c4f
--- /dev/null
+++ b/core/internal/network/dns/coredns.go
@@ -0,0 +1,171 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package DNS provides a DNS server using CoreDNS.
+package dns
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "os/exec"
+ "strings"
+ "sync"
+ "syscall"
+
+ "go.uber.org/zap"
+
+ "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
+ "git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
+ "git.monogon.dev/source/nexantic.git/core/pkg/logbuffer"
+)
+
+const corefileBase = `
+.:53 {
+ errors
+ hosts {
+ fallthrough
+ }
+
+ cache 30
+ loadbalance
+`
+
+type Service struct {
+ Logs *logbuffer.LogBuffer
+ directiveRegistration chan *ExtraDirective
+ directives map[string]ExtraDirective
+ cmd *exec.Cmd
+ args *fileargs.FileArgs
+ // stateMu guards access to the directives, cmd and args fields
+ stateMu sync.Mutex
+}
+
+// New creates a new CoreDNS service.
+// The given channel can then be used to dynamically register and unregister directives in the configuaration.
+// To register a new directive, send an ExtraDirective on the channel. To remove it again, use CancelDirective()
+// to create a removal message.
+func New(directiveRegistration chan *ExtraDirective) *Service {
+ return &Service{
+ Logs: logbuffer.New(5000, 16384),
+ directives: map[string]ExtraDirective{},
+ directiveRegistration: directiveRegistration,
+ }
+}
+
+func (s *Service) makeCorefile(fargs *fileargs.FileArgs) []byte {
+ corefile := bytes.Buffer{}
+ corefile.WriteString(corefileBase)
+ for _, dir := range s.directives {
+ resolvedDir := dir.directive
+ for fname, fcontent := range dir.files {
+ resolvedDir = strings.ReplaceAll(resolvedDir, fmt.Sprintf("$FILE(%v)", fname), fargs.ArgPath(fname, fcontent))
+ }
+ corefile.WriteString(resolvedDir)
+ corefile.WriteString("\n")
+ }
+ corefile.WriteString("\n}")
+ return corefile.Bytes()
+}
+
+// CancelDirective creates a message to cancel the given directive.
+func CancelDirective(d *ExtraDirective) *ExtraDirective {
+ return &ExtraDirective{
+ ID: d.ID,
+ }
+}
+
+// Run runs the DNS service consisting of the CoreDNS process and the directive registration process
+func (s *Service) Run(ctx context.Context) error {
+ supervisor.Run(ctx, "coredns", s.runCoreDNS)
+ supervisor.Run(ctx, "registration", s.runRegistration)
+ supervisor.Signal(ctx, supervisor.SignalHealthy)
+ supervisor.Signal(ctx, supervisor.SignalDone)
+ return nil
+}
+
+// runCoreDNS runs the CoreDNS proceess
+func (s *Service) runCoreDNS(ctx context.Context) error {
+ s.stateMu.Lock()
+ args, err := fileargs.New()
+ if err != nil {
+ s.stateMu.Unlock()
+ return fmt.Errorf("failed to create fileargs: %w", err)
+ }
+ defer args.Close()
+ s.args = args
+
+ cmd := exec.CommandContext(ctx, "/kubernetes/bin/coredns",
+ args.FileOpt("-conf", "Corefile", s.makeCorefile(args)),
+ )
+
+ if args.Error() != nil {
+ s.stateMu.Unlock()
+ return fmt.Errorf("failed to use fileargs: %w", err)
+ }
+
+ cmd.Stdout = s.Logs
+ cmd.Stderr = s.Logs
+
+ s.cmd = cmd
+
+ err = cmd.Start()
+
+ // Release stateMu after the process has attempted to start and is either dead or running
+ s.stateMu.Unlock()
+
+ if err != nil {
+ return fmt.Errorf("failed to start CoreDNS: %w", err)
+ }
+
+ supervisor.Signal(ctx, supervisor.SignalHealthy)
+
+ err = cmd.Wait()
+
+ fmt.Fprintf(s.Logs, "coredns stopped: %v\n", err)
+ return err
+}
+
+// runRegistration runs the background registration runnable which has a different lifecycle from the CoreDNS
+// runnable. It is responsible for managing dynamic directives.
+func (s *Service) runRegistration(ctx context.Context) error {
+ supervisor.Signal(ctx, supervisor.SignalHealthy)
+ for {
+ select {
+ case <-ctx.Done():
+ return nil
+ case d := <-s.directiveRegistration:
+ s.processRegistration(ctx, d)
+ }
+ }
+}
+
+func (s *Service) processRegistration(ctx context.Context, d *ExtraDirective) {
+ s.stateMu.Lock()
+ defer s.stateMu.Unlock()
+ if d.directive == "" {
+ delete(s.directives, d.ID)
+ } else {
+ s.directives[d.ID] = *d
+ }
+ // If the process is not currenty running we're relying on corefile regeneration on startup
+ if s.cmd != nil && s.cmd.Process != nil && s.cmd.ProcessState == nil {
+ s.args.ArgPath("Corefile", s.makeCorefile(s.args))
+ if err := s.cmd.Process.Signal(syscall.SIGUSR1); err != nil {
+ supervisor.Logger(ctx).Warn("Failed to send SIGUSR1 to CoreDNS for reload", zap.Error(err))
+ }
+ }
+}
diff --git a/core/internal/network/dns/directives.go b/core/internal/network/dns/directives.go
new file mode 100644
index 0000000..72c4f29
--- /dev/null
+++ b/core/internal/network/dns/directives.go
@@ -0,0 +1,73 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dns
+
+import (
+ "fmt"
+ "net"
+ "strings"
+)
+
+// Type ExtraDirective contains additional config directives for CoreDNS.
+type ExtraDirective struct {
+ // ID is the identifier of this directive. There can only be one directive with a given ID active at once.
+ // The ID is also used to identify which directive to purge.
+ ID string
+ // directive contains a full CoreDNS directive as a string. It can also use the $FILE(<filename>) macro,
+ // which will be expanded to the path of a file from the files field.
+ directive string
+ // files contains additional files used in the configuration. The map key is used as the filename.
+ files map[string][]byte
+}
+
+// NewUpstreamDirective creates a forward with no fallthrough that forwards all requests not yet matched to the given
+// upstream DNS servers.
+func NewUpstreamDirective(dnsServers []net.IP) *ExtraDirective {
+ strb := strings.Builder{}
+ if len(dnsServers) > 0 {
+ strb.WriteString("forward .")
+ for _, ip := range dnsServers {
+ strb.WriteString(" ")
+ strb.WriteString(ip.String())
+ }
+ }
+ return &ExtraDirective{
+ directive: strb.String(),
+ }
+}
+
+var kubernetesDirective = `
+kubernetes %v in-addr.arpa ip6.arpa {
+ kubeconfig $FILE(kubeconfig) default
+ pods insecure
+ fallthrough in-addr.arpa ip6.arpa
+ ttl 30
+}
+`
+
+// NewKubernetesDirective creates a directive running a "Kubernetes DNS-Based Service Discovery" [1] compliant service
+// under clusterDomain. The given kubeconfig needs at least read access to services, endpoints and endpointslices.
+// [1] https://github.com/kubernetes/dns/blob/master/docs/specification.md
+func NewKubernetesDirective(clusterDomain string, kubeconfig []byte) *ExtraDirective {
+ return &ExtraDirective{
+ ID: "k8s-clusterdns",
+ directive: fmt.Sprintf(kubernetesDirective, clusterDomain),
+ files: map[string][]byte{
+ "kubeconfig": kubeconfig,
+ },
+ }
+}
diff --git a/core/internal/network/main.go b/core/internal/network/main.go
index c92b21a..31c0b68 100644
--- a/core/internal/network/main.go
+++ b/core/internal/network/main.go
@@ -32,6 +32,7 @@
"git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
"git.monogon.dev/source/nexantic.git/core/internal/network/dhcp"
+ "git.monogon.dev/source/nexantic.git/core/internal/network/dns"
)
const (
@@ -47,6 +48,7 @@
}
type Config struct {
+ CorednsRegistrationChan chan *dns.ExtraDirective
}
func New(config Config) *Service {
@@ -117,9 +119,8 @@
return fmt.Errorf("could not get DHCP Status: %w", err)
}
- if err := setResolvconf(status.DNS, []string{}); err != nil {
- s.logger.Warn("failed to set resolvconf", zap.Error(err))
- }
+ // We're currently never removing this directive just like we're not removing routes and IPs
+ s.config.CorednsRegistrationChan <- dns.NewUpstreamDirective(status.DNS)
if err := s.addNetworkRoutes(iface, status.Address, status.Gateway); err != nil {
s.logger.Warn("failed to add routes", zap.Error(err))
@@ -172,18 +173,34 @@
}
func (s *Service) Run(ctx context.Context) error {
+ logger := supervisor.Logger(ctx)
+ dnsSvc := dns.New(s.config.CorednsRegistrationChan)
+ supervisor.Run(ctx, "dns", dnsSvc.Run)
+ supervisor.Run(ctx, "interfaces", s.runInterfaces)
+
+ if err := ioutil.WriteFile("/proc/sys/net/ipv4/ip_forward", []byte("1\n"), 0644); err != nil {
+ logger.Panic("Failed to enable IPv4 forwarding", zap.Error(err))
+ }
+
+ // We're handling all DNS requests with CoreDNS, including local ones
+ if err := setResolvconf([]net.IP{{127, 0, 0, 1}}, []string{}); err != nil {
+ logger.Warn("failed to set resolvconf", zap.Error(err))
+ }
+
+ supervisor.Signal(ctx, supervisor.SignalHealthy)
+ supervisor.Signal(ctx, supervisor.SignalDone)
+ return nil
+}
+
+func (s *Service) runInterfaces(ctx context.Context) error {
s.logger = supervisor.Logger(ctx)
- s.logger.Info("Starting network service")
+ s.logger.Info("Starting network interface management")
links, err := netlink.LinkList()
if err != nil {
s.logger.Fatal("Failed to list network links", zap.Error(err))
}
- if err := ioutil.WriteFile("/proc/sys/net/ipv4/ip_forward", []byte("1\n"), 0644); err != nil {
- s.logger.Panic("Failed to enable IPv4 forwarding", zap.Error(err))
- }
-
var ethernetLinks []netlink.Link
for _, link := range links {
attrs := link.Attrs()