Use CoreDNS for everything and make directives dynamic
This moves CoreDNS from Kubernetes to the network tree and uses
it for OS-side resolution too. For this to work together with Kubernetes it now
contains a dynamic directive system which allows various parts of the OS
to register and unregister directives at runtime. This system is used to hook
Kubernetes and DHCP-supplied DNS servers into the configuration.
This also enables the hosts plugin to resolve the local hostname from within
CoreDNS to avoid querying external DNS servers for that (T773).
Test Plan:
CTS covers K8s-related tests, external resolution manually tested from
a container.
Bug: T860, T773
X-Origin-Diff: phab/D628
GitOrigin-RevId: f1729237f3d17d8801506f4d299b90e7dce0893a
diff --git a/core/internal/kubernetes/BUILD.bazel b/core/internal/kubernetes/BUILD.bazel
index e040a9d..69afe18 100644
--- a/core/internal/kubernetes/BUILD.bazel
+++ b/core/internal/kubernetes/BUILD.bazel
@@ -17,12 +17,12 @@
"//core/internal/common:go_default_library",
"//core/internal/common/supervisor:go_default_library",
"//core/internal/kubernetes/clusternet:go_default_library",
- "//core/internal/kubernetes/dns:go_default_library",
"//core/internal/kubernetes/nfproxy:go_default_library",
"//core/internal/kubernetes/pki:go_default_library",
"//core/internal/kubernetes/reconciler:go_default_library",
"//core/internal/localstorage:go_default_library",
"//core/internal/localstorage/declarative:go_default_library",
+ "//core/internal/network/dns:go_default_library",
"//core/pkg/fileargs:go_default_library",
"//core/pkg/fsquota:go_default_library",
"//core/pkg/logbuffer:go_default_library",
diff --git a/core/internal/kubernetes/dns/coredns.go b/core/internal/kubernetes/dns/coredns.go
deleted file mode 100644
index d020ec5..0000000
--- a/core/internal/kubernetes/dns/coredns.go
+++ /dev/null
@@ -1,91 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package DNS provides a Kubernetes DNS server using CoreDNS.
-package dns
-
-import (
- "bytes"
- "context"
- "fmt"
- "io"
- "os/exec"
- "text/template"
-
- "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
- "git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
-)
-
-type corefileSpec struct {
- KubeconfigPath string
- ClusterDomain string
-}
-
-var corefileTemplate = template.Must(template.New("corefile").Parse(`
-.:53 {
- errors
- health {
- lameduck 5s
- }
- kubernetes {{.ClusterDomain}} in-addr.arpa ip6.arpa {
- kubeconfig {{.KubeconfigPath}} default
- pods insecure
- fallthrough in-addr.arpa ip6.arpa
- ttl 30
- }
- forward . /etc/resolv.conf
- cache 30
- loadbalance
-}
-`))
-
-type Service struct {
- Output io.Writer
- Kubeconfig []byte
- ClusterDomain string
-}
-
-func (s *Service) Run(ctx context.Context) error {
- args, err := fileargs.New()
- if err != nil {
- return fmt.Errorf("failed to create fileargs: %w", err)
- }
- defer args.Close()
-
- var corefile bytes.Buffer
- if err := corefileTemplate.Execute(&corefile, &corefileSpec{
- KubeconfigPath: args.ArgPath("kubeconfig", s.Kubeconfig),
- ClusterDomain: s.ClusterDomain,
- }); err != nil {
- return fmt.Errorf("failed to execute Corefile template: %w", err)
- }
-
- cmd := exec.CommandContext(ctx, "/kubernetes/bin/coredns",
- args.FileOpt("-conf", "Corefile", corefile.Bytes()),
- )
-
- if args.Error() != nil {
- return fmt.Errorf("failed to use fileargs: %w", err)
- }
-
- cmd.Stdout = s.Output
- cmd.Stderr = s.Output
-
- supervisor.Signal(ctx, supervisor.SignalHealthy)
- err = cmd.Run()
- fmt.Fprintf(s.Output, "coredns stopped: %v\n", err)
- return err
-}
diff --git a/core/internal/kubernetes/service.go b/core/internal/kubernetes/service.go
index 55c20bf..e4391f5 100644
--- a/core/internal/kubernetes/service.go
+++ b/core/internal/kubernetes/service.go
@@ -33,11 +33,11 @@
"git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/clusternet"
- "git.monogon.dev/source/nexantic.git/core/internal/kubernetes/dns"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/nfproxy"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/pki"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/reconciler"
"git.monogon.dev/source/nexantic.git/core/internal/localstorage"
+ "git.monogon.dev/source/nexantic.git/core/internal/network/dns"
"git.monogon.dev/source/nexantic.git/core/pkg/logbuffer"
apb "git.monogon.dev/source/nexantic.git/core/proto/api"
)
@@ -47,8 +47,9 @@
ServiceIPRange net.IPNet
ClusterNet net.IPNet
- KPKI *pki.KubernetesPKI
- Root *localstorage.Root
+ KPKI *pki.KubernetesPKI
+ Root *localstorage.Root
+ CorednsRegistrationChan chan *dns.ExtraDirective
}
type state struct {
@@ -56,7 +57,6 @@
controllerManagerLogs *logbuffer.LogBuffer
schedulerLogs *logbuffer.LogBuffer
kubeletLogs *logbuffer.LogBuffer
- corednsLogs *logbuffer.LogBuffer
}
type Service struct {
@@ -84,7 +84,6 @@
controllerManagerLogs: logbuffer.New(5000, 16384),
schedulerLogs: logbuffer.New(5000, 16384),
kubeletLogs: logbuffer.New(5000, 16384),
- corednsLogs: logbuffer.New(5000, 16384),
}
s.stateMu.Lock()
s.state = st
@@ -167,12 +166,6 @@
ClientSet: clientSet,
}
- dns := dns.Service{
- Kubeconfig: masterKubeconfig,
- Output: s.state.corednsLogs,
- ClusterDomain: "cluster.local", // Hardcode this here until we make this configurable
- }
-
for _, sub := range []struct {
name string
runnable supervisor.Runnable
@@ -186,7 +179,6 @@
{"csi-provisioner", csiProvisioner.Run},
{"clusternet", clusternet.Run},
{"nfproxy", nfproxy.Run},
- {"dns", dns.Run},
} {
err := supervisor.Run(ctx, sub.name, sub.runnable)
if err != nil {
@@ -194,8 +186,13 @@
}
}
+ supervisor.Logger(ctx).Info("Registering K8s CoreDNS")
+ clusterDNSDirective := dns.NewKubernetesDirective("cluster.local", masterKubeconfig)
+ s.c.CorednsRegistrationChan <- clusterDNSDirective
+
supervisor.Signal(ctx, supervisor.SignalHealthy)
- supervisor.Signal(ctx, supervisor.SignalDone)
+ <-ctx.Done()
+ s.c.CorednsRegistrationChan <- dns.CancelDirective(clusterDNSDirective)
return nil
}
@@ -216,8 +213,6 @@
return s.state.schedulerLogs.ReadLinesTruncated(n, "..."), nil
case "kubelet":
return s.state.kubeletLogs.ReadLinesTruncated(n, "..."), nil
- case "coredns":
- return s.state.corednsLogs.ReadLinesTruncated(n, "..."), nil
default:
return nil, errors.New("component not available")
}
diff --git a/core/internal/localstorage/declarative/declarative.go b/core/internal/localstorage/declarative/declarative.go
index b6b1220..ce82c42 100644
--- a/core/internal/localstorage/declarative/declarative.go
+++ b/core/internal/localstorage/declarative/declarative.go
@@ -24,7 +24,7 @@
// Directory represents the intent of existence of a directory in a hierarchical filesystem (simplified to a tree).
// This structure can be embedded and still be interpreted as a Directory for purposes of use within this library. Any
-// inner fields of such an embedding structure that are in turn (embedded) Directories or Files will be treated as
+// inner fields of such an embedding structure that are in turn (embedded) Directories or files will be treated as
// children in the intent expressed by this Directory. All contained directory fields must have a `dir:"name"` struct
// tag that names them, and all contained file fields must have a `file:"name"` struct tag.
//
@@ -35,7 +35,7 @@
DirectoryPlacement
}
-// File represents the intent of existence of a file. Files are usually child structures in types that embed Directory.
+// File represents the intent of existence of a file. files are usually child structures in types that embed Directory.
// File can also be embedded in another structure, and this embedding type will still be interpreted as a File for
// purposes of use within this library.
//
diff --git a/core/internal/localstorage/declarative/placement.go b/core/internal/localstorage/declarative/placement.go
index c16da1d..c2ff53d 100644
--- a/core/internal/localstorage/declarative/placement.go
+++ b/core/internal/localstorage/declarative/placement.go
@@ -26,7 +26,7 @@
// when placed (ie., implementations like PlaceFS takes a *Directory, but Root as a declarative definition is defined as
// non-pointer).
-// Placement is an interface available on Placed Files and Directories. All *Placement interfaces on Files/Directories
+// Placement is an interface available on Placed files and Directories. All *Placement interfaces on files/Directories
// are only available on placed trees - eg., after a PlaceFS call. This is unfortunately not typesafe, callers need to
// either be sure about placement, or check the interface for null.
type Placement interface {
@@ -34,7 +34,7 @@
RootRef() interface{}
}
-// FilePlacement is an interface available on Placed Files. It is implemented by different placement backends, and
+// FilePlacement is an interface available on Placed files. It is implemented by different placement backends, and
// set on all files during placement by a given backend.
type FilePlacement interface {
Placement
diff --git a/core/internal/localstorage/storage.go b/core/internal/localstorage/storage.go
index fae92e4..26edae7 100644
--- a/core/internal/localstorage/storage.go
+++ b/core/internal/localstorage/storage.go
@@ -25,7 +25,7 @@
// correspond to locations on a filesystem).
//
// Every member of the storage hierarchy must either be, or inherit from Directory or File. In order to be placed
-// correctly, Directory embedding structures must use `dir:` or `file:` tags for child Directories and Files
+// correctly, Directory embedding structures must use `dir:` or `file:` tags for child Directories and files
// respectively. The content of the tag specifies the path part that this element will be placed at.
//
// Full placement path(available via FullPath()) format is placement implementation-specific. However, they're always
diff --git a/core/internal/network/BUILD.bazel b/core/internal/network/BUILD.bazel
index ad7de74..b2b486f 100644
--- a/core/internal/network/BUILD.bazel
+++ b/core/internal/network/BUILD.bazel
@@ -8,6 +8,7 @@
deps = [
"//core/internal/common/supervisor:go_default_library",
"//core/internal/network/dhcp:go_default_library",
+ "//core/internal/network/dns:go_default_library",
"@com_github_google_nftables//:go_default_library",
"@com_github_google_nftables//expr:go_default_library",
"@com_github_vishvananda_netlink//:go_default_library",
diff --git a/core/internal/kubernetes/dns/BUILD.bazel b/core/internal/network/dns/BUILD.bazel
similarity index 65%
rename from core/internal/kubernetes/dns/BUILD.bazel
rename to core/internal/network/dns/BUILD.bazel
index 173360d..d197191 100644
--- a/core/internal/kubernetes/dns/BUILD.bazel
+++ b/core/internal/network/dns/BUILD.bazel
@@ -2,11 +2,16 @@
go_library(
name = "go_default_library",
- srcs = ["coredns.go"],
- importpath = "git.monogon.dev/source/nexantic.git/core/internal/kubernetes/dns",
+ srcs = [
+ "coredns.go",
+ "directives.go",
+ ],
+ importpath = "git.monogon.dev/source/nexantic.git/core/internal/network/dns",
visibility = ["//core:__subpackages__"],
deps = [
"//core/internal/common/supervisor:go_default_library",
"//core/pkg/fileargs:go_default_library",
+ "//core/pkg/logbuffer:go_default_library",
+ "@org_uber_go_zap//:go_default_library",
],
)
diff --git a/core/internal/network/dns/coredns.go b/core/internal/network/dns/coredns.go
new file mode 100644
index 0000000..8c70c4f
--- /dev/null
+++ b/core/internal/network/dns/coredns.go
@@ -0,0 +1,171 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package DNS provides a DNS server using CoreDNS.
+package dns
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "os/exec"
+ "strings"
+ "sync"
+ "syscall"
+
+ "go.uber.org/zap"
+
+ "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
+ "git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
+ "git.monogon.dev/source/nexantic.git/core/pkg/logbuffer"
+)
+
+const corefileBase = `
+.:53 {
+ errors
+ hosts {
+ fallthrough
+ }
+
+ cache 30
+ loadbalance
+`
+
+type Service struct {
+ Logs *logbuffer.LogBuffer
+ directiveRegistration chan *ExtraDirective
+ directives map[string]ExtraDirective
+ cmd *exec.Cmd
+ args *fileargs.FileArgs
+ // stateMu guards access to the directives, cmd and args fields
+ stateMu sync.Mutex
+}
+
+// New creates a new CoreDNS service.
+// The given channel can then be used to dynamically register and unregister directives in the configuaration.
+// To register a new directive, send an ExtraDirective on the channel. To remove it again, use CancelDirective()
+// to create a removal message.
+func New(directiveRegistration chan *ExtraDirective) *Service {
+ return &Service{
+ Logs: logbuffer.New(5000, 16384),
+ directives: map[string]ExtraDirective{},
+ directiveRegistration: directiveRegistration,
+ }
+}
+
+func (s *Service) makeCorefile(fargs *fileargs.FileArgs) []byte {
+ corefile := bytes.Buffer{}
+ corefile.WriteString(corefileBase)
+ for _, dir := range s.directives {
+ resolvedDir := dir.directive
+ for fname, fcontent := range dir.files {
+ resolvedDir = strings.ReplaceAll(resolvedDir, fmt.Sprintf("$FILE(%v)", fname), fargs.ArgPath(fname, fcontent))
+ }
+ corefile.WriteString(resolvedDir)
+ corefile.WriteString("\n")
+ }
+ corefile.WriteString("\n}")
+ return corefile.Bytes()
+}
+
+// CancelDirective creates a message to cancel the given directive.
+func CancelDirective(d *ExtraDirective) *ExtraDirective {
+ return &ExtraDirective{
+ ID: d.ID,
+ }
+}
+
+// Run runs the DNS service consisting of the CoreDNS process and the directive registration process
+func (s *Service) Run(ctx context.Context) error {
+ supervisor.Run(ctx, "coredns", s.runCoreDNS)
+ supervisor.Run(ctx, "registration", s.runRegistration)
+ supervisor.Signal(ctx, supervisor.SignalHealthy)
+ supervisor.Signal(ctx, supervisor.SignalDone)
+ return nil
+}
+
+// runCoreDNS runs the CoreDNS proceess
+func (s *Service) runCoreDNS(ctx context.Context) error {
+ s.stateMu.Lock()
+ args, err := fileargs.New()
+ if err != nil {
+ s.stateMu.Unlock()
+ return fmt.Errorf("failed to create fileargs: %w", err)
+ }
+ defer args.Close()
+ s.args = args
+
+ cmd := exec.CommandContext(ctx, "/kubernetes/bin/coredns",
+ args.FileOpt("-conf", "Corefile", s.makeCorefile(args)),
+ )
+
+ if args.Error() != nil {
+ s.stateMu.Unlock()
+ return fmt.Errorf("failed to use fileargs: %w", err)
+ }
+
+ cmd.Stdout = s.Logs
+ cmd.Stderr = s.Logs
+
+ s.cmd = cmd
+
+ err = cmd.Start()
+
+ // Release stateMu after the process has attempted to start and is either dead or running
+ s.stateMu.Unlock()
+
+ if err != nil {
+ return fmt.Errorf("failed to start CoreDNS: %w", err)
+ }
+
+ supervisor.Signal(ctx, supervisor.SignalHealthy)
+
+ err = cmd.Wait()
+
+ fmt.Fprintf(s.Logs, "coredns stopped: %v\n", err)
+ return err
+}
+
+// runRegistration runs the background registration runnable which has a different lifecycle from the CoreDNS
+// runnable. It is responsible for managing dynamic directives.
+func (s *Service) runRegistration(ctx context.Context) error {
+ supervisor.Signal(ctx, supervisor.SignalHealthy)
+ for {
+ select {
+ case <-ctx.Done():
+ return nil
+ case d := <-s.directiveRegistration:
+ s.processRegistration(ctx, d)
+ }
+ }
+}
+
+func (s *Service) processRegistration(ctx context.Context, d *ExtraDirective) {
+ s.stateMu.Lock()
+ defer s.stateMu.Unlock()
+ if d.directive == "" {
+ delete(s.directives, d.ID)
+ } else {
+ s.directives[d.ID] = *d
+ }
+ // If the process is not currenty running we're relying on corefile regeneration on startup
+ if s.cmd != nil && s.cmd.Process != nil && s.cmd.ProcessState == nil {
+ s.args.ArgPath("Corefile", s.makeCorefile(s.args))
+ if err := s.cmd.Process.Signal(syscall.SIGUSR1); err != nil {
+ supervisor.Logger(ctx).Warn("Failed to send SIGUSR1 to CoreDNS for reload", zap.Error(err))
+ }
+ }
+}
diff --git a/core/internal/network/dns/directives.go b/core/internal/network/dns/directives.go
new file mode 100644
index 0000000..72c4f29
--- /dev/null
+++ b/core/internal/network/dns/directives.go
@@ -0,0 +1,73 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dns
+
+import (
+ "fmt"
+ "net"
+ "strings"
+)
+
+// Type ExtraDirective contains additional config directives for CoreDNS.
+type ExtraDirective struct {
+ // ID is the identifier of this directive. There can only be one directive with a given ID active at once.
+ // The ID is also used to identify which directive to purge.
+ ID string
+ // directive contains a full CoreDNS directive as a string. It can also use the $FILE(<filename>) macro,
+ // which will be expanded to the path of a file from the files field.
+ directive string
+ // files contains additional files used in the configuration. The map key is used as the filename.
+ files map[string][]byte
+}
+
+// NewUpstreamDirective creates a forward with no fallthrough that forwards all requests not yet matched to the given
+// upstream DNS servers.
+func NewUpstreamDirective(dnsServers []net.IP) *ExtraDirective {
+ strb := strings.Builder{}
+ if len(dnsServers) > 0 {
+ strb.WriteString("forward .")
+ for _, ip := range dnsServers {
+ strb.WriteString(" ")
+ strb.WriteString(ip.String())
+ }
+ }
+ return &ExtraDirective{
+ directive: strb.String(),
+ }
+}
+
+var kubernetesDirective = `
+kubernetes %v in-addr.arpa ip6.arpa {
+ kubeconfig $FILE(kubeconfig) default
+ pods insecure
+ fallthrough in-addr.arpa ip6.arpa
+ ttl 30
+}
+`
+
+// NewKubernetesDirective creates a directive running a "Kubernetes DNS-Based Service Discovery" [1] compliant service
+// under clusterDomain. The given kubeconfig needs at least read access to services, endpoints and endpointslices.
+// [1] https://github.com/kubernetes/dns/blob/master/docs/specification.md
+func NewKubernetesDirective(clusterDomain string, kubeconfig []byte) *ExtraDirective {
+ return &ExtraDirective{
+ ID: "k8s-clusterdns",
+ directive: fmt.Sprintf(kubernetesDirective, clusterDomain),
+ files: map[string][]byte{
+ "kubeconfig": kubeconfig,
+ },
+ }
+}
diff --git a/core/internal/network/main.go b/core/internal/network/main.go
index c92b21a..31c0b68 100644
--- a/core/internal/network/main.go
+++ b/core/internal/network/main.go
@@ -32,6 +32,7 @@
"git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
"git.monogon.dev/source/nexantic.git/core/internal/network/dhcp"
+ "git.monogon.dev/source/nexantic.git/core/internal/network/dns"
)
const (
@@ -47,6 +48,7 @@
}
type Config struct {
+ CorednsRegistrationChan chan *dns.ExtraDirective
}
func New(config Config) *Service {
@@ -117,9 +119,8 @@
return fmt.Errorf("could not get DHCP Status: %w", err)
}
- if err := setResolvconf(status.DNS, []string{}); err != nil {
- s.logger.Warn("failed to set resolvconf", zap.Error(err))
- }
+ // We're currently never removing this directive just like we're not removing routes and IPs
+ s.config.CorednsRegistrationChan <- dns.NewUpstreamDirective(status.DNS)
if err := s.addNetworkRoutes(iface, status.Address, status.Gateway); err != nil {
s.logger.Warn("failed to add routes", zap.Error(err))
@@ -172,18 +173,34 @@
}
func (s *Service) Run(ctx context.Context) error {
+ logger := supervisor.Logger(ctx)
+ dnsSvc := dns.New(s.config.CorednsRegistrationChan)
+ supervisor.Run(ctx, "dns", dnsSvc.Run)
+ supervisor.Run(ctx, "interfaces", s.runInterfaces)
+
+ if err := ioutil.WriteFile("/proc/sys/net/ipv4/ip_forward", []byte("1\n"), 0644); err != nil {
+ logger.Panic("Failed to enable IPv4 forwarding", zap.Error(err))
+ }
+
+ // We're handling all DNS requests with CoreDNS, including local ones
+ if err := setResolvconf([]net.IP{{127, 0, 0, 1}}, []string{}); err != nil {
+ logger.Warn("failed to set resolvconf", zap.Error(err))
+ }
+
+ supervisor.Signal(ctx, supervisor.SignalHealthy)
+ supervisor.Signal(ctx, supervisor.SignalDone)
+ return nil
+}
+
+func (s *Service) runInterfaces(ctx context.Context) error {
s.logger = supervisor.Logger(ctx)
- s.logger.Info("Starting network service")
+ s.logger.Info("Starting network interface management")
links, err := netlink.LinkList()
if err != nil {
s.logger.Fatal("Failed to list network links", zap.Error(err))
}
- if err := ioutil.WriteFile("/proc/sys/net/ipv4/ip_forward", []byte("1\n"), 0644); err != nil {
- s.logger.Panic("Failed to enable IPv4 forwarding", zap.Error(err))
- }
-
var ethernetLinks []netlink.Link
for _, link := range links {
attrs := link.Attrs()