Add E2E tests for basic functionality and port launching to Go
This adds a new E2E test suite replacing the old log-parsing
based one. It also moves launching and controlling Smalltown VMs into
a Go package and command and exposes the '//:launch' alias.
The new E2E test suite covers basic conditions (IP assigned, Data
available) and Kubernetes Node, Deployment and StatefulSet tests.
Test Plan: This consists of E2E tests
X-Origin-Diff: phab/D544
GitOrigin-RevId: 7c624c667c849068bafa544a3a6c635d6d406e1c
diff --git a/core/cmd/launch/BUILD.bazel b/core/cmd/launch/BUILD.bazel
new file mode 100644
index 0000000..59d8ecc
--- /dev/null
+++ b/core/cmd/launch/BUILD.bazel
@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["main.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/cmd/launch",
+ visibility = ["//visibility:private"],
+ deps = ["//core/internal/launch:go_default_library"],
+)
+
+go_binary(
+ name = "launch",
+ data = [
+ "//core:image",
+ "//core:swtpm_data",
+ "//third_party/edk2:firmware",
+ ],
+ embed = [":go_default_library"],
+ visibility = ["//visibility:public"],
+)
diff --git a/core/cmd/launch/main.go b/core/cmd/launch/main.go
new file mode 100644
index 0000000..100d350
--- /dev/null
+++ b/core/cmd/launch/main.go
@@ -0,0 +1,43 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "os/signal"
+ "syscall"
+
+ "git.monogon.dev/source/nexantic.git/core/internal/launch"
+)
+
+func main() {
+ sigs := make(chan os.Signal, 1)
+ signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+ ctx, cancel := context.WithCancel(context.Background())
+ go func() {
+ <-sigs
+ cancel()
+ }()
+ if err := launch.Launch(ctx, launch.Options{Ports: launch.IdentityPortMap()}); err != nil {
+ if err == ctx.Err() {
+ return
+ }
+ fmt.Printf("Failed to execute: %v\n", err)
+ }
+}
diff --git a/core/internal/common/setup.go b/core/internal/common/setup.go
index 7a268ae..db00692 100644
--- a/core/internal/common/setup.go
+++ b/core/internal/common/setup.go
@@ -27,6 +27,7 @@
MasterServicePort = 7833
ExternalServicePort = 7836
DebugServicePort = 7837
+ KubernetesAPIPort = 6443
)
const (
diff --git a/core/internal/consensus/BUILD.bazel b/core/internal/consensus/BUILD.bazel
index c8b2f25..f0246f7 100644
--- a/core/internal/consensus/BUILD.bazel
+++ b/core/internal/consensus/BUILD.bazel
@@ -18,6 +18,7 @@
"@io_etcd_go_etcd//pkg/types:go_default_library",
"@io_etcd_go_etcd//proxy/grpcproxy/adapter:go_default_library",
"@org_golang_x_sys//unix:go_default_library",
+ "@org_uber_go_atomic//:go_default_library",
"@org_uber_go_zap//:go_default_library",
"@org_uber_go_zap//zapcore:go_default_library",
],
diff --git a/core/internal/consensus/consensus.go b/core/internal/consensus/consensus.go
index 67bac1c..d401c1a 100644
--- a/core/internal/consensus/consensus.go
+++ b/core/internal/consensus/consensus.go
@@ -33,11 +33,6 @@
"strings"
"time"
- "git.monogon.dev/source/nexantic.git/core/internal/common"
- "git.monogon.dev/source/nexantic.git/core/internal/common/service"
-
- "git.monogon.dev/source/nexantic.git/core/generated/api"
-
"github.com/pkg/errors"
"go.etcd.io/etcd/clientv3"
"go.etcd.io/etcd/clientv3/namespace"
@@ -45,10 +40,14 @@
"go.etcd.io/etcd/etcdserver/api/membership"
"go.etcd.io/etcd/pkg/types"
"go.etcd.io/etcd/proxy/grpcproxy/adapter"
+ "go.uber.org/atomic"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"golang.org/x/sys/unix"
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "git.monogon.dev/source/nexantic.git/core/internal/common"
+ "git.monogon.dev/source/nexantic.git/core/internal/common/service"
"git.monogon.dev/source/nexantic.git/core/internal/consensus/ca"
)
@@ -75,7 +74,7 @@
etcd *embed.Etcd
kv clientv3.KV
- ready bool
+ ready atomic.Bool
// bootstrapCA and bootstrapCert cache the etcd cluster CA data during bootstrap.
bootstrapCA *ca.CA
@@ -192,6 +191,7 @@
go func() {
s.Logger.Info("waiting for etcd to become ready")
<-s.etcd.Server.ReadyNotify()
+ s.ready.Store(true)
s.Logger.Info("etcd is now ready")
}()
@@ -432,7 +432,7 @@
// IsReady returns whether etcd is ready and synced
func (s *Service) IsReady() bool {
- return s.ready
+ return s.ready.Load()
}
// AddMember adds a new etcd member to the cluster
diff --git a/core/internal/kubernetes/BUILD.bazel b/core/internal/kubernetes/BUILD.bazel
index 6778845..f3304cc 100644
--- a/core/internal/kubernetes/BUILD.bazel
+++ b/core/internal/kubernetes/BUILD.bazel
@@ -16,6 +16,7 @@
visibility = ["//core:__subpackages__"],
deps = [
"//core/api/api:go_default_library",
+ "//core/internal/common:go_default_library",
"//core/internal/common/supervisor:go_default_library",
"//core/internal/consensus:go_default_library",
"//core/internal/kubernetes/reconciler:go_default_library",
diff --git a/core/internal/kubernetes/apiserver.go b/core/internal/kubernetes/apiserver.go
index dc48b96..9bc32f3 100644
--- a/core/internal/kubernetes/apiserver.go
+++ b/core/internal/kubernetes/apiserver.go
@@ -26,6 +26,8 @@
"os/exec"
"path"
+ "git.monogon.dev/source/nexantic.git/core/internal/common"
+
"go.etcd.io/etcd/clientv3"
"git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
@@ -81,6 +83,7 @@
"--enable-admission-plugins=NodeRestriction,PodSecurityPolicy",
"--enable-aggregator-routing=true",
"--insecure-port=0",
+ fmt.Sprintf("--secure-port=%v", common.KubernetesAPIPort),
// Due to the magic of GRPC this really needs four slashes and a :0
fmt.Sprintf("--etcd-servers=%v", "unix:////consensus/listener.sock:0"),
args.FileOpt("--kubelet-client-certificate", "kubelet-client-cert.pem",
diff --git a/core/internal/kubernetes/auth.go b/core/internal/kubernetes/auth.go
index 25e2e4b..fe2fe59 100644
--- a/core/internal/kubernetes/auth.go
+++ b/core/internal/kubernetes/auth.go
@@ -34,6 +34,8 @@
"path"
"time"
+ "git.monogon.dev/source/nexantic.git/core/internal/common"
+
"go.etcd.io/etcd/clientv3"
"k8s.io/client-go/tools/clientcmd"
configapi "k8s.io/client-go/tools/clientcmd/api"
@@ -381,7 +383,7 @@
func makeLocalKubeconfig(ca, cert, key []byte) ([]byte, error) {
kubeconfig := configapi.NewConfig()
cluster := configapi.NewCluster()
- cluster.Server = "https://127.0.0.1:6443"
+ cluster.Server = fmt.Sprintf("https://127.0.0.1:%v", common.KubernetesAPIPort)
cluster.CertificateAuthorityData = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca})
kubeconfig.Clusters["default"] = cluster
authInfo := configapi.NewAuthInfo()
diff --git a/core/internal/kubernetes/service.go b/core/internal/kubernetes/service.go
index f95f03e..b2d340e 100644
--- a/core/internal/kubernetes/service.go
+++ b/core/internal/kubernetes/service.go
@@ -95,6 +95,9 @@
// GetDebugKubeconfig issues a kubeconfig for an arbitrary given identity. Useful for debugging and testing.
func (s *Service) GetDebugKubeconfig(ctx context.Context, request *schema.GetDebugKubeconfigRequest) (*schema.GetDebugKubeconfigResponse, error) {
+ if !s.consensusService.IsReady() {
+ return nil, status.Error(codes.Unavailable, "Consensus not ready yet")
+ }
idCA, idKeyRaw, err := getCert(s.getKV(), "id-ca")
idKey := ed25519.PrivateKey(idKeyRaw)
if err != nil {
diff --git a/core/internal/launch/BUILD.bazel b/core/internal/launch/BUILD.bazel
new file mode 100644
index 0000000..887932b
--- /dev/null
+++ b/core/internal/launch/BUILD.bazel
@@ -0,0 +1,12 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["launch.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/internal/launch",
+ visibility = ["//core:__subpackages__"],
+ deps = [
+ "//core/internal/common:go_default_library",
+ "@org_golang_google_grpc//:go_default_library",
+ ],
+)
diff --git a/core/internal/launch/launch.go b/core/internal/launch/launch.go
new file mode 100644
index 0000000..9aa277c
--- /dev/null
+++ b/core/internal/launch/launch.go
@@ -0,0 +1,227 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package launch
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+
+ "google.golang.org/grpc"
+
+ "git.monogon.dev/source/nexantic.git/core/internal/common"
+)
+
+// This is more of a best-effort solution and not guaranteed to give us unused ports (since we're not immediately using
+// them), but AFAIK qemu cannot dynamically select hostfwd ports
+func getFreePort() (uint16, io.Closer, error) {
+ addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
+ if err != nil {
+ return 0, nil, err
+ }
+
+ l, err := net.ListenTCP("tcp", addr)
+ if err != nil {
+ return 0, nil, err
+ }
+ return uint16(l.Addr().(*net.TCPAddr).Port), l, nil
+}
+
+type qemuValue map[string][]string
+
+// qemuValueToOption encodes structured data into a QEMU option.
+// Example: "test", {"key1": {"val1"}, "key2": {"val2", "val3"}} returns "test,key1=val1,key2=val2,key2=val3"
+func qemuValueToOption(name string, value qemuValue) string {
+ var optionValues []string
+ optionValues = append(optionValues, name)
+ for name, values := range value {
+ if len(values) == 0 {
+ optionValues = append(optionValues, name)
+ }
+ for _, val := range values {
+ optionValues = append(optionValues, fmt.Sprintf("%v=%v", name, val))
+ }
+ }
+ return strings.Join(optionValues, ",")
+}
+
+func copyFile(src, dst string) error {
+ in, err := os.Open(src)
+ if err != nil {
+ return err
+ }
+ defer in.Close()
+
+ out, err := os.Create(dst)
+ if err != nil {
+ return err
+ }
+ defer out.Close()
+
+ _, err = io.Copy(out, in)
+ if err != nil {
+ return err
+ }
+ return out.Close()
+}
+
+// PortMap represents where VM ports are mapped to on the host. It maps from the VM port number to the host port number.
+type PortMap map[uint16]uint16
+
+// toQemuForwards generates QEMU hostfwd values (https://qemu.weilnetz.de/doc/qemu-doc.html#:~:text=hostfwd=) for all
+// mapped ports.
+func (p PortMap) toQemuForwards() []string {
+ var hostfwdOptions []string
+ for vmPort, hostPort := range p {
+ hostfwdOptions = append(hostfwdOptions, fmt.Sprintf("tcp::%v-:%v", hostPort, vmPort))
+ }
+ return hostfwdOptions
+}
+
+// DialGRPC creates a gRPC client for a VM port that's forwarded/mapped to the host. The given port is automatically
+// resolved to the host-mapped port.
+func (p PortMap) DialGRPC(port uint16, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
+ mappedPort, ok := p[port]
+ if !ok {
+ return nil, fmt.Errorf("cannot dial port: port %v is not mapped/forwarded", port)
+ }
+ grpcClient, err := grpc.Dial(fmt.Sprintf("localhost:%v", mappedPort), opts...)
+ if err != nil {
+ return nil, fmt.Errorf("failed to dial port %v: %w", port, err)
+ }
+ return grpcClient, nil
+}
+
+// Options contains all options that can be passed to Launch()
+type Options struct {
+ // Ports contains the port mapping where to expose the internal ports of the VM to the host. See IdentityPortMap()
+ // and ConflictFreePortMap()
+ Ports PortMap
+
+ // If set to true, reboots are honored. Otherwise all reboots exit the Launch() command. Smalltown generally restarts
+ // on almost all errors, so unless you want to test reboot behavior this should be false.
+ AllowReboot bool
+}
+
+var requiredPorts = []uint16{common.ConsensusPort, common.NodeServicePort, common.MasterServicePort,
+ common.ExternalServicePort, common.DebugServicePort, common.KubernetesAPIPort}
+
+// IdentityPortMap returns a port map where each VM port is mapped onto itself on the host. This is mainly useful
+// for development against Smalltown. The dbg command requires this mapping.
+func IdentityPortMap() PortMap {
+ portMap := make(PortMap)
+ for _, port := range requiredPorts {
+ portMap[port] = port
+ }
+ return portMap
+}
+
+// ConflictFreePortMap returns a port map where each VM port is mapped onto a random free port on the host. This is
+// intended for automated testing where multiple instances of Smalltown might be running. Please call this function for
+// each Launch command separately and as close to it as possible since it cannot guarantee that the ports will remain
+// free.
+func ConflictFreePortMap() (PortMap, error) {
+ portMap := make(PortMap)
+ for _, port := range requiredPorts {
+ mappedPort, listenCloser, err := getFreePort()
+ if err != nil {
+ return portMap, fmt.Errorf("failed to get free host port: %w", err)
+ }
+ // Defer closing of the listening port until the function is done and all ports are allocated
+ defer listenCloser.Close()
+ portMap[port] = mappedPort
+ }
+ return portMap, nil
+}
+
+// Launch launches a Smalltown instance with the given options. The instance runs mostly paravirtualized but with some
+// emulated hardware similar to how a cloud provider might set up its VMs. The disk is fully writable but is run
+// in snapshot mode meaning that changes are not kept beyond a single invocation.
+func Launch(ctx context.Context, options Options) error {
+ // Pin temp directory to /tmp until we can use abstract socket namespace in QEMU (next release after 5.0,
+ // https://github.com/qemu/qemu/commit/776b97d3605ed0fc94443048fdf988c7725e38a9). swtpm accepts already-open FDs
+ // so we can pass in an abstract socket namespace FD that we open and pass the name of it to QEMU. Not pinning this
+ // crashes both swtpm and qemu because we run into UNIX socket length limitations (for legacy reasons 108 chars).
+ tempDir, err := ioutil.TempDir("/tmp", "launch*")
+ if err != nil {
+ return fmt.Errorf("Failed to create temporary directory: %w", err)
+ }
+ defer os.RemoveAll(tempDir)
+
+ // Copy TPM state into a temporary directory since it's being modified by the emulator
+ tpmTargetDir := filepath.Join(tempDir, "tpm")
+ tpmSrcDir := "core/tpm"
+ if err := os.Mkdir(tpmTargetDir, 0644); err != nil {
+ return fmt.Errorf("Failed to create TPM state directory: %w", err)
+ }
+ tpmFiles, err := ioutil.ReadDir(tpmSrcDir)
+ if err != nil {
+ return fmt.Errorf("Failed to read TPM directory: %w", err)
+ }
+ for _, file := range tpmFiles {
+ name := file.Name()
+ if err := copyFile(filepath.Join(tpmSrcDir, name), filepath.Join(tpmTargetDir, name)); err != nil {
+ return fmt.Errorf("Failed to copy TPM directory: %w", err)
+ }
+ }
+
+ qemuNetConfig := qemuValue{
+ "id": {"net0"},
+ "net": {"10.42.0.0/24"},
+ "dhcpstart": {"10.42.0.10"},
+ "hostfwd": options.Ports.toQemuForwards(),
+ }
+
+ tpmSocketPath := filepath.Join(tempDir, "tpm-socket")
+
+ qemuArgs := []string{"-machine", "q35", "-accel", "kvm", "-nographic", "-nodefaults", "-m", "2048",
+ "-cpu", "host", "-smp", "sockets=1,cpus=1,cores=2,threads=2,maxcpus=4",
+ "-drive", "if=pflash,format=raw,readonly,file=external/edk2/OVMF_CODE.fd",
+ "-drive", "if=pflash,format=raw,snapshot=on,file=external/edk2/OVMF_VARS.fd",
+ "-drive", "if=virtio,format=raw,snapshot=on,cache=unsafe,file=core/smalltown.img",
+ "-netdev", qemuValueToOption("user", qemuNetConfig),
+ "-device", "virtio-net-pci,netdev=net0",
+ "-chardev", "socket,id=chrtpm,path=" + tpmSocketPath,
+ "-tpmdev", "emulator,id=tpm0,chardev=chrtpm",
+ "-device", "tpm-tis,tpmdev=tpm0",
+ "-device", "virtio-rng-pci",
+ "-serial", "stdio"}
+
+ if !options.AllowReboot {
+ qemuArgs = append(qemuArgs, "-no-reboot")
+ }
+
+ tpmCtx, tpmStop := context.WithCancel(
+ ctx)
+ tpmEmuCmd := exec.CommandContext(tpmCtx, "swtpm", "socket", "--tpm2", "--tpmstate", "dir="+tpmTargetDir, "--ctrl", "type=unixio,path="+tpmSocketPath)
+ systemCmd := exec.CommandContext(ctx, "qemu-system-x86_64", qemuArgs...)
+
+ tpmEmuCmd.Stderr = os.Stderr
+ tpmEmuCmd.Stdout = os.Stdout
+ systemCmd.Stderr = os.Stderr
+ systemCmd.Stdout = os.Stdout
+ go tpmEmuCmd.Run()
+ err = systemCmd.Run()
+ tpmStop()
+ return err
+}
diff --git a/core/scripts/BUILD b/core/scripts/BUILD
deleted file mode 100644
index 87ba08b..0000000
--- a/core/scripts/BUILD
+++ /dev/null
@@ -1,34 +0,0 @@
-sh_library(
- name = "vm_deps",
- data = [
- "//core:image",
- "//core:swtpm_data",
- "//third_party/edk2:firmware",
- ],
-)
-
-sh_binary(
- name = "launch",
- srcs = ["launch.sh"],
- deps = [":vm_deps"],
-)
-
-sh_library(
- name = "test_deps",
- data = [
- ":launch",
- "@io_k8s_kubernetes//cmd/kubectl",
- ],
-)
-
-sh_test(
- name = "test_boot",
- size = "medium",
- srcs = ["test_boot.sh"],
- # expects wants a pty, which do not exist in the sandbox
- tags = ["local"],
- deps = [
- ":test_deps",
- ":vm_deps",
- ],
-)
diff --git a/core/scripts/launch.sh b/core/scripts/launch.sh
deleted file mode 100755
index 3fb3b57..0000000
--- a/core/scripts/launch.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/sh
-
-TMP=$(mktemp -d)
-trap "{ rm -rf "$TMP"; pkill -9 -P $$; }" EXIT
-
-# sandbox uses a symlink farm - without -L, we would just copy the symlinks
-cp -Lr core/tpm/* "${TMP}"
-
-swtpm socket --tpmstate dir=${TMP} --ctrl type=unixio,path=tpm-socket --tpm2 &
-
-qemu-system-x86_64 \
- -cpu host -smp sockets=1,cpus=1,cores=2,threads=2,maxcpus=4 -m 2048 -machine q35 -enable-kvm -nographic -nodefaults \
- -drive if=pflash,format=raw,readonly,file=external/edk2/OVMF_CODE.fd \
- -drive if=pflash,format=raw,snapshot=on,file=external/edk2/OVMF_VARS.fd \
- -drive if=virtio,format=raw,snapshot=on,cache=unsafe,file=core/smalltown.img \
- -netdev user,id=net0,net=10.42.0.0/24,dhcpstart=10.42.0.10,hostfwd=tcp::7833-:7833,hostfwd=tcp::7834-:7834,hostfwd=tcp::6443-:6443,hostfwd=tcp::7835-:7835,hostfwd=tcp::7837-:7837 \
- -device virtio-net-pci,netdev=net0 \
- -chardev socket,id=chrtpm,path=tpm-socket \
- -tpmdev emulator,id=tpm0,chardev=chrtpm \
- -device tpm-tis,tpmdev=tpm0 \
- -debugcon file:debug.log \
- -global isa-debugcon.iobase=0x402 \
- -device ipmi-bmc-sim,id=ipmi0 \
- -device virtio-rng-pci \
- -serial stdio \
- | stdbuf -oL tr -d '\r' | cat -v
diff --git a/core/scripts/test_boot.sh b/core/scripts/test_boot.sh
deleted file mode 100755
index 36ab079..0000000
--- a/core/scripts/test_boot.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/expect -f
-
-# Getting the actual path from a sh_test rule is not straight-forward and would involve
-# parsing the runfile at $RUNFILES_DIR, so just hardcode it.
-#
-# We'll want to replace this thing by a proper e2e testing suite sooner than we'll
-# have to worry about cross-compilation or varying build environments.
-#
-# (see https://github.com/bazelbuild/bazel/blob/master/tools/bash/runfiles/runfiles.bash)
-set kubectl_path "external/io_k8s_kubernetes/cmd/kubectl/kubectl_/kubectl"
-
-set timeout 120
-
-proc print_stderr {msg} {
- send_error "\[TEST\] $msg\n"
-}
-
-spawn core/scripts/launch.sh
-
-expect "DHCP client ASSIGNED" {} default {
- print_stderr "Failed while waiting for IP address\n"
- exit 1
-}
-
-expect "Initialized encrypted storage" {} default {
- print_stderr "Failed while waiting for encrypted storage\n"
- exit 1
-}
-
-# Make an educated guess if the control plane came up
-expect -timeout 3 "\n" {
- exp_continue
-} timeout {} default {
- print_stderr "Failed while waiting for k8s control plane\n"
- exit 1
-}
-
-spawn $kubectl_path cluster-info dump -s https://localhost:6443 --username none --password none --insecure-skip-tls-verify=true
-
-expect "User \"system:anonymous\" cannot list resource \"nodes\" in API group \"\" at the cluster scope" {} default {
- print_stderr "Failed while waiting for kubectl test\n"
- exit 1
-}
-
-print_stderr "Completed successfully"
-exit 0
diff --git a/core/tests/e2e/BUILD.bazel b/core/tests/e2e/BUILD.bazel
new file mode 100644
index 0000000..3e594fc
--- /dev/null
+++ b/core/tests/e2e/BUILD.bazel
@@ -0,0 +1,43 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "go_default_library",
+ srcs = [
+ "condition_helpers.go",
+ "kubernetes_helpers.go",
+ "utils.go",
+ ],
+ importpath = "git.monogon.dev/source/nexantic.git/core/tests/e2e",
+ visibility = ["//visibility:private"],
+ deps = [
+ "//core/api/api:go_default_library",
+ "@io_k8s_api//apps/v1:go_default_library",
+ "@io_k8s_api//core/v1:go_default_library",
+ "@io_k8s_apimachinery//pkg/api/resource:go_default_library",
+ "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+ "@io_k8s_apimachinery//pkg/util/intstr:go_default_library",
+ "@io_k8s_client_go//kubernetes:go_default_library",
+ "@io_k8s_client_go//tools/clientcmd:go_default_library",
+ ],
+)
+
+go_test(
+ name = "go_default_test",
+ srcs = ["main_test.go"],
+ data = [
+ "//core:image",
+ "//core:swtpm_data",
+ "//third_party/edk2:firmware",
+ ],
+ embed = [":go_default_library"],
+ rundir = ".",
+ deps = [
+ "//core/api/api:go_default_library",
+ "//core/internal/common:go_default_library",
+ "//core/internal/launch:go_default_library",
+ "@io_k8s_api//core/v1:go_default_library",
+ "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+ "@io_k8s_kubernetes//pkg/api/v1/pod:go_default_library",
+ "@org_golang_google_grpc//:go_default_library",
+ ],
+)
diff --git a/core/tests/e2e/condition_helpers.go b/core/tests/e2e/condition_helpers.go
new file mode 100644
index 0000000..f7d5c8e
--- /dev/null
+++ b/core/tests/e2e/condition_helpers.go
@@ -0,0 +1,46 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ apipb "git.monogon.dev/source/nexantic.git/core/generated/api"
+)
+
+func waitForCondition(ctx context.Context, client apipb.NodeDebugServiceClient, condition string) error {
+ var lastErr = errors.New("No RPC for checking condition completed")
+ for {
+ res, err := client.GetCondition(ctx, &apipb.GetConditionRequest{Name: condition})
+ if err != nil {
+ if err == ctx.Err() {
+ return err
+ }
+ lastErr = err
+ }
+ if err == nil && res.Ok {
+ return nil
+ }
+ select {
+ case <-time.After(1 * time.Second):
+ case <-ctx.Done():
+ return lastErr
+ }
+ }
+}
diff --git a/core/tests/e2e/kubernetes_helpers.go b/core/tests/e2e/kubernetes_helpers.go
new file mode 100644
index 0000000..264793a
--- /dev/null
+++ b/core/tests/e2e/kubernetes_helpers.go
@@ -0,0 +1,144 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "time"
+
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/util/intstr"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/tools/clientcmd"
+
+ apipb "git.monogon.dev/source/nexantic.git/core/generated/api"
+)
+
+// getKubeClientSet gets a Kubeconfig from the debug API and creates a K8s ClientSet using it. The identity used has
+// the system:masters group and thus has RBAC access to everything.
+func getKubeClientSet(ctx context.Context, client apipb.NodeDebugServiceClient, port uint16) (kubernetes.Interface, error) {
+ var lastErr = errors.New("context canceled before any operation completed")
+ for {
+ res, err := client.GetDebugKubeconfig(context.Background(), &apipb.GetDebugKubeconfigRequest{Id: "debug-user", Groups: []string{"system:masters"}})
+ if err == nil {
+ rawClientConfig, err := clientcmd.NewClientConfigFromBytes([]byte(res.DebugKubeconfig))
+ if err != nil {
+ return nil, err // Invalid Kubeconfigs are immediately fatal
+ }
+
+ clientConfig, err := rawClientConfig.ClientConfig()
+ clientConfig.Host = fmt.Sprintf("localhost:%v", port)
+ clientSet, err := kubernetes.NewForConfig(clientConfig)
+ if err != nil {
+ return nil, err
+ }
+ return clientSet, nil
+ }
+ if err != nil && err == ctx.Err() {
+ return nil, lastErr
+ }
+ lastErr = err
+ select {
+ case <-ctx.Done():
+ return nil, lastErr
+ case <-time.After(1 * time.Second):
+ }
+ }
+}
+
+// makeTestDeploymentSpec generates a Deployment spec for a single pod running NGINX with a readiness probe. This allows
+// verifying that the control plane is capable of scheduling simple pods and that kubelet works, its runtime is set up
+// well enough to run a simple container and the network to the pod can pass readiness probe traffic.
+func makeTestDeploymentSpec(name string) *appsv1.Deployment {
+ return &appsv1.Deployment{
+ ObjectMeta: metav1.ObjectMeta{Name: name},
+ Spec: appsv1.DeploymentSpec{
+ Selector: &metav1.LabelSelector{MatchLabels: map[string]string{
+ "name": name,
+ }},
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: map[string]string{
+ "name": name,
+ },
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "test",
+ // TODO(phab/T793): Build and preseed our own container images
+ Image: "nginx:alpine",
+ ReadinessProbe: &corev1.Probe{
+ Handler: corev1.Handler{
+ HTTPGet: &corev1.HTTPGetAction{Port: intstr.FromInt(80)},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+}
+
+// makeTestStatefulSet generates a StatefulSet spec
+func makeTestStatefulSet(name string) *appsv1.StatefulSet {
+ return &appsv1.StatefulSet{
+ ObjectMeta: metav1.ObjectMeta{Name: name},
+ Spec: appsv1.StatefulSetSpec{
+ Selector: &metav1.LabelSelector{MatchLabels: map[string]string{
+ "name": name,
+ }},
+ VolumeClaimTemplates: []corev1.PersistentVolumeClaim{
+ {
+ ObjectMeta: metav1.ObjectMeta{Name: "www"},
+ Spec: corev1.PersistentVolumeClaimSpec{
+ AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
+ Resources: corev1.ResourceRequirements{
+ Requests: map[corev1.ResourceName]resource.Quantity{corev1.ResourceStorage: resource.MustParse("50Mi")},
+ },
+ },
+ },
+ },
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: map[string]string{
+ "name": name,
+ },
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "test",
+ Image: "nginx:alpine",
+ ReadinessProbe: &corev1.Probe{
+ Handler: corev1.Handler{
+ HTTPGet: &corev1.HTTPGetAction{Port: intstr.FromInt(80)},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+}
diff --git a/core/tests/e2e/main_test.go b/core/tests/e2e/main_test.go
new file mode 100644
index 0000000..d400b9b
--- /dev/null
+++ b/core/tests/e2e/main_test.go
@@ -0,0 +1,169 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log"
+ "net/http"
+ _ "net/http"
+ _ "net/http/pprof"
+ "os"
+ "testing"
+ "time"
+
+ "google.golang.org/grpc"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ podv1 "k8s.io/kubernetes/pkg/api/v1/pod"
+
+ apipb "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "git.monogon.dev/source/nexantic.git/core/internal/common"
+ "git.monogon.dev/source/nexantic.git/core/internal/launch"
+)
+
+// TestE2E is the main E2E test entrypoint for single-node freshly-bootstrapped E2E tests. It starts a full Smalltown node
+// in bootstrap mode and then runs tests against it. The actual tests it performs are located in the RunGroup subtest.
+func TestE2E(t *testing.T) {
+ go func() {
+ log.Println(http.ListenAndServe("localhost:0", nil))
+ }()
+ // Set a global timeout to make sure this terminates
+ ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+ defer cancel()
+ portMap, err := launch.ConflictFreePortMap()
+ if err != nil {
+ t.Fatalf("Failed to acquire ports for e2e test: %v", err)
+ }
+ go func() {
+ if err := launch.Launch(ctx, launch.Options{Ports: portMap}); err != nil {
+ panic(err)
+ }
+ }()
+ grpcClient, err := portMap.DialGRPC(common.DebugServicePort, grpc.WithInsecure())
+ if err != nil {
+ fmt.Printf("Failed to dial debug service (is it running): %v\n", err)
+ }
+ debugClient := apipb.NewNodeDebugServiceClient(grpcClient)
+
+ go func() {
+ <-ctx.Done()
+ fmt.Fprintf(os.Stderr, "Main context canceled\n")
+ }()
+
+ // This exists to keep the parent around while all the children race
+ // It currently tests both a set of OS-level conditions and Kubernetes Deployments and StatefulSets
+ t.Run("RunGroup", func(t *testing.T) {
+ t.Run("IP available", func(t *testing.T) {
+ t.Parallel()
+ const timeoutSec = 10
+ ctx, cancel := context.WithTimeout(ctx, timeoutSec*time.Second)
+ defer cancel()
+ if err := waitForCondition(ctx, debugClient, "IPAssigned"); err != nil {
+ t.Errorf("Condition IPAvailable not met in %vs: %v", timeoutSec, err)
+ }
+ })
+ t.Run("Data available", func(t *testing.T) {
+ t.Parallel()
+ const timeoutSec = 30
+ ctx, cancel := context.WithTimeout(ctx, timeoutSec*time.Second)
+ defer cancel()
+ if err := waitForCondition(ctx, debugClient, "DataAvailable"); err != nil {
+ t.Errorf("Condition DataAvailable not met in %vs: %v", timeoutSec, err)
+ }
+ })
+ t.Run("Get Kubernetes Debug Kubeconfig", func(t *testing.T) {
+ t.Parallel()
+ selfCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ defer cancel()
+ clientSet, err := getKubeClientSet(selfCtx, debugClient, portMap[common.KubernetesAPIPort])
+ if err != nil {
+ t.Fatal(err)
+ }
+ testEventual(t, "Node is registered and ready", ctx, 30*time.Second, func(ctx context.Context) error {
+ nodes, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+ if err != nil {
+ return err
+ }
+ if len(nodes.Items) < 1 {
+ return errors.New("node not registered")
+ }
+ if len(nodes.Items) > 1 {
+ return errors.New("more than one node registered (but there is only one)")
+ }
+ node := nodes.Items[0]
+ for _, cond := range node.Status.Conditions {
+ if cond.Type != corev1.NodeReady {
+ continue
+ }
+ if cond.Status != corev1.ConditionTrue {
+ return fmt.Errorf("node not ready: %v", cond.Message)
+ }
+ }
+ return nil
+ })
+ testEventual(t, "Simple deployment", ctx, 30*time.Second, func(ctx context.Context) error {
+ _, err := clientSet.AppsV1().Deployments("default").Create(ctx, makeTestDeploymentSpec("test-deploy-1"), metav1.CreateOptions{})
+ return err
+ })
+ testEventual(t, "Simple deployment is running", ctx, 40*time.Second, func(ctx context.Context) error {
+ res, err := clientSet.CoreV1().Pods("default").List(ctx, metav1.ListOptions{LabelSelector: "name=test-deploy-1"})
+ if err != nil {
+ return err
+ }
+ if len(res.Items) == 0 {
+ return errors.New("pod didn't get created")
+ }
+ pod := res.Items[0]
+ if podv1.IsPodAvailable(&pod, 1, metav1.NewTime(time.Now())) {
+ return nil
+ }
+ events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
+ if err != nil || len(events.Items) == 0 {
+ return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
+ } else {
+ return fmt.Errorf("pod is not ready: %v", events.Items[0].Message)
+ }
+ })
+ testEventual(t, "Simple StatefulSet with PVC", ctx, 30*time.Second, func(ctx context.Context) error {
+ _, err := clientSet.AppsV1().StatefulSets("default").Create(ctx, makeTestStatefulSet("test-statefulset-1"), metav1.CreateOptions{})
+ return err
+ })
+ testEventual(t, "Simple StatefulSet with PVC is running", ctx, 40*time.Second, func(ctx context.Context) error {
+ res, err := clientSet.CoreV1().Pods("default").List(ctx, metav1.ListOptions{LabelSelector: "name=test-statefulset-1"})
+ if err != nil {
+ return err
+ }
+ if len(res.Items) == 0 {
+ return errors.New("pod didn't get created")
+ }
+ pod := res.Items[0]
+ if podv1.IsPodAvailable(&pod, 1, metav1.NewTime(time.Now())) {
+ return nil
+ }
+ events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
+ if err != nil || len(events.Items) == 0 {
+ return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
+ } else {
+ return fmt.Errorf("pod is not ready: %v", events.Items[0].Message)
+ }
+ })
+ })
+ })
+}
diff --git a/core/tests/e2e/utils.go b/core/tests/e2e/utils.go
new file mode 100644
index 0000000..f888189
--- /dev/null
+++ b/core/tests/e2e/utils.go
@@ -0,0 +1,51 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+)
+
+// testEventual creates a new subtest looping the given function until it either doesn't return an error anymore or
+// the timeout is exceeded. The last returned non-context-related error is being used as the test error.
+func testEventual(t *testing.T, name string, ctx context.Context, timeout time.Duration, f func(context.Context) error) {
+ ctx, cancel := context.WithTimeout(ctx, timeout)
+ t.Helper()
+ t.Run(name, func(t *testing.T) {
+ defer cancel()
+ var lastErr = errors.New("test didn't run to completion at least once")
+ t.Parallel()
+ for {
+ err := f(ctx)
+ if err == nil {
+ return
+ }
+ if err == ctx.Err() {
+ t.Fatal(lastErr)
+ }
+ lastErr = err
+ select {
+ case <-ctx.Done():
+ t.Fatal(lastErr)
+ case <-time.After(1 * time.Second):
+ }
+ }
+ })
+}