Add VM infrastructure smoke test
This adds an E2E test which exercises the VM infrastructure (Kubernetes, KVM device plugin and QEMU).
This test should ensure that nobody breaks the core infrastructure Metropolis VMs rely on.
Test Plan: This is a test
X-Origin-Diff: phab/D740
GitOrigin-RevId: ddf629725dfb664ace5a50efee9ed9442962d6f7
diff --git a/metropolis/node/BUILD.bazel b/metropolis/node/BUILD.bazel
index f037cf6..b5ed532 100644
--- a/metropolis/node/BUILD.bazel
+++ b/metropolis/node/BUILD.bazel
@@ -64,6 +64,7 @@
# Containerd preseed bundles
"//metropolis/test/e2e/preseedtest:preseedtest.tar": "/containerd/preseed/k8s.io/preseedtest.tar",
"//metropolis/test/e2e/k8s_cts:k8s_cts_image.tar": "/containerd/preseed/k8s.io/k8s_cts.tar",
+ "//metropolis/vm/smoketest:smoketest_container.tar": "/containerd/preseed/k8s.io/smoketest.tar",
# CNI Plugins
"@com_github_containernetworking_plugins//plugins/main/loopback": "/containerd/bin/cni/loopback",
diff --git a/metropolis/test/e2e/BUILD.bazel b/metropolis/test/e2e/BUILD.bazel
index 25a7dcd..117bbe9 100644
--- a/metropolis/test/e2e/BUILD.bazel
+++ b/metropolis/test/e2e/BUILD.bazel
@@ -36,6 +36,7 @@
"//metropolis/proto/api:go_default_library",
"//metropolis/test/launch:go_default_library",
"@io_k8s_api//core/v1:go_default_library",
+ "@io_k8s_apimachinery//pkg/api/resource:go_default_library",
"@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
"@io_k8s_kubernetes//pkg/api/v1/pod:go_default_library",
"@org_golang_google_grpc//:go_default_library",
diff --git a/metropolis/test/e2e/main_test.go b/metropolis/test/e2e/main_test.go
index 3e4ffb0..46e5c72 100644
--- a/metropolis/test/e2e/main_test.go
+++ b/metropolis/test/e2e/main_test.go
@@ -32,6 +32,7 @@
"google.golang.org/grpc"
corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
podv1 "k8s.io/kubernetes/pkg/api/v1/pod"
@@ -242,6 +243,46 @@
return fmt.Errorf("pod is not ready: %v", events.Items[len(events.Items)-1].Message)
}
})
+ if os.Getenv("HAVE_NESTED_KVM") != "" {
+ testEventual(t, "Pod for KVM/QEMU smoke test", ctx, smallTestTimeout, func(ctx context.Context) error {
+ runcRuntimeClass := "runc"
+ _, err := clientSet.CoreV1().Pods("default").Create(ctx, &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "vm-smoketest",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{{
+ Name: "vm-smoketest",
+ ImagePullPolicy: corev1.PullNever,
+ Image: "bazel/metropolis/vm/smoketest:smoketest_container",
+ Resources: corev1.ResourceRequirements{
+ Limits: corev1.ResourceList{
+ "devices.monogon.dev/kvm": *resource.NewQuantity(1, ""),
+ },
+ },
+ }},
+ RuntimeClassName: &runcRuntimeClass,
+ RestartPolicy: corev1.RestartPolicyNever,
+ },
+ }, metav1.CreateOptions{})
+ return err
+ })
+ testEventual(t, "KVM/QEMU smoke test completion", ctx, smallTestTimeout, func(ctx context.Context) error {
+ pod, err := clientSet.CoreV1().Pods("default").Get(ctx, "vm-smoketest", metav1.GetOptions{})
+ if err != nil {
+ return fmt.Errorf("failed to get pod: %w", err)
+ }
+ if pod.Status.Phase == corev1.PodSucceeded {
+ return nil
+ }
+ events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
+ if err != nil || len(events.Items) == 0 {
+ return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
+ } else {
+ return fmt.Errorf("pod is not ready: %v", events.Items[len(events.Items)-1].Message)
+ }
+ })
+ }
})
})
diff --git a/metropolis/vm/BUILD b/metropolis/vm/BUILD
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/metropolis/vm/BUILD
diff --git a/metropolis/vm/smoketest/BUILD.bazel b/metropolis/vm/smoketest/BUILD.bazel
new file mode 100644
index 0000000..7fe529d
--- /dev/null
+++ b/metropolis/vm/smoketest/BUILD.bazel
@@ -0,0 +1,43 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+load("@io_bazel_rules_docker//container:container.bzl", "container_image")
+load("//metropolis/node/build:def.bzl", "node_initramfs")
+load("//build/static_binary_tarball:def.bzl", "static_binary_tarball")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["main.go"],
+ importpath = "source.monogon.dev/metropolis/vm/smoketest",
+ visibility = ["//visibility:private"],
+)
+
+node_initramfs(
+ name = "initramfs",
+ files = {
+ "//metropolis/vm/smoketest/payload": "/init",
+ },
+)
+
+go_binary(
+ name = "smoketest",
+ data = [
+ ":initramfs",
+ "//metropolis/test/ktest:linux-testing",
+ "@qemu//:qemu-x86_64-softmmu",
+ ],
+ embed = [":go_default_library"],
+ visibility = ["//visibility:public"],
+)
+
+static_binary_tarball(
+ name = "smoketest_layer",
+ executable = ":smoketest",
+)
+
+container_image(
+ name = "smoketest_container",
+ base = "@go_image_base//image",
+ entrypoint = ["/app/metropolis/vm/smoketest/smoketest_/smoketest"],
+ tars = [":smoketest_layer"],
+ visibility = ["//visibility:public"],
+ workdir = "/app",
+)
diff --git a/metropolis/vm/smoketest/main.go b/metropolis/vm/smoketest/main.go
new file mode 100644
index 0000000..d9ff7e3
--- /dev/null
+++ b/metropolis/vm/smoketest/main.go
@@ -0,0 +1,77 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is a small smoke test which will run in a container on top of Metropolis Kubernetes. It exercises Metropolis'
+// KVM device plugin,
+package main
+
+import (
+ "bytes"
+ "io/ioutil"
+ "log"
+ "net"
+ "os"
+ "os/exec"
+)
+
+func main() {
+ testSocket, err := net.Listen("unix", "@metropolis/vm/smoketest")
+ if err != nil {
+ panic(err)
+ }
+
+ testResultChan := make(chan bool)
+ go func() {
+ conn, err := testSocket.Accept()
+ if err != nil {
+ panic(err)
+ }
+ testValue, _ := ioutil.ReadAll(conn)
+ if bytes.Equal(testValue, []byte("test123")) {
+ testResultChan <- true
+ } else {
+ testResultChan <- false
+ }
+ }()
+
+ baseArgs := []string{"-nodefaults", "-no-user-config", "-nographic", "-no-reboot",
+ "-accel", "kvm", "-cpu", "host",
+ // TODO(lorenz): This explicitly doesn't use our own qboot because it cannot be built in a musl configuration.
+ // This will be fixed once we have a proper multi-target toolchain.
+ "-bios", "external/qemu/pc-bios/qboot.rom",
+ "-M", "microvm,x-option-roms=off,pic=off,pit=off,rtc=off,isa-serial=off",
+ "-kernel", "metropolis/test/ktest/linux-testing.elf",
+ "-append", "reboot=t console=hvc0 quiet",
+ "-initrd", "metropolis/vm/smoketest/initramfs.lz4",
+ "-device", "virtio-rng-device,max-bytes=1024,period=1000",
+ "-device", "virtio-serial-device,max_ports=16",
+ "-chardev", "stdio,id=con0", "-device", "virtconsole,chardev=con0",
+ "-chardev", "socket,id=test,path=metropolis/vm/smoketest,abstract=on",
+ "-device", "virtserialport,chardev=test",
+ }
+ qemuCmd := exec.Command("external/qemu/qemu-x86_64-softmmu", baseArgs...)
+ qemuCmd.Stdout = os.Stdout
+ qemuCmd.Stderr = os.Stderr
+ if err := qemuCmd.Run(); err != nil {
+ log.Fatalf("running QEMU failed: %v", err)
+ }
+ testResult := <-testResultChan
+ if testResult {
+ return
+ } else {
+ os.Exit(1)
+ }
+}
diff --git a/metropolis/vm/smoketest/payload/BUILD.bazel b/metropolis/vm/smoketest/payload/BUILD.bazel
new file mode 100644
index 0000000..52b27e1
--- /dev/null
+++ b/metropolis/vm/smoketest/payload/BUILD.bazel
@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["main.go"],
+ importpath = "source.monogon.dev/metropolis/vm/smoketest/payload",
+ visibility = ["//visibility:private"],
+ deps = ["@org_golang_x_sys//unix:go_default_library"],
+)
+
+go_binary(
+ name = "payload",
+ embed = [":go_default_library"],
+ visibility = ["//visibility:public"],
+)
diff --git a/metropolis/vm/smoketest/payload/main.go b/metropolis/vm/smoketest/payload/main.go
new file mode 100644
index 0000000..2ea6485
--- /dev/null
+++ b/metropolis/vm/smoketest/payload/main.go
@@ -0,0 +1,36 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "os"
+
+ "golang.org/x/sys/unix"
+)
+
+func main() {
+ if err := unix.Mount("devtmpfs", "/dev", "devtmpfs", 0, ""); err != nil {
+ panic(err)
+ }
+ testPort, err := os.OpenFile("/dev/vport1p1", os.O_RDWR, 0)
+ if err != nil {
+ panic(err)
+ }
+ testPort.WriteString("test123")
+ testPort.Close()
+ unix.Reboot(unix.LINUX_REBOOT_CMD_POWER_OFF)
+}