Add minimal functionality test for k8s control plane
Basic functionality test that sends the bootstrap RPC call,
waits for the k8s control plane to come up and runs a simple
kubectl command (that is expected to fail).
Adds reflection to the server to make grpc_cli easier to use.
Test Plan:
Ran `:launch` (because we modified its config) and `:test_boot`,
saw a nicely booted k8s cluster:
{P90}
X-Origin-Diff: phab/D275
GitOrigin-RevId: fe01e3f3ed09877aa76c15946664c9d9bdc4751b
diff --git a/core/cmd/init/main.go b/core/cmd/init/main.go
index 1d068d4..902008d 100644
--- a/core/cmd/init/main.go
+++ b/core/cmd/init/main.go
@@ -29,6 +29,11 @@
"golang.org/x/sys/unix"
)
+const (
+ apiPort = 7833
+ consensusPort = 7834
+)
+
func main() {
defer func() {
if r := recover(); r != nil {
@@ -79,7 +84,7 @@
logger.Panic("Failed to start network service", zap.Error(err))
}
- nodeInstance, err := node.NewSmalltownNode(logger, 7833, 7834)
+ nodeInstance, err := node.NewSmalltownNode(logger, apiPort, consensusPort)
if err != nil {
panic(err)
}
diff --git a/core/internal/api/BUILD.bazel b/core/internal/api/BUILD.bazel
index b7aa48d..3daa397 100644
--- a/core/internal/api/BUILD.bazel
+++ b/core/internal/api/BUILD.bazel
@@ -16,6 +16,7 @@
"//core/internal/common/service:go_default_library",
"//core/internal/consensus:go_default_library",
"@org_golang_google_grpc//:go_default_library",
+ "@org_golang_google_grpc//reflection:go_default_library",
"@org_uber_go_zap//:go_default_library",
],
)
diff --git a/core/internal/api/server.go b/core/internal/api/server.go
index 715e99e..efd0be5 100644
--- a/core/internal/api/server.go
+++ b/core/internal/api/server.go
@@ -24,6 +24,7 @@
"git.monogon.dev/source/nexantic.git/core/internal/consensus"
"go.uber.org/zap"
"google.golang.org/grpc"
+ "google.golang.org/grpc/reflection"
"net"
)
@@ -57,6 +58,8 @@
schema.RegisterClusterManagementServer(grpcServer, s)
schema.RegisterSetupServiceServer(grpcServer, s)
+ reflection.Register(grpcServer)
+
s.grpcServer = grpcServer
return s, nil
diff --git a/core/internal/kubernetes/auth.go b/core/internal/kubernetes/auth.go
index afc51c1..89ae6dc 100644
--- a/core/internal/kubernetes/auth.go
+++ b/core/internal/kubernetes/auth.go
@@ -351,7 +351,7 @@
func makeLocalKubeconfig(ca, cert, key []byte) ([]byte, error) {
kubeconfig := configapi.NewConfig()
cluster := configapi.NewCluster()
- cluster.Server = "https://localhost:6443"
+ cluster.Server = "https://127.0.0.1:6443"
cluster.CertificateAuthorityData = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca})
kubeconfig.Clusters["default"] = cluster
authInfo := configapi.NewAuthInfo()
diff --git a/core/scripts/BUILD b/core/scripts/BUILD
index b03bc49..c023ed4 100644
--- a/core/scripts/BUILD
+++ b/core/scripts/BUILD
@@ -1,10 +1,24 @@
-sh_binary(
- name = "launch",
- srcs = ["launch.sh"],
+
+sh_library(
+ name = "vm_deps",
data = [
"@//core:image",
"@//core:swtpm_data",
"@edk2//:firmware",
+ ]
+)
+
+sh_binary(
+ name = "launch",
+ srcs = ["launch.sh"],
+ deps = [":vm_deps"],
+)
+
+sh_library(
+ name = "test_deps",
+ data = [
+ ":launch",
+ "//:kubectl",
],
)
@@ -14,5 +28,5 @@
srcs = ["test_boot.sh"],
# expects wants a pty, which do not exist in the sandbox
tags = ["local"],
- deps = [":launch"],
+ deps = [":test_deps", ":vm_deps"],
)
diff --git a/core/scripts/test_boot.sh b/core/scripts/test_boot.sh
index d380ad8..3b6674e 100755
--- a/core/scripts/test_boot.sh
+++ b/core/scripts/test_boot.sh
@@ -1,17 +1,52 @@
#!/usr/bin/expect -f
+# Getting the actual path from a sh_test rule is not straight-forward and would involve
+# parsing the runfile at $RUNFILES_DIR, so just hardcode it.
+#
+# We'll want to replace this thing by a proper e2e testing suite sooner than we'll
+# have to worry about cross-compilation or varying build environments.
+#
+# (see https://github.com/bazelbuild/bazel/blob/master/tools/bash/runfiles/runfiles.bash)
+set kubectl_path "external/kubernetes/cmd/kubectl/linux_amd64_pure_stripped/kubectl"
+
set timeout 60
+proc print_stderr {msg} {
+ send_error "\[TEST\] $msg\n"
+}
+
spawn core/scripts/launch.sh
expect "Network service got IP" {} default {
- send_error "Failed while waiting for IP address"
+ print_stderr "Failed while waiting for IP address\n"
exit 1
}
-expect "Initialized encrypted storage" {
- exit 0
-} default {
- send_error "Failed while waiting for encrypted storage"
+expect "Initialized encrypted storage" {} default {
+ print_stderr "Failed while waiting for encrypted storage\n"
exit 1
}
+
+print_stderr "Calling api.SetupService.Setup\n"
+system "grpc_cli --channel_creds_type=insecure call localhost:7833 api.SetupService.Setup -json_input '{\"nodeName\": \"node-1\"}'"
+
+print_stderr "Calling api.SetupService.BootstrapNewCluster\n"
+system "grpc_cli --channel_creds_type=insecure call localhost:7833 api.SetupService.BootstrapNewCluster ''"
+
+# Make an educated guess if the control plane came up
+expect -timeout 3 "\n" {
+ exp_continue
+} timeout {} default {
+ print_stderr "Failed while waiting for k8s control plane\n"
+ exit 1
+}
+
+spawn $kubectl_path cluster-info dump -s https://localhost:6443 --username none --password none --insecure-skip-tls-verify=true
+
+expect "User \"system:anonymous\" cannot list resource \"nodes\" in API group \"\" at the cluster scope" {} default {
+ print_stderr "Failed while waiting for encrypted storage\n"
+ exit 1
+}
+
+print_stderr "Completed successfully"
+exit 0