blob: 0b15a58c48e5656142db1a7331f1f87ab6b507dd [file] [log] [blame]
Lorenz Brunfc5dbc62020-05-28 12:18:07 +02001// Copyright 2020 The Monogon Project Authors.
2//
3// SPDX-License-Identifier: Apache-2.0
4//
5// Licensed under the Apache License, Version 2.0 (the "License");
6// you may not use this file except in compliance with the License.
7// You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the License is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the License for the specific language governing permissions and
15// limitations under the License.
16
Serge Bazanski66e58952021-10-05 17:06:56 +020017// launch implements test harnesses for running qemu VMs from tests.
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020018package launch
19
20import (
Lorenz Brun3ff5af32020-06-24 16:34:11 +020021 "bytes"
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020022 "context"
Lorenz Brun3ff5af32020-06-24 16:34:11 +020023 "errors"
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020024 "fmt"
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020025 "net"
26 "os"
27 "os/exec"
Lorenz Brun3ff5af32020-06-24 16:34:11 +020028 "strconv"
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020029 "strings"
Lorenz Brun3ff5af32020-06-24 16:34:11 +020030 "syscall"
Lorenz Bruned0503c2020-07-28 17:21:25 +020031
Lorenz Brun3ff5af32020-06-24 16:34:11 +020032 "golang.org/x/sys/unix"
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020033 "google.golang.org/grpc"
34
Serge Bazanski31370b02021-01-07 16:31:14 +010035 "source.monogon.dev/metropolis/pkg/freeport"
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020036)
37
Serge Bazanski66e58952021-10-05 17:06:56 +020038type QemuValue map[string][]string
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020039
Serge Bazanski66e58952021-10-05 17:06:56 +020040// ToOption encodes structured data into a QEMU option. Example: "test", {"key1":
Serge Bazanski216fe7b2021-05-21 18:36:16 +020041// {"val1"}, "key2": {"val2", "val3"}} returns "test,key1=val1,key2=val2,key2=val3"
Serge Bazanski66e58952021-10-05 17:06:56 +020042func (value QemuValue) ToOption(name string) string {
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020043 var optionValues []string
Lorenz Brun3ff5af32020-06-24 16:34:11 +020044 if name != "" {
45 optionValues = append(optionValues, name)
46 }
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020047 for name, values := range value {
48 if len(values) == 0 {
49 optionValues = append(optionValues, name)
50 }
51 for _, val := range values {
52 optionValues = append(optionValues, fmt.Sprintf("%v=%v", name, val))
53 }
54 }
55 return strings.Join(optionValues, ",")
56}
57
Serge Bazanski216fe7b2021-05-21 18:36:16 +020058// PortMap represents where VM ports are mapped to on the host. It maps from the VM
59// port number to the host port number.
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020060type PortMap map[uint16]uint16
61
Serge Bazanski66e58952021-10-05 17:06:56 +020062// ToQemuForwards generates QEMU hostfwd values (https://qemu.weilnetz.de/doc/qemu-
Serge Bazanski216fe7b2021-05-21 18:36:16 +020063// doc.html#:~:text=hostfwd=) for all mapped ports.
Serge Bazanski66e58952021-10-05 17:06:56 +020064func (p PortMap) ToQemuForwards() []string {
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020065 var hostfwdOptions []string
66 for vmPort, hostPort := range p {
67 hostfwdOptions = append(hostfwdOptions, fmt.Sprintf("tcp::%v-:%v", hostPort, vmPort))
68 }
69 return hostfwdOptions
70}
71
Serge Bazanski216fe7b2021-05-21 18:36:16 +020072// DialGRPC creates a gRPC client for a VM port that's forwarded/mapped to the
73// host. The given port is automatically resolved to the host-mapped port.
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020074func (p PortMap) DialGRPC(port uint16, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
75 mappedPort, ok := p[port]
76 if !ok {
77 return nil, fmt.Errorf("cannot dial port: port %v is not mapped/forwarded", port)
78 }
79 grpcClient, err := grpc.Dial(fmt.Sprintf("localhost:%v", mappedPort), opts...)
80 if err != nil {
81 return nil, fmt.Errorf("failed to dial port %v: %w", port, err)
82 }
83 return grpcClient, nil
84}
85
Serge Bazanski216fe7b2021-05-21 18:36:16 +020086// IdentityPortMap returns a port map where each given port is mapped onto itself
87// on the host. This is mainly useful for development against Metropolis. The dbg
88// command requires this mapping.
Lorenz Bruned0503c2020-07-28 17:21:25 +020089func IdentityPortMap(ports []uint16) PortMap {
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020090 portMap := make(PortMap)
Lorenz Bruned0503c2020-07-28 17:21:25 +020091 for _, port := range ports {
Lorenz Brunfc5dbc62020-05-28 12:18:07 +020092 portMap[port] = port
93 }
94 return portMap
95}
96
Serge Bazanski216fe7b2021-05-21 18:36:16 +020097// ConflictFreePortMap returns a port map where each given port is mapped onto a
98// random free port on the host. This is intended for automated testing where
99// multiple instances of Metropolis nodes might be running. Please call this
100// function for each Launch command separately and as close to it as possible since
101// it cannot guarantee that the ports will remain free.
Lorenz Bruned0503c2020-07-28 17:21:25 +0200102func ConflictFreePortMap(ports []uint16) (PortMap, error) {
Lorenz Brunfc5dbc62020-05-28 12:18:07 +0200103 portMap := make(PortMap)
Lorenz Bruned0503c2020-07-28 17:21:25 +0200104 for _, port := range ports {
Serge Bazanskicb883e22020-07-06 17:47:55 +0200105 mappedPort, listenCloser, err := freeport.AllocateTCPPort()
Lorenz Brunfc5dbc62020-05-28 12:18:07 +0200106 if err != nil {
107 return portMap, fmt.Errorf("failed to get free host port: %w", err)
108 }
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200109 // Defer closing of the listening port until the function is done and all ports are
110 // allocated
Lorenz Brunfc5dbc62020-05-28 12:18:07 +0200111 defer listenCloser.Close()
112 portMap[port] = mappedPort
113 }
114 return portMap, nil
115}
116
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200117// NewSocketPair creates a new socket pair. By connecting both ends to different
118// instances you can connect them with a virtual "network cable". The ends can be
119// passed into the ConnectToSocket option.
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200120func NewSocketPair() (*os.File, *os.File, error) {
121 fds, err := unix.Socketpair(unix.AF_UNIX, syscall.SOCK_STREAM, 0)
122 if err != nil {
123 return nil, nil, fmt.Errorf("failed to call socketpair: %w", err)
124 }
125
126 fd1 := os.NewFile(uintptr(fds[0]), "network0")
127 fd2 := os.NewFile(uintptr(fds[1]), "network1")
128 return fd1, fd2, nil
129}
130
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200131// HostInterfaceMAC is the MAC address the host SLIRP network interface has if it
132// is not disabled (see DisableHostNetworkInterface in MicroVMOptions)
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200133var HostInterfaceMAC = net.HardwareAddr{0x02, 0x72, 0x82, 0xbf, 0xc3, 0x56}
134
135// MicroVMOptions contains all options to start a MicroVM
136type MicroVMOptions struct {
137 // Path to the ELF kernel binary
138 KernelPath string
139
140 // Path to the Initramfs
141 InitramfsPath string
142
143 // Cmdline contains additional kernel commandline options
144 Cmdline string
145
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200146 // SerialPort is a File(descriptor) over which you can communicate with the serial
147 // port of the machine It can be set to an existing file descriptor (like
148 // os.Stdout/os.Stderr) or you can use NewSocketPair() to get one end to talk to
149 // from Go.
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200150 SerialPort *os.File
151
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200152 // ExtraChardevs can be used similar to SerialPort, but can contain an arbitrary
153 // number of additional serial ports
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200154 ExtraChardevs []*os.File
155
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200156 // ExtraNetworkInterfaces can contain an arbitrary number of file descriptors which
157 // are mapped into the VM as virtio network interfaces. The first interface is
158 // always a SLIRP-backed interface for communicating with the host.
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200159 ExtraNetworkInterfaces []*os.File
160
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200161 // PortMap contains ports that are mapped to the host through the built-in SLIRP
162 // network interface.
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200163 PortMap PortMap
164
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200165 // DisableHostNetworkInterface disables the SLIRP-backed host network interface
166 // that is normally the first network interface. If this is set PortMap is ignored.
167 // Mostly useful for speeding up QEMU's startup time for tests.
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200168 DisableHostNetworkInterface bool
169}
170
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200171// RunMicroVM launches a tiny VM mostly intended for testing. Very quick to boot
172// (<40ms).
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200173func RunMicroVM(ctx context.Context, opts *MicroVMOptions) error {
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200174 // Generate options for all the file descriptors we'll be passing as virtio "serial
175 // ports"
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200176 var extraArgs []string
177 for idx, _ := range opts.ExtraChardevs {
178 idxStr := strconv.Itoa(idx)
179 id := "extra" + idxStr
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200180 // That this works is pretty much a hack, but upstream QEMU doesn't have a
181 // bidirectional chardev backend not based around files/sockets on the disk which
182 // are a giant pain to work with. We're using QEMU's fdset functionality to make
183 // FDs available as pseudo-files and then "ab"using the pipe backend's fallback
184 // functionality to get a single bidirectional chardev backend backed by a passed-
185 // down RDWR fd. Ref https://lists.gnu.org/archive/html/qemu-devel/2015-
186 // 12/msg01256.html
Serge Bazanski66e58952021-10-05 17:06:56 +0200187 addFdConf := QemuValue{
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200188 "set": {idxStr},
189 "fd": {strconv.Itoa(idx + 3)},
190 }
Serge Bazanski66e58952021-10-05 17:06:56 +0200191 chardevConf := QemuValue{
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200192 "id": {id},
193 "path": {"/dev/fdset/" + idxStr},
194 }
Serge Bazanski66e58952021-10-05 17:06:56 +0200195 deviceConf := QemuValue{
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200196 "chardev": {id},
197 }
Serge Bazanski66e58952021-10-05 17:06:56 +0200198 extraArgs = append(extraArgs, "-add-fd", addFdConf.ToOption(""),
199 "-chardev", chardevConf.ToOption("pipe"), "-device", deviceConf.ToOption("virtserialport"))
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200200 }
201
202 for idx, _ := range opts.ExtraNetworkInterfaces {
203 id := fmt.Sprintf("net%v", idx)
Serge Bazanski66e58952021-10-05 17:06:56 +0200204 netdevConf := QemuValue{
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200205 "id": {id},
206 "fd": {strconv.Itoa(idx + 3 + len(opts.ExtraChardevs))},
207 }
Serge Bazanski66e58952021-10-05 17:06:56 +0200208 extraArgs = append(extraArgs, "-netdev", netdevConf.ToOption("socket"), "-device", "virtio-net-device,netdev="+id)
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200209 }
210
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200211 // This sets up a minimum viable environment for our Linux kernel. It clears all
212 // standard QEMU configuration and sets up a MicroVM machine
213 // (https://github.com/qemu/qemu/blob/master/docs/microvm.rst) with all legacy
214 // emulation turned off. This means the only "hardware" the Linux kernel inside can
215 // communicate with is a single virtio-mmio region. Over that MMIO interface we run
216 // a paravirtualized RNG (since the kernel in there has nothing to gather that from
217 // and it delays booting), a single paravirtualized console and an arbitrary number
218 // of extra serial ports for talking to various things that might run inside. The
219 // kernel, initramfs and command line are mapped into VM memory at boot time and
220 // not loaded from any sort of disk. Booting and shutting off one of these VMs
221 // takes <100ms.
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200222 baseArgs := []string{"-nodefaults", "-no-user-config", "-nographic", "-no-reboot",
223 "-accel", "kvm", "-cpu", "host",
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200224 // Needed until QEMU updates their bundled qboot version (needs
225 // https://github.com/bonzini/qboot/pull/28)
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200226 "-bios", "external/com_github_bonzini_qboot/bios.bin",
227 "-M", "microvm,x-option-roms=off,pic=off,pit=off,rtc=off,isa-serial=off",
228 "-kernel", opts.KernelPath,
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200229 // We force using a triple-fault reboot strategy since otherwise the kernel first
230 // tries others (like ACPI) which are not available in this very restricted
231 // environment. Similarly we need to override the boot console since there's
232 // nothing on the ISA bus that the kernel could talk to. We also force quiet for
233 // performance reasons.
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200234 "-append", "reboot=t console=hvc0 quiet " + opts.Cmdline,
235 "-initrd", opts.InitramfsPath,
236 "-device", "virtio-rng-device,max-bytes=1024,period=1000",
237 "-device", "virtio-serial-device,max_ports=16",
238 "-chardev", "stdio,id=con0", "-device", "virtconsole,chardev=con0",
239 }
240
241 if !opts.DisableHostNetworkInterface {
242 qemuNetType := "user"
Serge Bazanski66e58952021-10-05 17:06:56 +0200243 qemuNetConfig := QemuValue{
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200244 "id": {"usernet0"},
245 "net": {"10.42.0.0/24"},
246 "dhcpstart": {"10.42.0.10"},
247 }
248 if opts.PortMap != nil {
Serge Bazanski66e58952021-10-05 17:06:56 +0200249 qemuNetConfig["hostfwd"] = opts.PortMap.ToQemuForwards()
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200250 }
251
Serge Bazanski66e58952021-10-05 17:06:56 +0200252 baseArgs = append(baseArgs, "-netdev", qemuNetConfig.ToOption(qemuNetType),
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200253 "-device", "virtio-net-device,netdev=usernet0,mac="+HostInterfaceMAC.String())
254 }
255
256 var stdErrBuf bytes.Buffer
257 cmd := exec.CommandContext(ctx, "qemu-system-x86_64", append(baseArgs, extraArgs...)...)
258 cmd.Stdout = opts.SerialPort
259 cmd.Stderr = &stdErrBuf
260
261 cmd.ExtraFiles = append(cmd.ExtraFiles, opts.ExtraChardevs...)
262 cmd.ExtraFiles = append(cmd.ExtraFiles, opts.ExtraNetworkInterfaces...)
263
264 err := cmd.Run()
Serge Bazanski66e58952021-10-05 17:06:56 +0200265 // If it's a context error, just quit. There's no way to tell a
266 // killed-due-to-context vs killed-due-to-external-reason error returned by Run,
267 // so we approximate by looking at the context's status.
268 if err != nil && ctx.Err() != nil {
269 return ctx.Err()
270 }
271
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200272 var exerr *exec.ExitError
273 if err != nil && errors.As(err, &exerr) {
274 exerr.Stderr = stdErrBuf.Bytes()
275 newErr := QEMUError(*exerr)
276 return &newErr
277 }
278 return err
279}
280
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200281// QEMUError is a special type of ExitError used when QEMU fails. In addition to
282// normal ExitError features it prints stderr for debugging.
Lorenz Brun3ff5af32020-06-24 16:34:11 +0200283type QEMUError exec.ExitError
284
285func (e *QEMUError) Error() string {
286 return fmt.Sprintf("%v: %v", e.String(), string(e.Stderr))
Lorenz Brunfc5dbc62020-05-28 12:18:07 +0200287}