// Copyright 2019 Red Hat // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // qemu.go is a Go interface to running `qemu` as a subprocess. // // Why not libvirt? // Two main reasons. First, we really do want to use qemu, and not // something else. We rely on qemu features/APIs and there's a general // assumption that the qemu process is local (e.g. we expose 9p/virtiofs filesystem // sharing). Second, libvirt runs as a daemon, but we want the // VMs "lifecycle bound" to their creating process (e.g. kola), // so that e.g. Ctrl-C (SIGINT) kills both reliably. // // Other related projects (as a reference to share ideas if not code) // https://github.com/google/syzkaller/blob/3e84253bf41d63c55f92679b1aab9102f2f4949a/vm/qemu/qemu.go // https://github.com/intel/govmm package platform import ( "bufio" "bytes" "context" "fmt" "io" "math/rand" "net" "os" "path/filepath" "strconv" "strings" "sync" "syscall" "time" "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/util" coreosarch "github.com/coreos/stream-metadata-go/arch" "github.com/digitalocean/go-qemu/qmp" "github.com/coreos/coreos-assembler/mantle/system" "github.com/coreos/coreos-assembler/mantle/system/exec" "github.com/pkg/errors" ) var ( // ErrInitramfsEmergency is the marker error returned upon node blocking in emergency mode in initramfs. ErrInitramfsEmergency = errors.New("entered emergency.target in initramfs") ConsoleKernelArgument = map[string]string{ "x86_64": "ttyS0,115200n8", "ppc64le": "hvc0", "aarch64": "ttyAMA0", "s390x": "ttysclp0", } ) // HostForwardPort contains details about port-forwarding for the VM. type HostForwardPort struct { Service string HostPort int GuestPort int } // QEMUMachine represents a qemu instance. type QEMUMachine interface { // Embedding the Machine interface Machine // RemovePrimaryBlockDevice removes the primary device from a given qemu // instance and sets the secondary device as primary. RemovePrimaryBlockDevice() error // RemoveBlockDeviceForMultipath removes the specified device on multipath. RemoveBlockDeviceForMultipath(device string) error } // Disk holds the details of a virtual disk. type Disk struct { Size string // disk image size in bytes, optional suffixes "K", "M", "G", "T" allowed. BackingFile string // raw disk image to use. BackingFormat string // qcow2, raw, etc. If unspecified will be autodetected. Channel string // virtio (default), nvme, scsi DeviceOpts []string // extra options to pass to qemu -device. "serial=XXXX" makes disks show up as /dev/disk/by-id/virtio- DriveOpts []string // extra options to pass to -drive SectorSize int // if not 0, override disk sector size LogicalSectorSize int // if not 0, override disk sector size NbdDisk bool // if true, the disks should be presented over nbd:unix socket MultiPathDisk bool // if true, present multiple paths Wwn uint64 // Optional World wide name for the SCSI disk. If not set or set to 0, a random one will be generated. Used only with "channel=scsi". Must be an integer attachEndPoint string // qemuPath to attach to dstFileName string // the prepared file nbdServCmd exec.Cmd // command to serve the disk } func ParseDisk(spec string, allowNoSize bool) (*Disk, error) { var channel string sectorSize := 0 logicalSectorSize := 0 serialOpt := []string{} multipathed := false var wwn uint64 size, diskmap, err := util.ParseDiskSpec(spec, allowNoSize) if err != nil { return nil, fmt.Errorf("failed to parse disk spec %q: %w", spec, err) } for key, value := range diskmap { switch key { case "channel": channel = value case "4k": sectorSize = 4096 case "512e": sectorSize = 4096 logicalSectorSize = 512 case "mpath": multipathed = true case "serial": value = "serial=" + value serialOpt = append(serialOpt, value) case "wwn": wwn, err = strconv.ParseUint(value, 10, 64) if err != nil { return nil, fmt.Errorf("invalid value %s for wwn. Must be an integer", value) } default: return nil, fmt.Errorf("invalid key %q", key) } } sizeStr := "" if size > 0 { sizeStr = fmt.Sprintf("%dG", size) } return &Disk{ Size: sizeStr, Channel: channel, DeviceOpts: serialOpt, SectorSize: sectorSize, LogicalSectorSize: logicalSectorSize, MultiPathDisk: multipathed, Wwn: wwn, }, nil } // Parse the src/dest for BindRO and BindRW options func ParseBindOpt(spec string) (string, string, error) { parts := strings.SplitN(spec, ",", 2) if len(parts) == 1 { return "", "", fmt.Errorf("malformed bind option, required: SRC,DEST") } return parts[0], parts[1], nil } // bootIso is an internal struct used by AddIso() and setupIso() type bootIso struct { path string bootindex string } // QemuInstance holds an instantiated VM through its lifecycle. type QemuInstance struct { qemu exec.Cmd architecture string tempdir string swtpm exec.Cmd // Helpers are child processes such as nbd or virtiofsd that should be lifecycle bound to qemu helpers []exec.Cmd hostForwardedPorts []HostForwardPort journalPipe *os.File qmpSocket *qmp.SocketMonitor qmpSocketPath string } // Signaled returns whether QEMU process was signaled. func (inst *QemuInstance) Signaled() bool { return inst.qemu.Signaled() } // Pid returns the PID of QEMU process. func (inst *QemuInstance) Pid() int { return inst.qemu.Pid() } // Kill kills the VM instance. func (inst *QemuInstance) Kill() error { plog.Debugf("Killing qemu (%v)", inst.qemu.Pid()) return inst.qemu.Kill() } // SSHAddress returns the IP address with the forwarded port (host-side). func (inst *QemuInstance) SSHAddress() (string, error) { for _, fwdPorts := range inst.hostForwardedPorts { if fwdPorts.Service == "ssh" { return fmt.Sprintf("127.0.0.1:%d", fwdPorts.HostPort), nil } } return "", fmt.Errorf("didn't find an address") } // Wait for the qemu process to exit func (inst *QemuInstance) Wait() error { return inst.qemu.Wait() } // WaitIgnitionError will only return if the instance // failed inside the initramfs. The resulting string will // be a newline-delimited stream of JSON strings, as returned // by `journalctl -o json`. func (inst *QemuInstance) WaitIgnitionError(ctx context.Context) (string, error) { b := bufio.NewReaderSize(inst.journalPipe, 64768) var r strings.Builder iscorrupted := false _, err := b.Peek(1) if err != nil { // It's normal to get EOF if we didn't catch an error and qemu // is shutting down. We also need to handle when the Destroy() // function closes the journal FD on us. if e, ok := err.(*os.PathError); ok && e.Err == os.ErrClosed { return "", nil } else if err == io.EOF { return "", nil } return "", errors.Wrapf(err, "Reading from journal") } for { if err := ctx.Err(); err != nil { return "", err } line, prefix, err := b.ReadLine() if err != nil { return r.String(), errors.Wrapf(err, "Reading from journal channel") } if prefix { iscorrupted = true } if len(line) == 0 || string(line) == "{}" { break } r.Write(line) r.Write([]byte("\n")) } if iscorrupted { return r.String(), fmt.Errorf("journal was truncated due to overly long line") } return r.String(), nil } // WaitAll wraps the process exit as well as WaitIgnitionError, // returning an error if either fail. func (inst *QemuInstance) WaitAll(ctx context.Context) error { c := make(chan error) waitCtx, cancel := context.WithCancel(ctx) defer cancel() // Early stop due to failure in initramfs. go func() { buf, err := inst.WaitIgnitionError(waitCtx) if err != nil { c <- err return } // TODO: parse buf and try to nicely render something. if buf != "" { c <- ErrInitramfsEmergency return } }() // Machine terminated. go func() { select { case <-waitCtx.Done(): c <- waitCtx.Err() case c <- inst.Wait(): } }() return <-c } // Destroy kills the instance and associated sidecar processes. func (inst *QemuInstance) Destroy() { if inst.qmpSocket != nil { inst.qmpSocket.Disconnect() //nolint // Ignore Errors inst.qmpSocket = nil os.Remove(inst.qmpSocketPath) //nolint // Ignore Errors } if inst.journalPipe != nil { plog.Debugf("Sleep 1 to allow for more journal messages to get flushed") time.Sleep(1 * time.Second) inst.journalPipe.Close() inst.journalPipe = nil } // kill is safe if already dead if err := inst.Kill(); err != nil { plog.Errorf("Error killing qemu instance %v: %v", inst.Pid(), err) } if inst.swtpm != nil { inst.swtpm.Kill() //nolint // Ignore errors inst.swtpm = nil } for _, p := range inst.helpers { if p != nil { p.Kill() //nolint // Ignore errors } } inst.helpers = nil if inst.tempdir != "" { if err := os.RemoveAll(inst.tempdir); err != nil { plog.Errorf("Error removing tempdir: %v", err) } } } // SwitchBootOrder tweaks the boot order for the instance. // Currently effective on aarch64: switches the boot order to boot from disk on reboot. For s390x and aarch64, bootindex // is used to boot from the network device (boot once is not supported). For s390x, the boot ordering was not a problem as it // would always read from disk first. For aarch64, the bootindex needs to be switched to boot from disk before a reboot func (inst *QemuInstance) SwitchBootOrder() (err2 error) { switch inst.architecture { case "s390x", "aarch64": break default: //Not applicable for other arches return nil } devs, err := inst.listDevices() if err != nil { return errors.Wrapf(err, "Could not list devices through qmp") } blkdevs, err := inst.listBlkDevices() if err != nil { return errors.Wrapf(err, "Could not list blk devices through qmp") } var bootdev, primarydev, secondarydev string // Get boot device for PXE boots for _, dev := range devs.Return { switch dev.Type { case "child", "child": bootdev = filepath.Join("/machine/peripheral-anon", dev.Name) default: break } } // Get boot device for ISO boots and target block device for _, dev := range blkdevs.Return { devpath := filepath.Clean(strings.TrimSuffix(dev.DevicePath, "virtio-backend")) switch dev.Device { case "installiso": bootdev = devpath case "disk-1", "mpath10": primarydev = devpath case "mpath11": secondarydev = devpath case "": if dev.Inserted.NodeName == "installiso" { bootdev = devpath } default: break } } if bootdev == "" { return fmt.Errorf("Could not find boot device using QMP.\n"+ "Full list of peripherals: %v.\n"+ "Full list of block devices: %v.\n", devs.Return, blkdevs.Return) } if primarydev == "" { return fmt.Errorf("Could not find target disk using QMP.\n"+ "Full list of block devices: %v.\n", blkdevs.Return) } // unset bootindex for the boot device if err := inst.setBootIndexForDevice(bootdev, -1); err != nil { return errors.Wrapf(err, "Could not set bootindex for bootdev") } // set bootindex to 1 to boot from disk if err := inst.setBootIndexForDevice(primarydev, 1); err != nil { return errors.Wrapf(err, "Could not set bootindex for primarydev") } // set bootindex to 2 for secondary multipath disk if secondarydev != "" { if err := inst.setBootIndexForDevice(secondarydev, 2); err != nil { return errors.Wrapf(err, "Could not set bootindex for secondarydev") } } return nil } // RemovePrimaryBlockDevice deletes the primary device from a qemu instance // and sets the secondary device as primary. It expects that all block devices // with device name disk- are mirrors. func (inst *QemuInstance) RemovePrimaryBlockDevice() (err2 error) { var primaryDevice string var secondaryDevicePath string blkdevs, err := inst.listBlkDevices() if err != nil { return errors.Wrapf(err, "Could not list block devices through qmp") } // This tries to identify the primary device by looking into // a `BackingFileDepth` parameter of a device and check if // it is a removable and part of `virtio-blk-pci` devices. for _, dev := range blkdevs.Return { if !dev.Removable && strings.HasPrefix(dev.Device, "disk-") { if dev.Inserted.BackingFileDepth == 1 { primaryDevice = dev.DevicePath } else { secondaryDevicePath = dev.DevicePath } } } if err := inst.setBootIndexForDevice(primaryDevice, -1); err != nil { return errors.Wrapf(err, "Could not set bootindex for %v", primaryDevice) } primaryDevice = primaryDevice[:strings.LastIndex(primaryDevice, "/")] if err := inst.deleteBlockDevice(primaryDevice); err != nil { return errors.Wrapf(err, "Could not delete primary device %v", primaryDevice) } if len(secondaryDevicePath) == 0 { return errors.Wrapf(err, "Could not find secondary device") } if err := inst.setBootIndexForDevice(secondaryDevicePath, 1); err != nil { return errors.Wrapf(err, "Could not set bootindex for %v", secondaryDevicePath) } return nil } // RemoveBlockDeviceForMultipath remove the specified device on multipath. func (inst *QemuInstance) RemoveBlockDeviceForMultipath(device string) error { blkdevs, err := inst.listBlkDevices() if err != nil { return errors.Wrapf(err, "Could not list block devices through qmp") } var devicePath string for _, dev := range blkdevs.Return { if dev.Device == device { devicePath = dev.DevicePath break } } if devicePath == "" { return fmt.Errorf("Target device %q not found in block device list", device) } if err = inst.deleteBlockDevice(devicePath); err != nil { return errors.Wrapf(err, "Could not delete device %v", devicePath) } return nil } // A directory mounted from the host into the guest, via 9p or virtiofs type HostMount struct { src string dest string readonly bool } // QemuBuilder is a configurator that can then create a qemu instance type QemuBuilder struct { // ConfigFile is a path to Ignition configuration ConfigFile string // ForceConfigInjection is useful for booting `metal` images directly ForceConfigInjection bool configInjected bool // File to which to redirect the serial console ConsoleFile string // If set, use QEMU full emulation for the target architecture architecture string // MemoryMiB default determined using DefaultMemoryMiB() MemoryMiB int // If true, two NUMA nodes will be simulated, splitting memory and CPUs between them. // Each node gets half the resources, if there is an odd amount of some resource, node1 // will receive the remainder. NumaNodes bool // Processors < 0 means to use host count, unset means 1, values > 1 are directly used Processors int UUID string Firmware string Swtpm bool Pdeathsig bool Argv []string // AppendKernelArgs are appended to the bootloader config AppendKernelArgs string // AppendFirstbootKernelArgs are written to /boot/ignition AppendFirstbootKernelArgs string Hostname string InheritConsole bool iso *bootIso isoAsDisk bool primaryDisk *Disk // primaryIsBoot is true if the only boot media should be the primary disk primaryIsBoot bool // tempdir holds our temporary files tempdir string // ignition is a config object that can be used instead of // ConfigFile. ignition *conf.Conf ignitionSet bool ignitionRendered bool UsermodeNetworking bool usermodeNetworkingAddr string RestrictNetworking bool requestedHostForwardPorts []HostForwardPort additionalNics int netbootP string netbootDir string netbootIndex string finalized bool diskID uint disks []*Disk // virtioSerialID is incremented for each device virtioSerialID uint // hostMounts is an array of directories mounted (via 9p or virtiofs) from the host hostMounts []HostMount // fds is file descriptors we own to pass to qemu fds []*os.File // IBM Secure Execution secureExecution bool ignitionPubKey string } // NewQemuBuilder creates a new build for QEMU with default settings. func NewQemuBuilder() *QemuBuilder { var defaultFirmware string switch coreosarch.CurrentRpmArch() { case "x86_64": defaultFirmware = "bios" case "aarch64": defaultFirmware = "uefi" default: defaultFirmware = "" } ret := QemuBuilder{ Firmware: defaultFirmware, Swtpm: true, Pdeathsig: true, Argv: []string{}, architecture: coreosarch.CurrentRpmArch(), } return &ret } func (builder *QemuBuilder) ensureTempdir() error { if builder.tempdir != "" { return nil } tempdir, err := os.MkdirTemp("/var/tmp", "mantle-qemu") if err != nil { return err } builder.tempdir = tempdir return nil } // SetConfig injects Ignition; this can be used in place of ConfigFile. func (builder *QemuBuilder) SetConfig(config *conf.Conf) { if config == nil { return } if builder.ignitionRendered { panic("SetConfig called after config rendered") } if builder.ignitionSet { panic("SetConfig called multiple times") } builder.ignition = config builder.ignitionSet = true } // Small wrapper around os.CreateTemp() to avoid leaking our tempdir to // others. func (builder *QemuBuilder) TempFile(pattern string) (*os.File, error) { if err := builder.ensureTempdir(); err != nil { return nil, err } return os.CreateTemp(builder.tempdir, pattern) } // Returns the path where the Ignition config will be written. func (builder *QemuBuilder) IgnitionPath() (string, error) { if err := builder.ensureTempdir(); err != nil { return "", err } return filepath.Join(builder.tempdir, "config.ign"), nil } // renderIgnition lazily renders a parsed config if one is set func (builder *QemuBuilder) renderIgnition() error { if !builder.ignitionSet || builder.ignitionRendered { return nil } if builder.ConfigFile != "" { panic("Both ConfigFile and ignition set") } var err error builder.ConfigFile, err = builder.IgnitionPath() if err != nil { return err } if err := builder.ignition.WriteFile(builder.ConfigFile); err != nil { return err } builder.ignition = nil builder.ignitionRendered = true return nil } // AddFd appends a file descriptor that will be passed to qemu, // returning a "/dev/fdset/" argument that one can use with e.g. // -drive file=/dev/fdset/. func (builder *QemuBuilder) AddFd(fd *os.File) string { set := len(builder.fds) + 1 builder.fds = append(builder.fds, fd) return fmt.Sprintf("/dev/fdset/%d", set) } // virtio returns a virtio device argument for qemu, which is architecture dependent func virtio(arch, device, args string) string { var suffix string switch arch { case "x86_64", "ppc64le", "aarch64": suffix = "pci" case "s390x": suffix = "ccw" default: panic(fmt.Sprintf("RpmArch %s unhandled in virtio()", arch)) } return fmt.Sprintf("virtio-%s-%s,%s", device, suffix, args) } // EnableUsermodeNetworking configure forwarding for all requested ports, // via usermode network helpers. func (builder *QemuBuilder) EnableUsermodeNetworking(h []HostForwardPort, usernetAddr string) { builder.UsermodeNetworking = true builder.requestedHostForwardPorts = h builder.usermodeNetworkingAddr = usernetAddr } func (builder *QemuBuilder) SetNetbootP(filename, dir string) { builder.UsermodeNetworking = true builder.netbootP = filename builder.netbootDir = dir } func (builder *QemuBuilder) SetNetbootIndex(index string) { builder.netbootIndex = index } func (builder *QemuBuilder) AddAdditionalNics(additionalNics int) { builder.additionalNics = additionalNics } func (builder *QemuBuilder) setupNetworking() error { netdev := "user,id=eth0" for i := range builder.requestedHostForwardPorts { address := fmt.Sprintf(":%d", builder.requestedHostForwardPorts[i].HostPort) // Possible race condition between getting the port here and using it // with qemu -- trade off for simpler port management l, err := net.Listen("tcp", address) if err != nil { return err } l.Close() builder.requestedHostForwardPorts[i].HostPort = l.Addr().(*net.TCPAddr).Port netdev += fmt.Sprintf(",hostfwd=tcp:127.0.0.1:%d-:%d", builder.requestedHostForwardPorts[i].HostPort, builder.requestedHostForwardPorts[i].GuestPort) } if builder.Hostname != "" { netdev += fmt.Sprintf(",hostname=%s", builder.Hostname) } if builder.RestrictNetworking { netdev += ",restrict=on" } if builder.usermodeNetworkingAddr != "" { netdev += ",net=" + builder.usermodeNetworkingAddr } if builder.netbootP != "" { // do an early stat so we fail with a nicer error now instead of in the VM if _, err := os.Stat(filepath.Join(builder.netbootDir, builder.netbootP)); err != nil { return err } tftpDir := "" relpath := "" if builder.netbootDir == "" { absPath, err := filepath.Abs(builder.netbootP) if err != nil { return err } tftpDir = filepath.Dir(absPath) relpath = filepath.Base(absPath) } else { absPath, err := filepath.Abs(builder.netbootDir) if err != nil { return err } tftpDir = absPath relpath = builder.netbootP } netdev += fmt.Sprintf(",tftp=%s,bootfile=/%s", tftpDir, relpath) // It does not make sense to use the bootindex together with the -boot order=... (or -boot once=...). // The guest firmware implementations normally either support the one or the other. if builder.netbootIndex == "" { builder.Append("-boot", "order=n") } } args := "netdev=eth0" if builder.netbootIndex != "" { args += fmt.Sprintf(",bootindex=%s", builder.netbootIndex) } builder.Append("-netdev", netdev, "-device", virtio(builder.architecture, "net", args)) return nil } func (builder *QemuBuilder) setupAdditionalNetworking() error { macCounter := 0 netOffset := 30 for i := 1; i <= builder.additionalNics; i++ { idSuffix := fmt.Sprintf("%d", i) netSuffix := fmt.Sprintf("%d", netOffset+i) macSuffix := fmt.Sprintf("%02x", macCounter) netdev := fmt.Sprintf("user,id=eth%s,dhcpstart=10.0.2.%s", idSuffix, netSuffix) device := virtio(builder.architecture, "net", fmt.Sprintf("netdev=eth%s,mac=52:55:00:d1:56:%s", idSuffix, macSuffix)) // On s390x, devices use the CCW bus instead of PCI, which may cause them to appear in a different order. // By default, the CSSID is 0xFE and the SSID is 0x0. For additional NICs, set the SSID to 0x1 // to avoid interference with other CCW devices if builder.architecture == "s390x" { device += fmt.Sprintf(",devno=fe.1.%04x", i) } builder.Append("-netdev", netdev, "-device", device) macCounter++ } return nil } // SetArchitecture enables qemu full emulation for the target architecture. func (builder *QemuBuilder) SetArchitecture(arch string) error { switch arch { case "x86_64", "aarch64", "s390x", "ppc64le": builder.architecture = arch return nil } return fmt.Errorf("architecture %s not supported by coreos-assembler qemu", arch) } // SetSecureExecution enables qemu confidential guest support and adds hostkey to ignition config. func (builder *QemuBuilder) SetSecureExecution(gpgkey string, hostkey string, config *conf.Conf) error { if supports, err := builder.supportsSecureExecution(); err != nil { return err } else if !supports { return fmt.Errorf("Secure Execution was requested but isn't supported/enabled") } if gpgkey == "" { return fmt.Errorf("Secure Execution was requested, but we don't have a GPG Public Key to encrypt the config") } if config != nil { if hostkey == "" { // dummy hostkey; this is good enough at least for the first boot (to prevent genprotimg from failing) dummy, err := builder.TempFile("hostkey.*") if err != nil { return fmt.Errorf("creating hostkey: %v", err) } c := exec.Command("openssl", "req", "-x509", "-sha512", "-nodes", "-days", "1", "-subj", "/C=US/O=IBM/CN=secex", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:secp521r1", "-out", dummy.Name()) if err := c.Run(); err != nil { return fmt.Errorf("generating hostkey: %v", err) } hostkey = dummy.Name() } if contents, err := os.ReadFile(hostkey); err != nil { return fmt.Errorf("reading hostkey: %v", err) } else { config.AddFile("/etc/se-hostkeys/ibm-z-hostkey-1", string(contents), 0644) } } builder.secureExecution = true builder.ignitionPubKey = gpgkey builder.Append("-object", "s390-pv-guest,id=pv0", "-machine", "confidential-guest-support=pv0") return nil } // When running kola secex tests with '--parallel=auto', this function fails with: // // kola: retryloop: failed to bring up machines: encrypting ignition_crypted.1234: exit status 2 // // Use mutex to protect `gpg --encrypt` var gpgMutex sync.Mutex func (builder *QemuBuilder) encryptIgnitionConfig() error { crypted, err := builder.TempFile("ignition_crypted.*") if err != nil { return fmt.Errorf("creating crypted config: %v", err) } gpgMutex.Lock() defer gpgMutex.Unlock() c := exec.Command("gpg", "--recipient-file", builder.ignitionPubKey, "--yes", "--output", crypted.Name(), "--armor", "--encrypt", builder.ConfigFile) if err := c.Run(); err != nil { return fmt.Errorf("encrypting %s: %v", crypted.Name(), err) } builder.ConfigFile = crypted.Name() return nil } // MountHost sets up a mount point from the host to guest. // Note that virtiofs does not currently support read-only mounts (which is really surprising!). // We do mount it read-only by default in the guest, however. func (builder *QemuBuilder) MountHost(source, dest string, readonly bool) { builder.hostMounts = append(builder.hostMounts, HostMount{src: source, dest: dest, readonly: readonly}) } // supportsFwCfg if the target system supports injecting // Ignition via the qemu -fw_cfg option. func (builder *QemuBuilder) supportsFwCfg() bool { switch builder.architecture { case "s390x", "ppc64le": return false } return true } // supportsSecureExecution if s390x host (zKVM/LPAR) has "Secure Execution for Linux" feature enabled func (builder *QemuBuilder) supportsSecureExecution() (bool, error) { if builder.architecture != "s390x" { return false, nil } content, err := os.ReadFile("/sys/firmware/uv/prot_virt_host") if err != nil { if os.IsNotExist(err) { return false, nil } return false, fmt.Errorf("reading protvirt flag: %v", err) } if len(content) < 1 { return false, nil } enabled := content[0] == '1' return enabled, nil } // supportsSwtpm if the target system supports a virtual TPM device func (builder *QemuBuilder) supportsSwtpm() bool { switch builder.architecture { case "s390x": // s390x does not support a backend for TPM return false } return true } // fileRemoteLocation is a bit misleading. We are NOT putting the ignition config in the root parition. We mount the boot partition on / just to get around the fact that // the root partition does not need to be mounted to inject ignition config. Now that we have LUKS , we have to do more work to detect a LUKS root partition // and it is not needed here. const fileRemoteLocation = "/ignition/config.ign" // findLabel finds the partition based on the label. The partition belongs to the image attached to the guestfish instance identified by pid. func findLabel(label, pid string) (string, error) { if pid == "" { return "", fmt.Errorf("The pid cannot be empty") } if label == "" { return "", fmt.Errorf("The label cannot be empty") } remote := fmt.Sprintf("--remote=%s", pid) cmd := exec.Command("guestfish", remote, "findfs-label", label) stdout, err := cmd.Output() if err != nil { return "", errors.Wrapf(err, "get stdout for findfs-label failed") } return strings.TrimSpace(string(stdout)), nil } type coreosGuestfish struct { cmd *exec.ExecCmd remote string } func newGuestfish(arch, diskImagePath string, diskSectorSize int) (*coreosGuestfish, error) { // Set guestfish backend to direct in order to avoid libvirt as backend. // Using libvirt can lead to permission denied issues if it does not have access // rights to the qcow image guestfishArgs := []string{"--listen"} if diskSectorSize != 0 { guestfishArgs = append(guestfishArgs, fmt.Sprintf("--blocksize=%d", diskSectorSize)) } guestfishArgs = append(guestfishArgs, "-a", diskImagePath) cmd := exec.Command("guestfish", guestfishArgs...) cmd.Env = append(os.Environ(), "LIBGUESTFS_BACKEND=direct") // make sure it inherits stderr so we see any error message cmd.Stderr = os.Stderr stdout, err := cmd.StdoutPipe() if err != nil { return nil, errors.Wrapf(err, "getting stdout pipe") } defer stdout.Close() if err := cmd.Start(); err != nil { return nil, errors.Wrapf(err, "running guestfish") } buf, err := io.ReadAll(stdout) if err != nil { return nil, errors.Wrapf(err, "reading guestfish output") } if err := cmd.Wait(); err != nil { return nil, errors.Wrapf(err, "waiting for guestfish response") } //GUESTFISH_PID=$PID; export GUESTFISH_PID gfVarPid := strings.Split(string(buf), ";") if len(gfVarPid) != 2 { return nil, fmt.Errorf("Failing parsing GUESTFISH_PID got: expecting length 2 got instead %d", len(gfVarPid)) } gfVarPidArr := strings.Split(gfVarPid[0], "=") if len(gfVarPidArr) != 2 { return nil, fmt.Errorf("Failing parsing GUESTFISH_PID got: expecting length 2 got instead %d", len(gfVarPid)) } pid := gfVarPidArr[1] remote := fmt.Sprintf("--remote=%s", pid) if err := exec.Command("guestfish", remote, "run").Run(); err != nil { return nil, errors.Wrapf(err, "guestfish launch failed") } rootfs, err := findLabel("root", pid) if err != nil { return nil, errors.Wrapf(err, "guestfish command failed to find root label") } if err := exec.Command("guestfish", remote, "mount", rootfs, "/").Run(); err != nil { return nil, errors.Wrapf(err, "guestfish root mount failed") } bootfs, err := findLabel("boot", pid) if err != nil { return nil, errors.Wrapf(err, "guestfish command failed to find boot label") } if err := exec.Command("guestfish", remote, "mount", bootfs, "/boot").Run(); err != nil { return nil, errors.Wrapf(err, "guestfish boot mount failed") } return &coreosGuestfish{ cmd: cmd, remote: remote, }, nil } func (gf *coreosGuestfish) destroy() { if err := exec.Command("guestfish", gf.remote, "exit").Run(); err != nil { plog.Errorf("guestfish exit failed: %v", err) } } // setupPreboot performs changes necessary before the disk is booted func setupPreboot(arch, confPath, firstbootkargs, kargs string, diskImagePath string, diskSectorSize int) error { gf, err := newGuestfish(arch, diskImagePath, diskSectorSize) if err != nil { return err } defer gf.destroy() if confPath != "" { if err := exec.Command("guestfish", gf.remote, "mkdir-p", "/boot/ignition").Run(); err != nil { return errors.Wrapf(err, "guestfish directory creation failed") } if err := exec.Command("guestfish", gf.remote, "upload", confPath, "/boot"+fileRemoteLocation).Run(); err != nil { return errors.Wrapf(err, "guestfish upload failed") } } // See /boot/grub2/grub.cfg if firstbootkargs != "" { grubStr := fmt.Sprintf("set ignition_network_kcmdline=\"%s\"\n", firstbootkargs) if err := exec.Command("guestfish", gf.remote, "write", "/boot/ignition.firstboot", grubStr).Run(); err != nil { return errors.Wrapf(err, "guestfish write") } } // Parsing BLS var linux string var initrd string var allkargs string zipl_sync := arch == "s390x" && (firstbootkargs != "" || kargs != "") if kargs != "" || zipl_sync { confpathout, err := exec.Command("guestfish", gf.remote, "glob-expand", "/boot/loader/entries/ostree*conf").Output() if err != nil { return errors.Wrapf(err, "finding bootloader config path") } confs := strings.Split(strings.TrimSpace(string(confpathout)), "\n") if len(confs) != 1 { return fmt.Errorf("Multiple values for bootloader config: %v", confpathout) } confpath := confs[0] origconf, err := exec.Command("guestfish", gf.remote, "read-file", confpath).Output() if err != nil { return errors.Wrapf(err, "reading bootloader config") } var buf strings.Builder for _, line := range strings.Split(string(origconf), "\n") { if strings.HasPrefix(line, "options ") { line += " " + kargs allkargs = strings.TrimPrefix(line, "options ") } else if strings.HasPrefix(line, "linux ") { linux = "/boot" + strings.TrimPrefix(line, "linux ") } else if strings.HasPrefix(line, "initrd ") { initrd = "/boot" + strings.TrimPrefix(line, "initrd ") } buf.Write([]byte(line)) buf.Write([]byte("\n")) } if kargs != "" { if err := exec.Command("guestfish", gf.remote, "write", confpath, buf.String()).Run(); err != nil { return errors.Wrapf(err, "writing bootloader config") } } } // s390x requires zipl to update low-level data on block device if zipl_sync { allkargs = strings.TrimSpace(allkargs + " ignition.firstboot " + firstbootkargs) if err := runZipl(gf, linux, initrd, allkargs); err != nil { return errors.Wrapf(err, "running zipl") } } if err := exec.Command("guestfish", gf.remote, "umount-all").Run(); err != nil { return errors.Wrapf(err, "guestfish umount failed") } return nil } func runZipl(gf *coreosGuestfish, linux string, initrd string, options string) error { // Detecting ostree commit deploy, err := exec.Command("guestfish", gf.remote, "glob-expand", "/ostree/deploy/*/deploy/*.0").Output() if err != nil { return errors.Wrapf(err, "finding deploy path") } sysroot := strings.TrimSpace(string(deploy)) // Saving cmdline if err := exec.Command("guestfish", gf.remote, "write", "/boot/zipl.cmdline", options+"\n").Run(); err != nil { return errors.Wrapf(err, "writing zipl cmdline") } // Bind-mounting for chroot if err := exec.Command("guestfish", gf.remote, "debug", "sh", fmt.Sprintf("'mount -t devtmpfs none /sysroot/%s/dev'", sysroot)).Run(); err != nil { return errors.Wrapf(err, "bind-mounting devtmpfs") } if err := exec.Command("guestfish", gf.remote, "debug", "sh", fmt.Sprintf("'mount -t proc none /sysroot/%s/proc'", sysroot)).Run(); err != nil { return errors.Wrapf(err, "bind-mounting /proc") } if err := exec.Command("guestfish", gf.remote, "debug", "sh", fmt.Sprintf("'mount -o bind /sysroot/boot /sysroot/%s/boot'", sysroot)).Run(); err != nil { return errors.Wrapf(err, "bind-mounting /boot") } // chroot zipl cmd := exec.Command("guestfish", gf.remote, "debug", "sh", fmt.Sprintf("'chroot /sysroot/%s /sbin/zipl -i %s -r %s -p /boot/zipl.cmdline -t /boot'", sysroot, linux, initrd)) cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { return errors.Wrapf(err, "running zipl") } // clean-up if err := exec.Command("guestfish", gf.remote, "rm-f", "/boot/zipl.cmdline").Run(); err != nil { return errors.Wrapf(err, "writing zipl cmdline") } return nil } func resolveBackingFile(backingFile string) (string, error) { backingFile, err := filepath.Abs(backingFile) if err != nil { return "", err } // Keep the COW image from breaking if the "latest" symlink changes. // Ignore /proc/*/fd/* paths, since they look like symlinks but // really aren't. if !strings.HasPrefix(backingFile, "/proc/") { backingFile, err = filepath.EvalSymlinks(backingFile) if err != nil { return "", err } } return backingFile, nil } // prepare creates the target disk and sets all the runtime attributes // for use by the QemuBuilder. func (disk *Disk) prepare(builder *QemuBuilder) error { if err := builder.ensureTempdir(); err != nil { return err } tmpf, err := os.CreateTemp(builder.tempdir, "disk") if err != nil { return err } disk.dstFileName = tmpf.Name() imgOpts := []string{"create", "-f", "qcow2", disk.dstFileName} // On filesystems like btrfs, qcow2 files can become much more fragmented // if copy-on-write is enabled. We don't need that, our disks are ephemeral. // https://gitlab.gnome.org/GNOME/gnome-boxes/-/issues/88 // https://btrfs.wiki.kernel.org/index.php/Gotchas#Fragmentation // https://www.redhat.com/archives/libvir-list/2014-July/msg00361.html qcow2Opts := "nocow=on" if disk.BackingFile != "" { backingFile, err := resolveBackingFile(disk.BackingFile) if err != nil { return err } qcow2Opts += fmt.Sprintf(",backing_file=%s,lazy_refcounts=on", backingFile) format := disk.BackingFormat if format == "" { // QEMU 5 warns if format is omitted, let's do detection for the common case // on our own. if strings.HasSuffix(backingFile, "qcow2") { format = "qcow2" } else if strings.HasSuffix(backingFile, "raw") { format = "raw" } } if format != "" { qcow2Opts += fmt.Sprintf(",backing_fmt=%s", format) } } imgOpts = append(imgOpts, "-o", qcow2Opts) if disk.Size != "" { imgOpts = append(imgOpts, disk.Size) } qemuImg := exec.Command("qemu-img", imgOpts...) qemuImg.Stderr = os.Stderr if err := qemuImg.Run(); err != nil { return err } fdSet := builder.AddFd(tmpf) disk.attachEndPoint = fdSet // MultiPathDisks must be NBD remote mounted if disk.MultiPathDisk || disk.NbdDisk { socketName := fmt.Sprintf("%s.socket", disk.dstFileName) shareCount := "1" if disk.MultiPathDisk { shareCount = "2" } disk.nbdServCmd = exec.Command("qemu-nbd", "--format", "qcow2", "--cache", "unsafe", "--discard", "unmap", "--socket", socketName, "--share", shareCount, disk.dstFileName) disk.attachEndPoint = fmt.Sprintf("nbd:unix:%s", socketName) } builder.diskID++ builder.disks = append(builder.disks, disk) return nil } func (builder *QemuBuilder) addDiskImpl(disk *Disk, primary bool) error { if err := disk.prepare(builder); err != nil { return err } if primary { // Only try to inject config if it hasn't already been injected somewhere // else, which can happen when running an ISO install. if !builder.configInjected { // If the board doesn't support -fw_cfg or we were explicitly // requested, inject via libguestfs on the primary disk. if err := builder.renderIgnition(); err != nil { return errors.Wrapf(err, "rendering ignition") } requiresInjection := builder.ConfigFile != "" && builder.ForceConfigInjection if requiresInjection || builder.AppendFirstbootKernelArgs != "" || builder.AppendKernelArgs != "" { if err := setupPreboot(builder.architecture, builder.ConfigFile, builder.AppendFirstbootKernelArgs, builder.AppendKernelArgs, disk.dstFileName, disk.SectorSize); err != nil { return errors.Wrapf(err, "ignition injection with guestfs failed") } builder.configInjected = true } } } diskOpts := disk.DeviceOpts if primary { diskOpts = append(diskOpts, "serial=primary-disk") } else { foundserial := false for _, opt := range diskOpts { if strings.HasPrefix(opt, "serial=") { foundserial = true } } if !foundserial { diskOpts = append(diskOpts, "serial="+fmt.Sprintf("disk%d", builder.diskID)) } } channel := disk.Channel if channel == "" { channel = "virtio" } if disk.SectorSize != 0 { logical := disk.LogicalSectorSize if logical == 0 { logical = disk.SectorSize } diskOpts = append(diskOpts, fmt.Sprintf("physical_block_size=%d,logical_block_size=%d", disk.SectorSize, logical)) } // Primary disk gets bootindex 1, all other disks have unspecified // bootindex, which means lower priority. if primary { diskOpts = append(diskOpts, "bootindex=1") } opts := "" if len(diskOpts) > 0 { opts = "," + strings.Join(diskOpts, ",") } id := fmt.Sprintf("disk-%d", builder.diskID) // Avoid file locking detection, and the disks we create // here are always currently ephemeral. defaultDiskOpts := "auto-read-only=off,cache=unsafe" if len(disk.DriveOpts) > 0 { defaultDiskOpts += "," + strings.Join(disk.DriveOpts, ",") } if disk.MultiPathDisk || channel == "scsi" { // Fake a NVME or SCSI device with a fake WWN. // The WWN needs to be a unique uint64 number wwn := disk.Wwn if wwn == 0 { wwn = rand.Uint64() } var bus string switch builder.architecture { case "x86_64", "ppc64le", "aarch64": bus = "pci" case "s390x": bus = "ccw" default: panic(fmt.Sprintf("Mantle doesn't know which bus type to use on %s", builder.architecture)) } if disk.MultiPathDisk { // All these attributes are needed in order // to trick multipath-tools that this is a "real" multipath device. // Each disk is presented on its own controller. for i := 0; i < 2; i++ { if i == 1 { opts = strings.Replace(opts, "bootindex=1", "bootindex=2", -1) } pID := fmt.Sprintf("mpath%d%d", builder.diskID, i) scsiID := fmt.Sprintf("scsi_%s", pID) builder.Append("-device", fmt.Sprintf("virtio-scsi-%s,id=%s", bus, scsiID)) builder.Append("-device", fmt.Sprintf("scsi-hd,bus=%s.0,drive=%s,vendor=NVME,product=VirtualMultipath,wwn=%d%s", scsiID, pID, wwn, opts)) builder.Append("-drive", fmt.Sprintf("if=none,id=%s,format=raw,file=%s,media=disk,%s", pID, disk.attachEndPoint, defaultDiskOpts)) } } else { scsiID := fmt.Sprintf("scsi_%d", builder.diskID) builder.Append("-device", fmt.Sprintf("virtio-scsi-%s,id=%s", bus, scsiID)) builder.Append("-device", fmt.Sprintf("scsi-hd,bus=%s.0,drive=%s,wwn=%d%s", scsiID, id, wwn, opts)) builder.Append("-drive", fmt.Sprintf("if=none,id=%s,file=%s,media=disk,%s", id, disk.attachEndPoint, defaultDiskOpts)) } } else { if !disk.NbdDisk { // In the non-multipath/nbd case we can just unlink the disk now // and avoid leaking space if we get Ctrl-C'd (though it's best if // higher level code catches SIGINT and cleans up the directory) os.Remove(disk.dstFileName) } disk.dstFileName = "" switch channel { case "virtio": builder.Append("-device", virtio(builder.architecture, "blk", fmt.Sprintf("drive=%s%s", id, opts))) case "nvme": builder.Append("-device", fmt.Sprintf("nvme,drive=%s%s", id, opts)) default: panic(fmt.Sprintf("Unhandled channel: %s", channel)) } // Default to cache=unsafe builder.Append("-drive", fmt.Sprintf("if=none,id=%s,file=%s,%s", id, disk.attachEndPoint, defaultDiskOpts)) } return nil } // AddPrimaryDisk sets up the primary disk for the instance. func (builder *QemuBuilder) AddPrimaryDisk(disk *Disk) error { if builder.primaryDisk != nil { return errors.New("Multiple primary disks specified") } // We do this one lazily in order to break an ordering requirement // for SetConfig() and AddPrimaryDisk() in the case where the // config needs to be injected into the disk. builder.primaryDisk = disk return nil } // AddBootDisk sets the instance to boot only from the target disk func (builder *QemuBuilder) AddBootDisk(disk *Disk) error { if err := builder.AddPrimaryDisk(disk); err != nil { return err } builder.primaryIsBoot = true return nil } // AddDisk adds a secondary disk for the instance. func (builder *QemuBuilder) AddDisk(disk *Disk) error { return builder.addDiskImpl(disk, false) } // AddDisksFromSpecs adds multiple secondary disks from their specs. func (builder *QemuBuilder) AddDisksFromSpecs(specs []string) error { for _, spec := range specs { if disk, err := ParseDisk(spec, false); err != nil { return errors.Wrapf(err, "parsing additional disk spec '%s'", spec) } else if err = builder.AddDisk(disk); err != nil { return errors.Wrapf(err, "adding additional disk '%s'", spec) } } return nil } // AddIso adds an ISO image, optionally configuring its boot index // If asDisk is set, attach the ISO as a disk drive (as though it was copied // to a USB stick) and overwrite the El Torito signature in the image // (to force QEMU's UEFI firmware to boot via the hybrid ESP). func (builder *QemuBuilder) AddIso(path string, bootindexStr string, asDisk bool) error { builder.iso = &bootIso{ path: path, bootindex: bootindexStr, } builder.isoAsDisk = asDisk return nil } // DefaultMemoryMiB returns the default QEMU guest memory in MiB for // the given architecture. If no architecture is provided, the current // host architecture is used. The historical defaults are 1024 for // x86_64 and 2048 for all other architectures. func DefaultMemoryMiB(arch ...string) int { a := "" if len(arch) > 0 { a = arch[0] } else { a = coreosarch.CurrentRpmArch() } switch a { case "aarch64", "s390x", "ppc64le": return 2048 default: return 1024 } } func (builder *QemuBuilder) finalize() { if builder.finalized { return } if builder.MemoryMiB == 0 { builder.MemoryMiB = DefaultMemoryMiB(builder.architecture) } builder.finalized = true } // Append appends additional arguments for QEMU. func (builder *QemuBuilder) Append(args ...string) { builder.Argv = append(builder.Argv, args...) } // platformQemuArgs returns platform-specific QEMU arguments for the given architecture. func platformQemuArgs(arch, machineArg string) ([]string, error) { kvm := true hostArch := coreosarch.CurrentRpmArch() accelArg := "accel=kvm" if _, ok := os.LookupEnv("COSA_NO_KVM"); ok || hostArch != arch { accelArg = "accel=tcg" kvm = false } if machineArg == "" { machineArg = accelArg } else { machineArg += "," + accelArg } var ret []string switch arch { case "x86_64": ret = []string{ "qemu-system-x86_64", "-machine", machineArg, } case "aarch64": ret = []string{ "qemu-system-aarch64", "-machine", "virt,gic-version=max," + machineArg, } case "s390x": ret = []string{ "qemu-system-s390x", "-machine", "s390-ccw-virtio," + machineArg, } case "ppc64le": ret = []string{ "qemu-system-ppc64", // kvm-type=HV ensures we use bare metal KVM and not "user mode" // https://www.qemu.org/docs/master/system/ppc/pseries.html "-machine", "pseries,kvm-type=HV,ic-mode=xics," + machineArg, } default: return nil, fmt.Errorf("architecture %s not supported for qemu", arch) } if kvm { ret = append(ret, "-cpu", "host") } else { if arch == "x86_64" { // the default qemu64 CPU model does not support x86_64_v2 // causing crashes on EL9+ kernels // see https://bugzilla.redhat.com/show_bug.cgi?id=2060839 ret = append(ret, "-cpu", "Nehalem") } } return ret, nil } // baseNumaQemuArgs returns QEMU arguments to simulate two NUMA nodes. Node1 and Node0 will try to // split the machine's resources evenly between them. // See: https://www.qemu.org/docs/master/system/invocation.html#cmdoption-smp func baseNumaQemuArgs(arch string, memoryMiB, cpus int) ([]string, error) { if cpus < 2 || cpus%2 != 0 { return nil, fmt.Errorf("must have an even number of CPUs to simulate NUMA nodes") } ret, err := platformQemuArgs(arch, "") if err != nil { return nil, err } // Due to a limitation, all memory must be assigned. Memory is split as evenly as possible between the NUMA nodes; when // the total memory is odd, node1 will receive the remainder. const node0MemoryDevice = "mem0" const node1MemoryDevice = "mem1" node0MemoryMiB := memoryMiB / 2 node1MemoryMiB := memoryMiB - node0MemoryMiB // Split CPUs evenly between nodes (node1 gets the extra CPU if total is odd) // Node0: 0 to half-1, Node1: half to total-1 node0StartCpu, node0EndCpu := 0, cpus/2-1 node1StartCpu, node1EndCpu := cpus/2, cpus-1 // Define memory using a memfd (in shared mode), which is needed for virtiofs. // Also preallocate all memory up front so scheduling tests based on memory // availability can be more predictable. ret = append(ret, "-object", fmt.Sprintf("memory-backend-memfd,id=%s,size=%dM,share=on,prealloc=on", node0MemoryDevice, node0MemoryMiB)) ret = append(ret, "-object", fmt.Sprintf("memory-backend-memfd,id=%s,size=%dM,share=on,prealloc=on", node1MemoryDevice, node1MemoryMiB)) ret = append(ret, "-numa", fmt.Sprintf("node,memdev=%s,cpus=%d-%d,nodeid=0", node0MemoryDevice, node0StartCpu, node0EndCpu)) ret = append(ret, "-numa", fmt.Sprintf("node,memdev=%s,cpus=%d-%d,nodeid=1", node1MemoryDevice, node1StartCpu, node1EndCpu)) ret = append(ret, "-m", fmt.Sprintf("%d", memoryMiB)) // to avoid kernel warnings, we need to have two sockets ret = append(ret, "-smp", fmt.Sprintf("%d,sockets=2", cpus)) return ret, nil } // baseQemuArgs takes a board and returns the basic qemu // arguments needed for the current architecture. func baseQemuArgs(arch string, memoryMiB, cpus int) ([]string, error) { // memoryDevice is the object identifier we use for the backing RAM const memoryDevice = "mem" // The machine argument needs to reference our memory device; see below machineArg := "memory-backend=" + memoryDevice ret, err := platformQemuArgs(arch, machineArg) if err != nil { return nil, err } // Define memory using a memfd (in shared mode), which is needed for virtiofs. // Also preallocate all memory up front so scheduling tests based on memory // availability can be more predictable. ret = append(ret, "-object", fmt.Sprintf("memory-backend-memfd,id=%s,size=%dM,share=on,prealloc=on", memoryDevice, memoryMiB)) ret = append(ret, "-m", fmt.Sprintf("%d", memoryMiB)) ret = append(ret, "-smp", fmt.Sprintf("%d", cpus)) return ret, nil } func (builder *QemuBuilder) setupUefi(secureBoot bool) error { switch coreosarch.CurrentRpmArch() { case "x86_64": varsVariant := "" if secureBoot { varsVariant = ".secboot" } varsSrc, err := os.Open(fmt.Sprintf("/usr/share/edk2/ovmf/OVMF_VARS%s.fd", varsVariant)) if err != nil { return err } defer varsSrc.Close() vars, err := os.CreateTemp("", "mantle-qemu") if err != nil { return err } if _, err := io.Copy(vars, varsSrc); err != nil { return err } _, err = vars.Seek(0, 0) if err != nil { return err } fdset := builder.AddFd(vars) builder.Append("-drive", fmt.Sprintf("file=/usr/share/edk2/ovmf/OVMF_CODE%s.fd,if=pflash,format=raw,unit=0,readonly=on,auto-read-only=off", varsVariant)) builder.Append("-drive", fmt.Sprintf("file=%s,if=pflash,format=raw,unit=1,readonly=off,auto-read-only=off", fdset)) builder.Append("-machine", "q35") case "aarch64": if secureBoot { return fmt.Errorf("architecture %s doesn't have support for secure boot in kola", coreosarch.CurrentRpmArch()) } vars, err := os.CreateTemp("", "mantle-qemu") if err != nil { return err } //67108864 bytes is expected size of the "VARS" by qemu err = vars.Truncate(67108864) if err != nil { return err } _, err = vars.Seek(0, 0) if err != nil { return err } fdset := builder.AddFd(vars) builder.Append("-drive", "file=/usr/share/edk2/aarch64/QEMU_EFI-silent-pflash.raw,if=pflash,format=raw,unit=0,readonly=on,auto-read-only=off") builder.Append("-drive", fmt.Sprintf("file=%s,if=pflash,format=raw,unit=1,readonly=off,auto-read-only=off", fdset)) default: panic(fmt.Sprintf("Architecture %s doesn't have support for UEFI in qemu.", coreosarch.CurrentRpmArch())) } return nil } // Checks whether coreos-installer has // https://github.com/coreos/coreos-installer/pull/341. Can be dropped once // that PR is in all the cosa branches we care about. func coreosInstallerSupportsISOKargs() (bool, error) { cmd := exec.Command("coreos-installer", "iso", "--help") cmd.Stderr = os.Stderr var outb bytes.Buffer cmd.Stdout = &outb if err := cmd.Run(); err != nil { return false, errors.Wrapf(err, "running coreos-installer iso --help") } out := outb.String() return strings.Contains(out, "kargs"), nil } func (builder *QemuBuilder) setupIso() error { if err := builder.ensureTempdir(); err != nil { return err } // TODO change to use something like an unlinked tempfile (or O_TMPFILE) // in the same filesystem as the source so that reflinks (if available) // will work isoEmbeddedPath := filepath.Join(builder.tempdir, "install.iso") cpcmd := exec.Command("cp", "--reflink=auto", builder.iso.path, isoEmbeddedPath) cpcmd.Stderr = os.Stderr if err := cpcmd.Run(); err != nil { return errors.Wrapf(err, "copying iso") } // Make it writable so we can modify it if err := os.Chmod(isoEmbeddedPath, 0644); err != nil { return errors.Wrapf(err, "setting permissions on iso") } if builder.ConfigFile != "" { if builder.configInjected { panic("config already injected?") } configf, err := os.Open(builder.ConfigFile) if err != nil { return err } instCmd := exec.Command("coreos-installer", "iso", "ignition", "embed", isoEmbeddedPath) instCmd.Stdin = configf instCmd.Stderr = os.Stderr if err := instCmd.Run(); err != nil { return errors.Wrapf(err, "running coreos-installer iso embed") } builder.configInjected = true } if kargsSupported, err := coreosInstallerSupportsISOKargs(); err != nil { return err } else if kargsSupported { allargs := fmt.Sprintf("console=%s %s", ConsoleKernelArgument[coreosarch.CurrentRpmArch()], builder.AppendKernelArgs) instCmdKargs := exec.Command("coreos-installer", "iso", "kargs", "modify", "--append", allargs, isoEmbeddedPath) var stderrb bytes.Buffer instCmdKargs.Stderr = &stderrb if err := instCmdKargs.Run(); err != nil { // Don't make this a hard error if it's just for console; we // may be operating on an old live ISO if len(builder.AppendKernelArgs) > 0 { return errors.Wrapf(err, "running `coreos-installer iso kargs modify`; old CoreOS ISO?") } // Only actually emit a warning if we expected it to be supported stderr := stderrb.String() plog.Warningf("running coreos-installer iso kargs modify: %v: %q", err, stderr) plog.Warning("likely targeting an old CoreOS ISO; ignoring...") } } else if len(builder.AppendKernelArgs) > 0 { return fmt.Errorf("coreos-installer does not support appending kernel args") } if builder.isoAsDisk { f, err := os.OpenFile(isoEmbeddedPath, os.O_WRONLY, 0) if err != nil { return errors.Wrapf(err, "opening ISO image for writing") } defer f.Close() // Invalidate Boot System Identifier in the El Torito Boot // Record Volume Descriptor so the system must boot via the // MBR or ESP. If we don't do this, QEMU's UEFI firmware // will boot via El Torito anyway, which doesn't match what // a lot of UEFI firmware does. _, err = f.WriteAt([]byte("NO"), 34823) if err != nil { return errors.Wrapf(err, "overwriting El Torito signature") } } builder.iso.path = isoEmbeddedPath // Arches s390x and ppc64le don't support UEFI and use the cdrom option to boot the ISO. // For all other arches we use ide-cd device with bootindex=2 here: the idea is // that during an ISO install, the primary disk isn't bootable, so the bootloader // will fall back to the ISO boot. On reboot when the system is installed, the // primary disk is selected. This allows us to have "boot once" functionality on // both UEFI and BIOS (`-boot once=d` OTOH doesn't work with OVMF). switch coreosarch.CurrentRpmArch() { case "s390x": if builder.isoAsDisk { // we could do it, but boot would fail return errors.New("cannot attach ISO as disk; no hybrid ISO on this arch") } builder.Append("-blockdev", "file,node-name=installiso,filename="+builder.iso.path, "-device", "virtio-scsi", "-device", "scsi-cd,drive=installiso,bootindex=2") case "ppc64le", "aarch64": if builder.isoAsDisk { // we could do it, but boot would fail return errors.New("cannot attach ISO as disk; no hybrid ISO on this arch") } builder.Append("-drive", "file="+builder.iso.path+",id=installiso,index=2,media=cdrom") default: bootindexStr := "" if builder.iso.bootindex != "" { bootindexStr = "," + builder.iso.bootindex } builder.Append("-drive", "file="+builder.iso.path+",format=raw,if=none,readonly=on,id=installiso") if builder.isoAsDisk { builder.Append("-device", virtio(builder.architecture, "blk", "drive=installiso"+bootindexStr)) } else { builder.Append("-device", "ide-cd,drive=installiso"+bootindexStr) } } return nil } // VirtioChannelRead allocates a virtio-serial channel that will appear in // the guest as /dev/virtio-ports/. The guest can write to it, and // the host can read. func (builder *QemuBuilder) VirtioChannelRead(name string) (*os.File, error) { // Set up the virtio channel to get Ignition failures by default r, w, err := os.Pipe() if err != nil { return nil, errors.Wrapf(err, "virtioChannelRead creating pipe") } if builder.virtioSerialID == 0 { builder.Append("-device", "virtio-serial") } builder.virtioSerialID++ id := fmt.Sprintf("virtioserial%d", builder.virtioSerialID) // https://www.redhat.com/archives/libvir-list/2015-December/msg00305.html builder.Append("-chardev", fmt.Sprintf("file,id=%s,path=%s,append=on", id, builder.AddFd(w))) builder.Append("-device", fmt.Sprintf("virtserialport,chardev=%s,name=%s", id, name)) return r, nil } // SerialPipe reads the serial console output into a pipe func (builder *QemuBuilder) SerialPipe() (*os.File, error) { r, w, err := os.Pipe() if err != nil { return nil, errors.Wrapf(err, "virtioChannelRead creating pipe") } id := "serialpipe" builder.Append("-chardev", fmt.Sprintf("file,id=%s,path=%s,append=on", id, builder.AddFd(w))) builder.Append("-serial", fmt.Sprintf("chardev:%s", id)) return r, nil } // VirtioJournal configures the OS and VM to stream the systemd journal // (post-switchroot) over a virtio-serial channel. // - The first parameter is a pointer to the configuration of the target VM. // - The second parameter (optional) is queryArguments to filter the stream - // passed through to AddVirtioJournalUnit(queryArguments...) // - The return value is a file stream which will be newline-separated JSON. func (builder *QemuBuilder) VirtioJournal(config *conf.Conf, queryArguments ...string) (*os.File, error) { stream, err := builder.VirtioChannelRead(conf.VirtioJournalDeviceName) if err != nil { return nil, err } config.AddVirtioJournalUnit(queryArguments...) return stream, nil } // createVirtiofsCmd returns a new command instance configured to launch virtiofsd. func createVirtiofsCmd(directory, socketPath string) exec.Cmd { args := []string{"--sandbox", "none", "--socket-path", socketPath, "--shared-dir", "."} // Work around https://gitlab.com/virtio-fs/virtiofsd/-/merge_requests/197 if os.Getuid() == 0 { args = append(args, "--modcaps=-mknod:-setfcap") } // We don't need seccomp filtering; we trust our workloads. This incidentally // works around issues like https://gitlab.com/virtio-fs/virtiofsd/-/merge_requests/200. args = append(args, "--seccomp=none") cmd := exec.Command("/usr/libexec/virtiofsd", args...) // This sets things up so that the `.` we passed in the arguments is the target directory cmd.Dir = directory // Quiet the daemon by default cmd.Env = append(cmd.Env, "RUST_LOG=ERROR") // But we do want to see errors cmd.Stderr = os.Stderr // Like other processes, "lifecycle bind" it to us cmd.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGTERM, } return cmd } // Exec tries to run a QEMU instance with the given settings. func (builder *QemuBuilder) Exec() (*QemuInstance, error) { builder.finalize() var err error if err := builder.renderIgnition(); err != nil { return nil, errors.Wrapf(err, "rendering ignition") } inst := QemuInstance{} cleanupInst := false defer func() { if cleanupInst { inst.Destroy() } }() if builder.Processors < 0 { nproc, err := system.GetProcessors() if err != nil { return nil, errors.Wrapf(err, "qemu estimating processors") } // cap qemu smp at some reasonable level; sometimes our tooling runs // on 32-core servers (64 hyperthreads) and there's no reason to // try to match that. if nproc > 16 { nproc = 16 } builder.Processors = int(nproc) } else if builder.Processors == 0 { builder.Processors = 1 } // Ensure that there is at least one CPU for each NUMA node if builder.NumaNodes && builder.Processors < 2 { nproc, err := system.GetProcessors() if err != nil { return nil, errors.Wrapf(err, "qemu estimating processors") } if nproc < 2 { return nil, fmt.Errorf("there are not enough processors for the NUMA nodes") } builder.Processors = 2 } var argv []string if builder.NumaNodes { argv, err = baseNumaQemuArgs(builder.architecture, builder.MemoryMiB, builder.Processors) } else { argv, err = baseQemuArgs(builder.architecture, builder.MemoryMiB, builder.Processors) } if err != nil { return nil, err } switch builder.Firmware { case "": // Nothing to do, use qemu default case "uefi": if err := builder.setupUefi(false); err != nil { return nil, err } case "uefi-secure": if err := builder.setupUefi(true); err != nil { return nil, err } case "bios": if coreosarch.CurrentRpmArch() != "x86_64" { return nil, fmt.Errorf("unknown firmware: %s", builder.Firmware) } default: return nil, fmt.Errorf("unknown firmware: %s", builder.Firmware) } // We always provide a random source argv = append(argv, "-object", "rng-random,filename=/dev/urandom,id=rng0", "-device", virtio(builder.architecture, "rng", "rng=rng0")) if builder.UUID != "" { argv = append(argv, "-uuid", builder.UUID) } // We never want a popup window argv = append(argv, "-nographic") // We want to customize everything from scratch, so avoid defaults argv = append(argv, "-nodefaults") // We only render Ignition lazily, because we want to support calling // SetConfig() after AddPrimaryDisk() or AddInstallIso(). if builder.iso != nil { if err := builder.setupIso(); err != nil { return nil, err } } if builder.primaryDisk != nil { if err := builder.addDiskImpl(builder.primaryDisk, true); err != nil { return nil, err } if builder.primaryIsBoot { argv = append(argv, "-boot", "order=c,strict=on") } } // Handle Ignition if it wasn't already injected above if builder.ConfigFile != "" && !builder.configInjected { if builder.supportsFwCfg() { builder.Append("-fw_cfg", "name=opt/com.coreos/config,file="+builder.ConfigFile) } else { serial := "ignition" if builder.secureExecution { // SE case: we have to encrypt the config and attach it with 'serial=ignition_crypted' if err := builder.encryptIgnitionConfig(); err != nil { return nil, err } serial = "ignition_crypted" } // Alternative to fw_cfg, should be generally usable on all arches, // especially those without fw_cfg support. // See https://github.com/coreos/ignition/pull/905 builder.Append("-drive", fmt.Sprintf("if=none,id=ignition,format=raw,file=%s,readonly=on", builder.ConfigFile), "-device", fmt.Sprintf("virtio-blk,serial=%s,drive=ignition", serial)) } } // Start up the disks. Since the disk may be served via NBD, // we can't use builder.AddFd (no support for fdsets), so we at the disk to the tmpFiles. for _, disk := range builder.disks { if disk.nbdServCmd != nil { cmd := disk.nbdServCmd.(*exec.ExecCmd) if err := cmd.Start(); err != nil { return nil, errors.Wrapf(err, "spawing nbd server") } inst.helpers = append(inst.helpers, cmd) } } // Handle Usermode Networking if builder.UsermodeNetworking { if err := builder.setupNetworking(); err != nil { return nil, err } inst.hostForwardedPorts = builder.requestedHostForwardPorts } // Handle Additional NICs networking if builder.additionalNics > 0 { if err := builder.setupAdditionalNetworking(); err != nil { return nil, err } } // Handle Software TPM if builder.Swtpm && builder.supportsSwtpm() { err = builder.ensureTempdir() if err != nil { return nil, err } swtpmSock := filepath.Join(builder.tempdir, "swtpm-sock") swtpmdir := filepath.Join(builder.tempdir, "swtpm") if err := os.Mkdir(swtpmdir, 0755); err != nil { return nil, err } inst.swtpm = exec.Command("swtpm", "socket", "--tpm2", "--ctrl", fmt.Sprintf("type=unixio,path=%s", swtpmSock), "--terminate", "--tpmstate", fmt.Sprintf("dir=%s", swtpmdir)) cmd := inst.swtpm.(*exec.ExecCmd) // For now silence the swtpm stderr as it prints errors when // disconnected, but that's normal. if builder.Pdeathsig { cmd.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGTERM, } } if err = inst.swtpm.Start(); err != nil { return nil, err } // We need to wait until the swtpm starts up err = util.Retry(10, 500*time.Millisecond, func() error { _, err := os.Stat(swtpmSock) return err }) if err != nil { return nil, err } argv = append(argv, "-chardev", fmt.Sprintf("socket,id=chrtpm,path=%s", swtpmSock), "-tpmdev", "emulator,id=tpm0,chardev=chrtpm") // There are different device backends on each architecture switch builder.architecture { case "x86_64": argv = append(argv, "-device", "tpm-tis,tpmdev=tpm0") case "aarch64": argv = append(argv, "-device", "tpm-tis-device,tpmdev=tpm0") case "ppc64le": argv = append(argv, "-device", "tpm-spapr,tpmdev=tpm0") } } // Set up QMP (currently used to switch boot order after first boot on aarch64. // The qmp socket path must be unique to the instance. inst.qmpSocketPath = filepath.Join(builder.tempdir, fmt.Sprintf("qmp-%d.sock", time.Now().UnixNano())) qmpID := "qemu-qmp" builder.Append("-chardev", fmt.Sprintf("socket,id=%s,path=%s,server=on,wait=off", qmpID, inst.qmpSocketPath)) builder.Append("-mon", fmt.Sprintf("chardev=%s,mode=control", qmpID)) // Set up the virtio channel to get Ignition failures by default journalPipeR, err := builder.VirtioChannelRead("com.coreos.ignition.journal") inst.journalPipe = journalPipeR if err != nil { return nil, err } // Process virtiofs mounts if len(builder.hostMounts) > 0 { if err := builder.ensureTempdir(); err != nil { return nil, err } plog.Debug("creating virtiofs helpers") // Spawn off a virtiofsd helper per mounted path virtiofsHelpers := make(map[string]exec.Cmd) for i, hostmnt := range builder.hostMounts { // By far the most common failure to spawn virtiofsd will be a typo'd source directory, // so let's synchronously check that ourselves here. if _, err := os.Stat(hostmnt.src); err != nil { return nil, fmt.Errorf("failed to access virtiofs source directory %s", hostmnt.src) } virtiofsChar := fmt.Sprintf("virtiofschar%d", i) virtiofsdSocket := filepath.Join(builder.tempdir, fmt.Sprintf("virtiofsd-%d.sock", i)) builder.Append("-chardev", fmt.Sprintf("socket,id=%s,path=%s", virtiofsChar, virtiofsdSocket)) builder.Append("-device", fmt.Sprintf("vhost-user-fs-pci,queue-size=1024,chardev=%s,tag=%s", virtiofsChar, hostmnt.dest)) plog.Debugf("creating virtiofs helper for %s", hostmnt.src) // TODO: Honor hostmnt.readonly somehow here (add an option to virtiofsd) p := createVirtiofsCmd(hostmnt.src, virtiofsdSocket) if err := p.Start(); err != nil { return nil, fmt.Errorf("failed to start virtiofsd") } virtiofsHelpers[virtiofsdSocket] = p } // Loop waiting for the sockets to appear err := util.RetryUntilTimeout(10*time.Minute, 1*time.Second, func() error { found := []string{} for sockpath := range virtiofsHelpers { if _, err := os.Stat(sockpath); err == nil { found = append(found, sockpath) } } for _, sockpath := range found { helper := virtiofsHelpers[sockpath] inst.helpers = append(inst.helpers, helper) delete(virtiofsHelpers, sockpath) } if len(virtiofsHelpers) == 0 { return nil } waitingFor := []string{} for socket := range virtiofsHelpers { waitingFor = append(waitingFor, socket) } return fmt.Errorf("waiting for virtiofsd sockets: %s", strings.Join(waitingFor, " ")) }) if err != nil { return nil, err } } fdnum := 3 // first additional file starts at position 3 for i := range builder.fds { fdset := i + 1 // Start at 1 argv = append(argv, "-add-fd", fmt.Sprintf("fd=%d,set=%d", fdnum, fdset)) fdnum++ } if builder.ConsoleFile != "" { builder.Append("-display", "none", "-chardev", "file,id=log,path="+builder.ConsoleFile, "-serial", "chardev:log") } else { builder.Append("-serial", "mon:stdio") } // And the custom arguments argv = append(argv, builder.Argv...) inst.qemu = exec.Command(argv[0], argv[1:]...) inst.architecture = builder.architecture cmd := inst.qemu.(*exec.ExecCmd) cmd.Stderr = os.Stderr if builder.Pdeathsig { cmd.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGTERM, } } cmd.ExtraFiles = append(cmd.ExtraFiles, builder.fds...) if builder.InheritConsole { cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr } if err = inst.qemu.Start(); err != nil { return nil, err } plog.Debugf("Started qemu (%v) with args: %v", inst.qemu.Pid(), argv) // Transfer ownership of the tempdir inst.tempdir = builder.tempdir builder.tempdir = "" cleanupInst = false // Connect to the QMP socket which allows us to control qemu. We wait up to 30s // to avoid flakes on loaded CI systems. But, probably rather than bumping this // any higher it'd be better to try to reduce parallelism. if err := util.Retry(30, 1*time.Second, func() error { sockMonitor, err := qmp.NewSocketMonitor("unix", inst.qmpSocketPath, 2*time.Second) if err != nil { return err } inst.qmpSocket = sockMonitor return nil }); err != nil { return nil, fmt.Errorf("failed to establish qmp connection: %w", err) } if err := inst.qmpSocket.Connect(); err != nil { return nil, fmt.Errorf("failed to connect over qmp to qemu instance") } // Hacky code to test https://github.com/openshift/os/pull/1346 if timeout, ok := os.LookupEnv("COSA_TEST_CDROM_UNPLUG"); ok { val, err := time.ParseDuration(timeout) if err != nil { return nil, err } go func() { devs, err := inst.listDevices() if err != nil { plog.Error("failed to list devices") return } var cdrom string for _, dev := range devs.Return { switch dev.Type { case "child": cdrom = filepath.Join("/machine/peripheral-anon", dev.Name) default: break } } if cdrom == "" { plog.Errorf("failed to get scsi-cd id") return } plog.Debugf("get cdrom id %s", cdrom) time.Sleep(val) if err := inst.deleteBlockDevice(cdrom); err != nil { plog.Errorf("failed to delete block device: %s", cdrom) return } plog.Info("delete cdrom") }() } return &inst, nil } // Close drops all resources owned by the builder. func (builder *QemuBuilder) Close() { if builder.fds == nil { return } for _, f := range builder.fds { f.Close() } builder.fds = nil if builder.tempdir != "" { os.RemoveAll(builder.tempdir) } } // supports IBM Cex based LUKS encryption if it is s390x host (zKVM/LPAR) func (builder *QemuBuilder) AddCexDevice() error { cex_uuid := os.Getenv("KOLA_CEX_UUID") if cex_uuid == "" { return errors.New("cannot add CEX device: KOLA_CEX_UUID env var undefined") } builder.Append("-device", fmt.Sprintf("vfio-ap,sysfsdev=/sys/devices/vfio_ap/matrix/%s", cex_uuid)) return nil }