Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

31 changes: 26 additions & 5 deletions bin/propolis-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@ use clap::{Args, Parser, Subcommand};
use futures::{future, SinkExt};
use newtype_uuid::{GenericUuid, TypedUuid, TypedUuidKind, TypedUuidTag};
use propolis_client::instance_spec::{
BlobStorageBackend, Board, Chipset, Component, CrucibleStorageBackend,
I440Fx, InstanceMetadata, InstanceProperties, InstanceSpec,
InstanceSpecGetResponse, NvmeDisk, PciPath, QemuPvpanic,
ReplacementComponent, SerialPort, SerialPortNumber, SpecKey, VirtioDisk,
BlobStorageBackend, Board, Chipset, Component, CpuTopology,
CrucibleStorageBackend, I440Fx, InstanceMetadata, InstanceProperties,
InstanceSpec, InstanceSpecGetResponse, NvmeDisk, PciPath, QemuPvpanic,
ReplacementComponent, SerialPort, SerialPortNumber, SpecKey, TopoLeaves,
VirtioDisk,
};
use propolis_client::support::nvme_serial_from_str;
use propolis_client::types::{
Expand Down Expand Up @@ -174,7 +175,7 @@ struct VmConfig {

/// Number of vCPUs allocated to instance
#[clap(short = 'c', default_value = "4", action, requires = "config_toml")]
vcpus: u8,
vcpus: u16,

/// Memory allocated to instance (MiB)
#[clap(short, default_value = "1024", action, requires = "config_toml")]
Expand Down Expand Up @@ -330,11 +331,31 @@ impl VmConfig {
.map(|cfg| cfg.hv_interface.clone())
.unwrap_or_default();

let smt = parsed_toml
.as_ref()
.map(|cfg| cfg.machine_settings.smt)
.unwrap_or(true);

let sockets = parsed_toml
.as_ref()
.map(|cfg| cfg.machine_settings.sockets)
.unwrap_or(1);

let mut spec = InstanceSpec {
board: Board {
chipset: Chipset::I440Fx(I440Fx { enable_pcie }),
cpuid: cpuid_profile,
cpus: self.vcpus,
cpu_topology: CpuTopology {
has_smt: smt,
sockets,
topo_leaves: TopoLeaves {
std_4: true,
std_b: true,
ext_8: true,
ext_1e: false,
},
},
memory_mb: self.memory,
guest_hv_interface,
},
Expand Down
114 changes: 79 additions & 35 deletions bin/propolis-server/src/lib/initializer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

use std::convert::TryInto;
use std::fs::File;
use std::num::{NonZeroU8, NonZeroUsize};
use std::num::{NonZeroU16, NonZeroUsize};
use std::os::unix::fs::FileTypeExt;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
Expand Down Expand Up @@ -51,6 +51,7 @@ use propolis::intr_pins;
use propolis::vmm::{self, Builder, Machine};
use propolis::vsock::GuestCid;
use propolis_api_types::instance::InstanceProperties;
use propolis_api_types::instance_spec::components::board::TopoLeaves;
use propolis_api_types::instance_spec::components::devices::SerialPortNumber;
use propolis_api_types::instance_spec::{self, SpecKey};
use propolis_types::{CpuidIdent, CpuidVendor};
Expand Down Expand Up @@ -119,6 +120,9 @@ pub enum MachineInitError {
#[error("failed to start attestation server")]
AttestationServer(#[source] std::io::Error),

#[error("unsupported VM configuration")]
UnsupportedCpuConfig(&'static str),

#[cfg(feature = "falcon")]
#[error("softnpu p9 device missing")]
SoftNpuP9Missing,
Expand Down Expand Up @@ -1220,7 +1224,7 @@ impl MachineInitializer<'_> {
fn generate_smbios(
&self,
bootrom_version: &Option<String>,
) -> smbios::TableBytes {
) -> Result<smbios::TableBytes, MachineInitError> {
use smbios::table::{type0, type1, type16, type4};

let rom_size =
Expand Down Expand Up @@ -1328,6 +1332,12 @@ impl MachineInitializer<'_> {
.and_then(|vals| propolis::cpuid::parse_brand_string(vals).ok())
.unwrap_or_default();

let num_cpus: u8 = self.spec.board.cpus.try_into().map_err(|_| {
MachineInitError::UnsupportedCpuConfig(
"more CPUs than expressible in Type 4 SMBIOS v2.7 tables",
)
})?;

let smb_type4 = smbios::table::Type4 {
proc_type: type4::ProcType::Central,
proc_family,
Expand All @@ -1338,9 +1348,9 @@ impl MachineInitializer<'_> {
// unknown
proc_upgrade: 0x2,
// make core and thread counts equal for now
core_count: self.spec.board.cpus,
core_enabled: self.spec.board.cpus,
thread_count: self.spec.board.cpus,
core_count: num_cpus,
core_enabled: num_cpus,
thread_count: num_cpus,
proc_characteristics: type4::Characteristics::IS_64_BIT
| type4::Characteristics::MULTI_CORE,
..Default::default()
Expand Down Expand Up @@ -1382,7 +1392,7 @@ impl MachineInitializer<'_> {
smb_tables.add(0x1700.into(), &smb_type17).unwrap();
smb_tables.add(0x3200.into(), &smb_type32).unwrap();

smb_tables.commit()
Ok(smb_tables.commit())
}

fn generate_e820(&self) -> Result<Entry, MachineInitError> {
Expand Down Expand Up @@ -1469,7 +1479,7 @@ impl MachineInitializer<'_> {
fn generate_acpi_tables(
&self,
acpi_variant: acpi::AcpiVariant,
cpus: u8,
cpus: u16,
) -> Result<fwcfg::formats::AcpiTables, MachineInitError> {
let (lowmem, _) = get_spec_guest_ram_limits(self.spec);
let generators: Vec<_> = self
Expand Down Expand Up @@ -1509,7 +1519,7 @@ impl MachineInitializer<'_> {
/// Should not be called before [`Self::initialize_rom()`].
pub fn initialize_fwcfg(
&mut self,
cpus: u8,
cpus: u16,
bootrom_version: &Option<String>,
acpi_variant: acpi::AcpiVariant,
) -> Result<Arc<ramfb::RamFb>, MachineInitError> {
Expand All @@ -1522,7 +1532,7 @@ impl MachineInitializer<'_> {
.map_err(|e| MachineInitError::FwcfgInsertFailed("cpu count", e))?;

let smbios::TableBytes { entry_point, structure_table } =
self.generate_smbios(bootrom_version);
self.generate_smbios(bootrom_version)?;
fwcfg
.insert_named(
"etc/smbios/smbios-tables",
Expand Down Expand Up @@ -1580,11 +1590,49 @@ impl MachineInitializer<'_> {
Ok(ramfb)
}

fn topo_cpuid_leaves(&self) -> Result<Vec<TopoKind>, MachineInitError> {
// This is trying to find a local minima of annoying.. the alternative
// here would to have `CpuTopology` hold a `Vec<TopoKind>` directly.
// That opens questions like, what is the semantic of including a leaf
// multiple times? And we would want another glue type between the
// `propolis-server`-internal `TopoKind` that we use, to whatever
// version's external `TopoKind`. On those axes, it would be mildly
// simpler to just have a bag of bools for what leaves we want to
// specialize in this CPU topology.
//
// The downside of a bag of bools is that we have to map them to the
// internal `TopoKinds` enum at some point. This is that point. This
// conversion is incredibly typo-sensitive: would you spot
// `if std_4 { topo_leaves.push(TopoKind::StdB); }`? But converting
// across `TopoKinds` enums in versioned conversions would be similarly
// sensitive, and at least with a bag of booleans the fields are named
// and `std_4: std_b` would stand out.
let mut topo_leaves = Vec::new();

let TopoLeaves { std_4, std_b, ext_8, ext_1e } =
self.spec.board.cpu_topology.topo_leaves;

if std_4 {
topo_leaves.push(TopoKind::Std4);
}
if std_b {
topo_leaves.push(TopoKind::StdB);
}
if ext_8 {
topo_leaves.push(TopoKind::Ext8);
}
if ext_1e {
topo_leaves.push(TopoKind::Ext1E);
}
Ok(topo_leaves)
}

/// Initialize virtual CPUs by first setting their capabilities, inserting
/// them into the device map, and then, if a kstat sampler is provided,
/// tracking their kstats.
pub async fn initialize_cpus(&mut self) -> Result<(), MachineInitError> {
let hv_interface = self.machine.guest_hv_interface.as_ref();

for vcpu in self.machine.vcpus.iter() {
// Report that the guest is running on bhyve.
//
Expand All @@ -1596,33 +1644,29 @@ impl MachineInitializer<'_> {
requests to set hypervisor leaves",
);

// Instead of `TopoKind::supported`, we use an intentionally-reduced
// list of Intel-only leaves for the moment. This is because if we
// specialize leaves used by AMD (or just both vendors), we'll
// change the topology a guest sees.
let has_smt = self.spec.board.cpu_topology.has_smt;
// Instead of `TopoKind::supported`, the instance spec selects what
// leaves may get specialized topology data. This is to maintain the
// reported topology of VMs created either before CPU platforms or
// with platforms that inadvertently reported `board.cpus`-many
// single-core sockets.
//
// The initial CPU platform defined in Nexus (Omicron#8728) hews to
// the pre-specialization topology, which won't have leaf B at all.
// Before that is sent, though, we'll see the present-but-zero
// leaves from bhyve, which we would happily specialize into
// something reflecting the guest if requested here. Once
// Omicron#8728 lands and propolis-server receives explicit CPUID
// profiles, we can add AMD leaves here too.
let cpu_topo_leaves = [TopoKind::Std4];

let specialized = propolis::cpuid::Specializer::new()
.with_vcpu_count(
NonZeroU8::new(self.spec.board.cpus).unwrap(),
true,
)
.with_vcpuid(vcpu.id)
.with_cache_topo()
.clear_cpu_topo(TopoKind::iter())
.with_cpu_topo(cpu_topo_leaves.into_iter())
.execute(set)
.map_err(|e| {
MachineInitError::CpuidSpecializationFailed(vcpu.id, e)
})?;
// While it's probably OK to go change that, one would not expect
// their CPU sockets to unify across a reboot.
let cpu_topo_leaves = self.topo_cpuid_leaves()?;
info!(self.log, "computed cpu topo leaves {:?}", cpu_topo_leaves);

let specialized = propolis::cpuid::Specializer::new(
NonZeroU16::new(self.spec.board.cpus).unwrap(),
)
.with_cpu_topo(has_smt, cpu_topo_leaves.into_iter())
.with_vcpuid(vcpu.id)
.with_cache_topo()
.clear_cpu_topo(TopoKind::iter())
.execute(set)
.map_err(|e| {
MachineInitError::CpuidSpecializationFailed(vcpu.id, e)
})?;

info!(self.log, "setting CPUID for vCPU";
"vcpu" => vcpu.id,
Expand Down
14 changes: 12 additions & 2 deletions bin/propolis-server/src/lib/migrate/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -89,12 +89,14 @@
use serde::{Deserialize, Serialize};

use propolis_api_types_versions::v1::instance::ReplacementComponent;
use propolis_api_types_versions::{v1, v2, v3, v6};
use propolis_api_types_versions::{v1, v2, v3, v6, v7};

use std::collections::BTreeMap;

use crate::migrate::MigrateError;
use crate::spec::{api_spec_v1, api_spec_v2, api_spec_v3, api_spec_v6, Spec};
use crate::spec::{
api_spec_v1, api_spec_v2, api_spec_v3, api_spec_v6, api_spec_v7, Spec,
};

/// A wrapper for one of any supported `InstanceSpec` that describe a
/// to-be-migrated VM.
Expand All @@ -113,6 +115,7 @@ pub(crate) enum VersionedInstanceSpec {
V2(v2::instance_spec::InstanceSpec),
V3(v3::instance_spec::InstanceSpec),
V6(v6::instance_spec::InstanceSpec),
V7(v7::instance_spec::InstanceSpec),
}

impl VersionedInstanceSpec {
Expand Down Expand Up @@ -186,6 +189,13 @@ impl VersionedInstanceSpec {
.map_err(|e| MigrateError::PreambleParse(e.to_string()))?
.finish()
}
VersionedInstanceSpec::V7(mut source_spec) => {
api_spec_v7::amend(&mut source_spec, replacements)?;

api_spec_v7::v7_to_spec_builder(source_spec)
.map_err(|e| MigrateError::PreambleParse(e.to_string()))?
.finish()
}
};

Ok(amended_spec)
Expand Down
15 changes: 9 additions & 6 deletions bin/propolis-server/src/lib/spec/api_spec_v1.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,7 @@
use std::collections::BTreeMap;

use propolis_api_types::instance_spec::{
components::{
board::Board as InstanceSpecBoard,
devices::{BootSettings, SerialPort as SerialPortDesc},
},
components::devices::{BootSettings, SerialPort as SerialPortDesc},
SpecKey,
};
use propolis_api_types_versions::{
Expand Down Expand Up @@ -137,8 +134,14 @@ impl TryFrom<Spec> for v1::instance_spec::InstanceSpec {
spec.components.insert(key, val);
}

let board = InstanceSpecBoard {
cpus: board.cpus,
let cpus_u8: u8 = board.cpus.try_into().map_err(|_| {
LegacyApiSpecError::IncompatibleComponent(
"v1 Board cannot express >255 vCPUs".to_string(),
)
})?;

let board = v1::components::board::Board {
cpus: cpus_u8,
memory_mb: board.memory_mb,
chipset: board.chipset,
guest_hv_interface: board.guest_hv_interface,
Expand Down
3 changes: 2 additions & 1 deletion bin/propolis-server/src/lib/spec/api_spec_v2.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ pub(crate) fn amend(
#[cfg(test)]
mod test {
use propolis_api_types_versions::latest::components::board::{
Board, Chipset, Cpuid, GuestHypervisorInterface, I440Fx,
Board, Chipset, CpuTopology, Cpuid, GuestHypervisorInterface, I440Fx,
};
use propolis_api_types_versions::latest::{
self,
Expand Down Expand Up @@ -108,6 +108,7 @@ mod test {
entries: vec![],
vendor: CpuidVendor::Amd,
}),
cpu_topology: CpuTopology::single_socket_smt(),
},
components: Default::default(),
smbios: Some(smbios_input.clone()),
Expand Down
3 changes: 2 additions & 1 deletion bin/propolis-server/src/lib/spec/api_spec_v3.rs
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ pub(crate) fn amend(
#[cfg(test)]
mod test {
use propolis_api_types_versions::latest::components::board::{
Board, Chipset, Cpuid, GuestHypervisorInterface, I440Fx,
Board, Chipset, CpuTopology, Cpuid, GuestHypervisorInterface, I440Fx,
};
use propolis_api_types_versions::latest::{
self,
Expand All @@ -154,6 +154,7 @@ mod test {
entries: vec![],
vendor: CpuidVendor::Amd,
}),
cpu_topology: CpuTopology::single_socket_smt(),
},
components: Default::default(),
smbios: Some(SmbiosType1Input {
Expand Down
Loading