diff --git a/AGENTS.md b/AGENTS.md index 6ca9d38..acc17db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,7 +41,7 @@ claim Pi boot or physical power-cycle recovery is verified. Read `fds-os.wiki/Workstation.md`, `fds-os.wiki/Software-Format.md` and `fds-os.wiki/Developer-Workstation-Tooling-Plan.md` for this extension. Software builds and new PROGRAM cartridge creation run on generic Linux workstations. The public native -Clap tools are `fds-cartridge` and `fds-emulator`. Software images have GPT +Clap tools are `fds-cartridge`, `fds-emulator` and `fds-flash`. Software images have GPT metadata partition 1 plus m EROFS payload partitions containing installed Void package trees. Build source templates with xbps-src on the workstation; do not create new xz software bundles or extract programs at guest launch. Keep diff --git a/Makefile b/Makefile index decfb3c..884c35d 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,8 @@ help: 'make workstation-install Install the Arch fds-tools package with pacman' \ 'make workstation-binaries Build native binaries on generic Linux' \ 'make workstation-test Verify software builds, multi-partition images and file write/readback' \ + 'make flash-test Verify guided/unattended flashing with disposable files' \ + 'make flash-vm-test Verify block-device flashing and rejection paths in an isolated ARM VM' \ 'make emulator-test Boot FDS and test cartridge hotplug with the public emulator' \ 'make bootstrap Prepare Arch x86_64 host and Void build container' \ 'make packages Build and export every FDS base package' \ @@ -265,7 +267,18 @@ workstation-binaries: workstation-test: workstation-binaries cargo test --locked --offline --target $$(rustc -vV | sed -n 's/^host: //p') -p fds-common -p fds-burn -p fds-software -p fds-workstation + python3 tests/integration/workstation-flash.py --cli out/workstation/fds-flash python3 tests/integration/workstation-images.py --cli out/workstation/fds-cartridge --image-tool-runner tools/in-image-tools --xbps-tool-runner tools/in-void --xbps-bin "$(CURDIR)/.host/xbps/usr/bin" 2>&1 | tee out/logs/workstation-images.log emulator-test: workstation-binaries python3 tests/integration/workstation-emulator.py --cli out/workstation/fds-emulator --qemu-runner tools/in-void 2>&1 | tee out/logs/workstation-emulator.log + +.PHONY: flash-test flash-vm-test +flash-test: workstation-binaries + cargo test --locked --offline --target $$(rustc -vV | sed -n 's/^host: //p') -p fds-workstation flash:: + python3 tests/integration/workstation-flash.py --cli out/workstation/fds-flash + +flash-vm-test: flash-test + ./tools/cargo-build --locked --offline --release --target aarch64-unknown-linux-musl -p fds-workstation --bin fds-flash + ./tools/verify-elf target/aarch64-unknown-linux-musl/release/fds-flash aarch64 static + python3 tests/integration/workstation-flash-vm.py diff --git a/README.md b/README.md index c98529d..1e30ed6 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ authoritative manual: - [Project overview](http://gitea.home.arpa/felis/fds-os/wiki/Overview) - [Run on real hardware, including HDMI + USB wiring](http://gitea.home.arpa/felis/fds-os/wiki/Hardware-Setup) +- [Flash NVMe and cartridge disks](http://gitea.home.arpa/felis/fds-os/wiki/Flashing) - [Build the OS and start the emulator](http://gitea.home.arpa/felis/fds-os/wiki/Getting-Started) - [Build software cartridges and use workstation tools](http://gitea.home.arpa/felis/fds-os/wiki/Workstation) - [Use cartridges](http://gitea.home.arpa/felis/fds-os/wiki/Cartridges), [DATA](http://gitea.home.arpa/felis/fds-os/wiki/Data) and [the desktop](http://gitea.home.arpa/felis/fds-os/wiki/Desktop) diff --git a/fds-os.wiki b/fds-os.wiki index ddae6bf..ef88914 160000 --- a/fds-os.wiki +++ b/fds-os.wiki @@ -1 +1 @@ -Subproject commit ddae6bf6a6c0115e8e4deddd3dd7b313f32069e8 +Subproject commit ef88914454b46bf905cc2a37ce1e1f46af0539c6 diff --git a/packaging/arch/PKGBUILD b/packaging/arch/PKGBUILD index e4492ef..bb907a1 100644 --- a/packaging/arch/PKGBUILD +++ b/packaging/arch/PKGBUILD @@ -3,7 +3,7 @@ pkgname=fds-tools _fds_source=${FDS_SOURCE_DIR:?Run make workstation from the FDS checkout} pkgver=$("$_fds_source/tools/version") pkgrel=1 -pkgdesc='FDS/OS workstation cartridge builder and twelve-bay ARM emulator' +pkgdesc='FDS/OS workstation cartridge builder, disk flasher and twelve-bay ARM emulator' arch=('x86_64' 'aarch64') license=('MIT') depends=('glibc' 'gcc-libs' 'xz') @@ -27,7 +27,7 @@ build() { check() { local tool - for tool in fds-cartridge fds-emulator; do + for tool in fds-cartridge fds-emulator fds-flash; do [[ $("$_fds_source/out/workstation/$tool" --version) == "$tool $pkgver" ]] "$_fds_source/out/workstation/$tool" --help >/dev/null done @@ -35,7 +35,7 @@ check() { package() { local tool - for tool in fds-cartridge fds-emulator; do + for tool in fds-cartridge fds-emulator fds-flash; do install -Dm755 "$_fds_source/out/workstation/$tool" "$pkgdir/usr/bin/$tool" done install -Dm644 "$_fds_source/LICENSE" "$pkgdir/usr/share/licenses/fds-tools/LICENSE" diff --git a/rust/fds-workstation/Cargo.toml b/rust/fds-workstation/Cargo.toml index 70d25da..8ab952c 100644 --- a/rust/fds-workstation/Cargo.toml +++ b/rust/fds-workstation/Cargo.toml @@ -24,3 +24,7 @@ path = "src/main.rs" [[bin]] name = "fds-emulator" path = "src/emulator-main.rs" + +[[bin]] +name = "fds-flash" +path = "src/flash-main.rs" diff --git a/rust/fds-workstation/src/flash-main.rs b/rust/fds-workstation/src/flash-main.rs new file mode 100644 index 0000000..f2908e5 --- /dev/null +++ b/rust/fds-workstation/src/flash-main.rs @@ -0,0 +1,12 @@ +use clap::Parser; +use std::process::ExitCode; + +fn main() -> ExitCode { + match fds_workstation::flash::cli::run(fds_workstation::flash::cli::Cli::parse()) { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("fds-flash: {error}"); + ExitCode::from(2) + } + } +} diff --git a/rust/fds-workstation/src/flash/cli.rs b/rust/fds-workstation/src/flash/cli.rs new file mode 100644 index 0000000..6e47a3d --- /dev/null +++ b/rust/fds-workstation/src/flash/cli.rs @@ -0,0 +1,282 @@ +use super::{Prepared, device, prepare}; +use clap::{Parser, Subcommand}; +use fds_common::{Error, Result}; +use std::{ + io::{self, BufRead, IsTerminal, Write}, + path::{Path, PathBuf}, +}; + +fn digest(value: &str) -> std::result::Result { + if value.len() != 64 || !value.bytes().all(|c| c.is_ascii_hexdigit()) { + return Err("expected 64 hexadecimal characters".into()); + } + Ok(value.to_ascii_lowercase()) +} + +#[derive(Debug, Parser)] +#[command(name = "fds-flash", version = env!("FDS_BUILD_VERSION"), + about = "Flash complete FDS internal and cartridge disk images on Linux", + after_help = "With no flags, choose an image and disk interactively. Physical writes need root. Every write flushes and verifies readback; no partition or filesystem is expanded. Verify downloaded release signatures separately.", + args_conflicts_with_subcommands = true)] +pub struct Cli { + #[command(subcommand)] + command: Option, + /// Complete, uncompressed GPT image. Prompted for in interactive mode. + #[arg(long)] + image: Option, + /// Whole disk, preferably /dev/disk/by-id/...; never a partition. + #[arg(long)] + device: Option, + /// Do not prompt. Requires explicit image, disk, target ID and image SHA-256. + #[arg(long, requires_all = ["image", "device", "expect_target", "sha256"], conflicts_with = "dry_run")] + unattended: bool, + /// Exact target_id obtained from a reviewed dry run or disk listing. + #[arg(long, value_parser = digest, requires = "unattended")] + expect_target: Option, + /// Expected image SHA-256 (required for unattended writes). + #[arg(long, value_parser = digest)] + sha256: Option, + /// Validate and show the plan without opening the target for writing. + #[arg(long, requires_all = ["image", "device"])] + dry_run: bool, + /// Emit the plan/result as JSON on stdout; prompts and progress use stderr. + #[arg(long)] + json: bool, + /// Test only: use an existing disposable regular file as an ordinary user. + #[arg(long, requires = "device")] + file_target: bool, +} + +#[derive(Debug, Subcommand)] +enum Action { + /// List physical whole disks, identities and reasons they cannot be flashed. + List { + #[arg(long)] + json: bool, + }, +} + +fn prompt(text: &str) -> Result { + eprint!("{text}"); + io::stderr().flush()?; + let mut value = String::new(); + if io::stdin().lock().read_line(&mut value)? == 0 { + return Err(Error("Input closed; flash cancelled".into())); + } + if value.len() > 4096 { + return Err(Error("Input is too long".into())); + } + let value = value.trim().to_owned(); + if value.is_empty() { + return Err(Error("Empty input; flash cancelled".into())); + } + Ok(value) +} + +fn describe(target: &device::Target) -> String { + match target { + device::Target::Disk { + path, + bytes, + model, + serial, + sector_bytes, + .. + } => format!( + "{} | {:.2} GiB | model: {} | serial: {} | {}-byte sectors", + path.display(), + *bytes as f64 / 1024f64.powi(3), + if model.is_empty() { "unknown" } else { model }, + if serial.is_empty() { + "unavailable" + } else { + serial + }, + sector_bytes + ), + device::Target::File { path, bytes, .. } => format!( + "{} | {} bytes | DISPOSABLE TEST FILE", + path.display(), + bytes + ), + } +} + +fn choose_device() -> Result<(PathBuf, String)> { + let disks = device::list()?; + if disks.is_empty() { + return Err(Error("No physical disks found".into())); + } + eprintln!("Choose the destination by model, serial and capacity:"); + for (n, disk) in disks.iter().enumerate() { + eprintln!( + " {}. {}{}", + n + 1, + describe(&disk.target), + disk.blocked + .as_ref() + .map(|r| format!(" | BLOCKED: {r}")) + .unwrap_or_default() + ); + } + let choice: usize = prompt("Disk number (no default): ")? + .parse() + .map_err(|_| Error("Invalid disk number".into()))?; + let selected = choice + .checked_sub(1) + .and_then(|n| disks.get(n)) + .ok_or_else(|| Error("Disk number is out of range".into()))?; + if let Some(reason) = &selected.blocked { + return Err(Error(format!("Selected disk is blocked: {reason}"))); + } + Ok(( + selected.target.path().to_owned(), + selected.target_id.clone(), + )) +} + +fn show(prepared: &Prepared) { + let plan = &prepared.plan; + eprintln!( + "Image: {}\nKind: {}\nImage bytes: {}\nSHA-256: {}\nTarget: {}\nTarget ID: {}", + plan.image_path.display(), + plan.image.kind, + plan.image.bytes, + plan.sha256, + describe(&plan.target), + plan.target_id + ); + for partition in &plan.image.partitions { + eprintln!( + " Partition {}: {} ({} bytes)", + partition.number, partition.name, partition.bytes + ); + } + eprintln!( + "This replaces the selected disk's partition table and image contents. Backup GPT relocation and readback are automatic; filesystems keep their image sizes." + ); +} + +pub fn run(cli: Cli) -> Result<()> { + if let Some(Action::List { json }) = cli.command { + let disks = device::list()?; + if json { + println!( + "{}", + serde_json::to_string_pretty(&disks).map_err(|e| Error(e.to_string()))? + ); + } else { + for disk in disks { + println!( + "{}\n target_id: {}\n {}", + describe(&disk.target), + disk.target_id, + disk.blocked + .map(|r| format!("BLOCKED: {r}")) + .unwrap_or_else(|| "Available for explicit selection".into()) + ); + } + } + return Ok(()); + } + if !cli.unattended && !cli.dry_run && !io::stdin().is_terminal() { + return Err(Error("Interactive flashing requires a terminal; use --dry-run or explicit --unattended options".into())); + } + if !cli.dry_run && !cli.file_target && unsafe { libc::geteuid() } != 0 { + return Err(Error( + "Physical disk writes require root; run fds-flash with sudo".into(), + )); + } + let image = match cli.image { + Some(path) => path, + None => { + eprintln!( + "Use a complete FDS .img file, such as out/fds-internal.img or out/fds-system-cli.img." + ); + PathBuf::from(prompt("Image path: ")?) + } + }; + let (device, expected_target) = match cli.device { + Some(path) => (path, cli.expect_target), + None => { + let (path, identity) = choose_device()?; + (path, Some(identity)) + } + }; + eprintln!("Inspecting image and destination..."); + let prepared = prepare( + Path::new(&image), + &device, + cli.file_target, + cli.sha256.as_deref(), + )?; + if expected_target + .as_ref() + .is_some_and(|expected| *expected != prepared.plan.target_id) + { + return Err(Error( + "Target identity changed or does not match the expected target; no bytes written" + .into(), + )); + } + show(&prepared); + if !cli.dry_run { + if !cli.unattended + && prompt(&format!("Type {} to proceed: ", prepared.plan.confirmation))? + != prepared.plan.confirmation + { + return Err(Error( + "Confirmation did not match; flash cancelled without writing".into(), + )); + } + prepared.write()?; + } + let status = if cli.dry_run { "dry_run" } else { "verified" }; + if cli.json { + println!( + "{}", + serde_json::to_string_pretty( + &serde_json::json!({"format": 1, "status": status, "plan": prepared.plan}) + ) + .map_err(|e| Error(e.to_string()))? + ); + } else if cli.dry_run { + println!("DRY RUN: target untouched"); + } else { + println!( + "VERIFIED: image written, flushed and read back; backup GPT is at the end of the disk" + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::CommandFactory; + #[test] + fn unattended_requires_explicit_image_and_target_bindings() { + Cli::command().debug_assert(); + assert!(Cli::try_parse_from(["fds-flash"]).is_ok()); + assert!(Cli::try_parse_from(["fds-flash", "list", "--json"]).is_ok()); + assert!( + Cli::try_parse_from(["fds-flash", "--unattended", "--image", "x", "--device", "y"]) + .is_err() + ); + assert!(Cli::try_parse_from(["fds-flash", "--dry-run"]).is_err()); + let args = [ + "fds-flash", + "--unattended", + "--image", + "x", + "--device", + "y", + "--expect-target", + &"a".repeat(64), + "--sha256", + &"b".repeat(64), + ]; + assert!(Cli::try_parse_from(args).is_ok()); + assert!(digest("wrong").is_err()); + } +} diff --git a/rust/fds-workstation/src/flash/device.rs b/rust/fds-workstation/src/flash/device.rs new file mode 100644 index 0000000..89b7929 --- /dev/null +++ b/rust/fds-workstation/src/flash/device.rs @@ -0,0 +1,535 @@ +//! Workstation destinations: physical whole disks or explicit disposable files. +use fds_burn::image; +use fds_common::{Error, Result, read_text}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeSet, + fs::{self, File, OpenOptions}, + os::{ + fd::AsRawFd, + unix::fs::{FileTypeExt, MetadataExt, OpenOptionsExt}, + }, + path::{Path, PathBuf}, +}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Target { + Disk { + path: PathBuf, + sysfs: PathBuf, + major: u32, + minor: u32, + diskseq: u64, + boot_id: String, + bytes: u64, + sector_bytes: u32, + model: String, + serial: String, + }, + File { + path: PathBuf, + device: u64, + inode: u64, + bytes: u64, + mtime: i64, + mtime_ns: i64, + sha256: String, + }, +} + +fn number(path: &Path) -> Result { + read_text(path, 128)? + .trim() + .parse() + .map_err(|_| Error(format!("Invalid kernel number: {}", path.display()))) +} +fn dev(path: &Path) -> Result<(u32, u32)> { + let value = read_text(&path.join("dev"), 128)?; + let (major, minor) = value + .trim() + .split_once(':') + .ok_or_else(|| Error("Invalid block device number".into()))?; + Ok(( + major + .parse() + .map_err(|_| Error("Invalid device major".into()))?, + minor + .parse() + .map_err(|_| Error("Invalid device minor".into()))?, + )) +} +fn text(path: &Path) -> String { + read_text(path, 4096) + .unwrap_or_default() + .trim() + .chars() + .filter(|c| !c.is_control()) + .take(128) + .collect() +} +fn inspect_disk(path: &Path, sysfs: &Path, major: u32, minor: u32) -> Result { + if sysfs.join("partition").exists() { + return Err(Error("Select a whole disk, not a partition".into())); + } + if sysfs.starts_with("/sys/devices/virtual") { + return Err(Error( + "Virtual, loop, RAID and device-mapper destinations are not physical disks".into(), + )); + } + if dev(sysfs)? != (major, minor) { + return Err(Error("Block device identity mismatch".into())); + } + let bytes = number(&sysfs.join("size"))? + .checked_mul(512) + .ok_or_else(|| Error("Disk size overflow".into()))?; + let sector_bytes = number(&sysfs.join("queue/logical_block_size"))? + .try_into() + .map_err(|_| Error("Invalid logical sector size".into()))?; + let mut model = String::new(); + let mut serial = String::new(); + for ancestor in sysfs + .ancestors() + .take_while(|p| *p != Path::new("/sys/devices")) + { + if model.is_empty() { + model = text(&ancestor.join("device/model")); + } + if model.is_empty() { + model = text(&ancestor.join("model")); + } + if serial.is_empty() { + serial = text(&ancestor.join("device/serial")); + } + if serial.is_empty() { + serial = text(&ancestor.join("serial")); + } + } + Ok(Target::Disk { + path: path.into(), + sysfs: sysfs.into(), + major, + minor, + diskseq: number(&sysfs.join("diskseq"))?, + boot_id: read_text(Path::new("/proc/sys/kernel/random/boot_id"), 128)? + .trim() + .into(), + bytes, + sector_bytes, + model, + serial, + }) +} + +fn inspect_file(path: &Path, file: &File) -> Result { + let before = file.metadata()?; + if !before.is_file() { + return Err(Error( + "--file-target requires an existing disposable regular file".into(), + )); + } + let sha256 = image::digest(file, before.len(), |_| Ok(()))?; + let after = file.metadata()?; + if before.len() != after.len() + || before.mtime() != after.mtime() + || before.mtime_nsec() != after.mtime_nsec() + { + return Err(Error("Disposable target changed during inspection".into())); + } + Ok(Target::File { + path: path.into(), + device: before.dev(), + inode: before.ino(), + bytes: before.len(), + mtime: before.mtime(), + mtime_ns: before.mtime_nsec(), + sha256, + }) +} + +pub fn inspect(path: &Path, file_target: bool) -> Result { + let path = path.canonicalize()?; + if file_target { + if unsafe { libc::geteuid() } == 0 { + return Err(Error( + "Disposable file tests must run as an ordinary user".into(), + )); + } + return inspect_file(&path, &crate::cartridge::open_image(&path)?); + } + let metadata = fs::metadata(&path)?; + if !metadata.file_type().is_block_device() { + return Err(Error( + "Destination must be a whole block device; use --file-target only for disposable files" + .into(), + )); + } + let major = libc::major(metadata.rdev()); + let minor = libc::minor(metadata.rdev()); + let sysfs = fs::canonicalize(format!("/sys/dev/block/{major}:{minor}"))?; + inspect_disk(&path, &sysfs, major, minor) +} + +impl Target { + pub fn path(&self) -> &Path { + match self { + Self::Disk { path, .. } | Self::File { path, .. } => path, + } + } + pub fn bytes(&self) -> u64 { + match self { + Self::Disk { bytes, .. } | Self::File { bytes, .. } => *bytes, + } + } + pub fn id(&self) -> Result { + Ok(image::hex(&Sha256::digest( + serde_json::to_vec(self).map_err(|e| Error(e.to_string()))?, + ))) + } + pub fn is_file(&self) -> bool { + matches!(self, Self::File { .. }) + } + pub fn present(&self) -> Result<()> { + if let Self::Disk { + sysfs, + major, + minor, + diskseq, + bytes, + .. + } = self + { + if fs::canonicalize(format!("/sys/dev/block/{major}:{minor}"))? != *sysfs + || number(&sysfs.join("diskseq"))? != *diskseq + || number(&sysfs.join("size"))?.checked_mul(512) != Some(*bytes) + { + return Err(Error( + "Target insertion or capacity changed; flash aborted".into(), + )); + } + } + Ok(()) + } + pub fn protect(&self) -> Result<()> { + if let Self::Disk { + sysfs, + major, + minor, + sector_bytes, + .. + } = self + { + self.present()?; + protect( + sysfs, + Path::new("/sys"), + Path::new("/proc"), + (*major, *minor), + *sector_bytes, + )?; + } + Ok(()) + } + pub fn open(&self) -> Result { + if !self.is_file() && unsafe { libc::geteuid() } != 0 { + return Err(Error( + "Physical disk writes require root; run fds-flash with sudo".into(), + )); + } + if inspect(self.path(), self.is_file())? != *self { + return Err(Error( + "Target identity changed after preview; no bytes written".into(), + )); + } + self.protect()?; + let mut flags = libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK; + if !self.is_file() { + flags |= libc::O_EXCL; + } + let file = OpenOptions::new() + .read(true) + .write(true) + .custom_flags(flags) + .open(self.path()) + .map_err(|e| { + Error(format!( + "Cannot exclusively open {}: {e}. Physical writes need root and an unused disk", + self.path().display() + )) + })?; + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } < 0 { + return Err(Error("Target is locked by another process".into())); + } + match self { + Self::File { .. } => { + if inspect_file(self.path(), &file)? != *self { + return Err(Error("Disposable target changed; no bytes written".into())); + } + } + Self::Disk { + major, + minor, + diskseq, + bytes, + sector_bytes, + .. + } => { + let metadata = file.metadata()?; + if !metadata.file_type().is_block_device() + || libc::major(metadata.rdev()) != *major + || libc::minor(metadata.rdev()) != *minor + { + return Err(Error("Opened disk identity mismatch".into())); + } + let mut actual_bytes = 0u64; + let mut actual_sector = 0u32; + let mut actual_sequence = 0u64; + let mut readonly = 0u32; + for (request, pointer) in [ + ( + 0x80081272u64, + (&mut actual_bytes as *mut u64).cast::(), + ), + (0x1268, (&mut actual_sector as *mut u32).cast()), + (0x80081280, (&mut actual_sequence as *mut u64).cast()), + (0x125e, (&mut readonly as *mut u32).cast()), + ] { + if unsafe { libc::ioctl(file.as_raw_fd(), request as libc::Ioctl, pointer) } < 0 + { + return Err(std::io::Error::last_os_error().into()); + } + } + if actual_bytes != *bytes + || actual_sector != *sector_bytes + || actual_sequence != *diskseq + || readonly != 0 + { + return Err(Error( + "Opened disk capacity, sector size, insertion or write protection changed" + .into(), + )); + } + } + } + self.protect()?; + Ok(file) + } +} + +fn protect(disk: &Path, sys: &Path, proc: &Path, identity: (u32, u32), sector: u32) -> Result<()> { + if sector != 512 { + return Err(Error("FDS images require 512-byte logical sectors".into())); + } + if number(&disk.join("ro"))? != 0 { + return Err(Error("Disk is read-only".into())); + } + let mut devices = BTreeSet::new(); + for entry in fs::read_dir(sys.join("class/block"))? { + let path = entry?.path().canonicalize()?; + if path != disk && !path.starts_with(disk) { + continue; + } + devices.insert(dev(&path)?); + if fs::read_dir(path.join("holders"))? + .next() + .transpose()? + .is_some() + { + return Err(Error( + "Disk or partition has an active kernel holder (LVM, RAID or encryption)".into(), + )); + } + } + if !devices.contains(&identity) { + return Err(Error("Disk vanished during protection checks".into())); + } + for line in read_text(&proc.join("self/mountinfo"), 4 * 1024 * 1024)?.lines() { + let fields: Vec<_> = line.split_whitespace().collect(); + if fields.len() < 6 { + return Err(Error("Malformed mount table".into())); + } + let number = fields[2] + .split_once(':') + .ok_or_else(|| Error("Invalid mount device".into()))?; + let id = ( + number.0.parse::().map_err(|e| Error(e.to_string()))?, + number.1.parse::().map_err(|e| Error(e.to_string()))?, + ); + if devices.contains(&id) { + return Err(Error(format!( + "Target is mounted at {}; unmount it explicitly before flashing", + fields[4] + ))); + } + } + for line in read_text(&proc.join("swaps"), 1024 * 1024)?.lines().skip(1) { + let path = line + .split_whitespace() + .next() + .ok_or_else(|| Error("Invalid swap table".into()))?; + let metadata = fs::metadata(unescape(path)?)?; + let id = if metadata.file_type().is_block_device() { + metadata.rdev() + } else { + metadata.dev() + }; + if devices.contains(&(libc::major(id), libc::minor(id))) { + return Err(Error( + "Target contains active swap; disable it explicitly before flashing".into(), + )); + } + } + Ok(()) +} + +fn unescape(value: &str) -> Result { + use std::os::unix::ffi::OsStringExt; + let bytes = value.as_bytes(); + let mut out = Vec::new(); + let mut n = 0; + while n < bytes.len() { + if bytes[n] == b'\\' { + if n + 3 >= bytes.len() + || !bytes[n + 1..n + 4] + .iter() + .all(|c| (b'0'..=b'7').contains(c)) + { + return Err(Error("Invalid escaped swap path".into())); + } + let number = ((bytes[n + 1] - b'0') as u16 * 64) + + ((bytes[n + 2] - b'0') as u16 * 8) + + (bytes[n + 3] - b'0') as u16; + out.push(u8::try_from(number).map_err(|_| Error("Invalid swap path byte".into()))?); + n += 4; + } else { + out.push(bytes[n]); + n += 1; + } + } + Ok(std::ffi::OsString::from_vec(out).into()) +} + +#[derive(Serialize)] +pub struct Candidate { + pub target: Target, + pub target_id: String, + pub blocked: Option, +} +pub fn list() -> Result> { + let mut result = Vec::new(); + for entry in fs::read_dir("/sys/class/block")? { + let entry = entry?; + let sysfs = match entry.path().canonicalize() { + Ok(path) => path, + Err(_) if !entry.path().exists() => continue, + Err(error) => return Err(error.into()), + }; + if sysfs.join("partition").exists() || sysfs.starts_with("/sys/devices/virtual") { + continue; + } + let path = Path::new("/dev").join(entry.file_name()); + let target = match inspect(&path, false) { + Ok(target) => target, + Err(error) => { + // Enumeration is a snapshot: sysfs and /dev appear/disappear + // independently during hotplug. A selected target is always + // inspected afresh and failures there still prohibit writing. + eprintln!("Skipping unavailable disk {}: {error}", path.display()); + continue; + } + }; + result.push(Candidate { + target_id: target.id()?, + blocked: target.protect().err().map(|e| e.to_string()), + target, + }); + } + result.sort_by(|a, b| a.target.path().cmp(b.target.path())); + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::symlink; + #[test] + fn mount_holder_swap_and_sector_protections() { + let root = std::env::temp_dir().join(format!( + "fds-flash-protection-{}", + image::hex(&image::random_id().unwrap()) + )); + let sys = root.join("sys"); + let proc = root.join("proc"); + let disk = sys.join("devices/block/test"); + for directory in [ + disk.join("holders"), + disk.join("test1/holders"), + sys.join("class/block"), + proc.join("self"), + ] { + fs::create_dir_all(directory).unwrap(); + } + let swap = root.join("swap file"); + fs::write(&swap, b"fixture").unwrap(); + let host_dev = swap.metadata().unwrap().dev(); + let identity = (libc::major(host_dev), libc::minor(host_dev)); + fs::write(disk.join("dev"), format!("{}:{}\n", identity.0, identity.1)).unwrap(); + fs::write(disk.join("test1/dev"), "254:241\n").unwrap(); + fs::write(disk.join("ro"), "0\n").unwrap(); + symlink(&disk, sys.join("class/block/test")).unwrap(); + symlink(disk.join("test1"), sys.join("class/block/test1")).unwrap(); + fs::write(proc.join("self/mountinfo"), "").unwrap(); + fs::write(proc.join("swaps"), "Filename Type Size Used Priority\n").unwrap(); + protect(&disk, &sys, &proc, identity, 512).unwrap(); + fs::write( + proc.join("self/mountinfo"), + "1 0 254:241 / /data rw - ext4 /dev/test1 rw\n", + ) + .unwrap(); + assert!( + protect(&disk, &sys, &proc, identity, 512) + .unwrap_err() + .to_string() + .contains("mounted") + ); + fs::write(proc.join("self/mountinfo"), "").unwrap(); + fs::write(disk.join("test1/holders/dm-0"), "").unwrap(); + assert!( + protect(&disk, &sys, &proc, identity, 512) + .unwrap_err() + .to_string() + .contains("holder") + ); + fs::remove_file(disk.join("test1/holders/dm-0")).unwrap(); + fs::write( + proc.join("swaps"), + format!( + "Filename Type Size Used Priority\n{} file 1 0 -1\n", + swap.display().to_string().replace(' ', "\\040") + ), + ) + .unwrap(); + assert!( + protect(&disk, &sys, &proc, identity, 512) + .unwrap_err() + .to_string() + .contains("swap") + ); + fs::write(proc.join("swaps"), "Filename Type Size Used Priority\n").unwrap(); + assert!( + protect(&disk, &sys, &proc, identity, 4096) + .unwrap_err() + .to_string() + .contains("512-byte") + ); + fs::write(disk.join("ro"), "1\n").unwrap(); + assert!( + protect(&disk, &sys, &proc, identity, 512) + .unwrap_err() + .to_string() + .contains("read-only") + ); + fs::remove_dir_all(root).unwrap(); + } +} diff --git a/rust/fds-workstation/src/flash/image.rs b/rust/fds-workstation/src/flash/image.rs new file mode 100644 index 0000000..213b5c3 --- /dev/null +++ b/rust/fds-workstation/src/flash/image.rs @@ -0,0 +1,407 @@ +//! Complete FDS disk images, including the internal FAT/EROFS/ext4 disk. +//! Cartridge validation remains shared with the cartridge writer. Internal +//! installation is deliberately separate from its reserved-partition policy. +use fds_burn::image::{ + self, LINUX_TYPE, Partition, TABLE_BYTES, crc32, put32, put64, u32le, u64le, +}; +use fds_common::{Error, Result}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::{ + fs::File, + os::{ + fd::AsRawFd, + unix::fs::{FileExt, FileTypeExt}, + }, +}; + +const EFI_TYPE: [u8; 16] = [ + 0x28, 0x73, 0x2a, 0xc1, 0x1f, 0xf8, 0xd2, 0x11, 0xba, 0x4b, 0, 0xa0, 0xc9, 0x3e, 0xc9, 0x3b, +]; +fn bad(message: &str) -> Error { + Error(format!("Invalid FDS disk image: {message}")) +} + +#[derive(Debug, PartialEq, Eq, Serialize)] +pub struct Geometry { + pub bytes: u64, + pub kind: String, + pub disk_uuid: String, + pub partitions: Vec, + #[serde(skip)] + head: Vec, + #[serde(skip)] + tail: Vec, +} + +pub fn inspect(file: &File, bytes: u64) -> Result { + if bytes % 512 != 0 || bytes < (2048 + 2048 + 33) * 512 { + return Err(bad( + "use a complete, sector-aligned GPT .img, not a compressed archive or partition payload", + )); + } + let mut head = vec![0; 1024 + TABLE_BYTES]; + let mut tail = vec![0; 512 + TABLE_BYTES]; + file.read_exact_at(&mut head, 0)?; + let tail_offset = bytes - tail.len() as u64; + file.read_exact_at(&mut tail, tail_offset)?; + if let Ok(cartridge) = image::inspect(file, bytes) { + return Ok(Geometry { + bytes, + kind: format!("{:?}", cartridge.class).to_lowercase(), + disk_uuid: cartridge.disk_uuid, + partitions: cartridge.partitions, + head, + tail, + }); + } + // The only additional accepted layout is the complete internal disk. + // Both GPT copies, CRCs, bounds, types, names and filesystem signatures + // must agree before a destination can be opened for writing. + let sectors = bytes / 512; + if head[510..512] != [0x55, 0xaa] + || head[446] != 0 + || head[450] != 0xee + || u32le(&head, 454) != 1 + || u32le(&head, 458) != (sectors - 1).min(u32::MAX as u64) as u32 + || head[462..510].iter().any(|b| *b != 0) + { + return Err(bad( + "missing protective MBR; expected a complete FDS GPT image", + )); + } + let primary = &head[512..1024]; + let backup = &tail[TABLE_BYTES..]; + for (header, current, alternate, table) in [ + (primary, 1, sectors - 1, 2), + (backup, sectors - 1, 1, sectors - 33), + ] { + let mut checked = header.to_vec(); + put32(&mut checked, 16, 0); + if &header[..8] != b"EFI PART" + || u32le(header, 8) != 0x10000 + || u32le(header, 12) != 92 + || u32le(header, 20) != 0 + || header[92..].iter().any(|b| *b != 0) + || crc32(&checked[..92]) != u32le(header, 16) + || u64le(header, 24) != current + || u64le(header, 32) != alternate + || u64le(header, 40) != 34 + || u64le(header, 48) != sectors - 34 + || u64le(header, 72) != table + || u32le(header, 80) != 128 + || u32le(header, 84) != 128 + || header[56..72].iter().all(|b| *b == 0) + { + return Err(bad("GPT geometry or header checksum is invalid")); + } + } + let table = &head[1024..]; + if primary[40..72] != backup[40..72] + || primary[80..92] != backup[80..92] + || table != &tail[..TABLE_BYTES] + || crc32(table) != u32le(primary, 88) + || table[3 * 128..].iter().any(|b| *b != 0) + { + return Err(bad( + "internal disk GPT copies disagree or do not contain exactly three partitions", + )); + } + let mut partitions = Vec::new(); + let mut ids = std::collections::BTreeSet::new(); + let mut next = 2048; + for (index, name) in ["FDS_BOOT", "FDS_RECOVERY", "FDS_INTERNAL"] + .iter() + .enumerate() + { + let entry = &table[index * 128..(index + 1) * 128]; + let first = u64le(entry, 32); + let last = u64le(entry, 40); + let mut encoded = [0; 72]; + for (n, unit) in name.encode_utf16().enumerate() { + encoded[n * 2..n * 2 + 2].copy_from_slice(&unit.to_le_bytes()); + } + if entry[..16] != if index == 0 { EFI_TYPE } else { LINUX_TYPE } + || entry[16..32].iter().all(|b| *b == 0) + || !ids.insert(entry[16..32].to_vec()) + || u64le(entry, 48) != 0 + || entry[56..] != encoded + || first < next + || first % 2048 != 0 + || (index == 0 && first != 2048) + || first > sectors - 34 + || last < first + || last > sectors - 34 + || (last - first + 1) % 2048 != 0 + { + return Err(bad( + "unsupported internal partition type, name, UUID or bounds", + )); + } + let start = first * 512; + let mut signature = [0; 2048]; + file.read_exact_at(&mut signature, start)?; + let valid = match index { + 0 => { + signature[510..512] == [0x55, 0xaa] + && &signature[82..90] == b"FAT32 " + && signature[11..13] == [0, 2] + } + 1 => signature[1024..1028] == [0xe2, 0xe1, 0xf5, 0xe0], + _ => signature[1080..1082] == [0x53, 0xef], + }; + if !valid { + return Err(bad("internal disk requires FAT32, EROFS and ext4 in order")); + } + partitions.push(Partition { + number: (index + 1) as u8, + name: (*name).into(), + start, + bytes: (last - first + 1) * 512, + }); + next = last + 1; + } + Ok(Geometry { + bytes, + kind: "internal".into(), + disk_uuid: image::hex(&primary[56..72]), + partitions, + head, + tail, + }) +} + +fn overlay(buffer: &mut [u8], offset: u64, patch: &[u8], position: u64) { + let begin = offset.max(position); + let end = (offset + buffer.len() as u64).min(position + patch.len() as u64); + if begin < end { + buffer[(begin - offset) as usize..(end - offset) as usize] + .copy_from_slice(&patch[(begin - position) as usize..(end - position) as usize]); + } +} + +/// Verify all written bytes, including the relocated backup GPT. Unallocated +/// space outside the image is not erased, and no filesystem is expanded. +pub fn transfer( + source: &File, + target: &File, + geometry: &Geometry, + sha256: &str, + target_bytes: u64, + mut progress: impl FnMut(&str, u64) -> Result<()>, +) -> Result<()> { + if target_bytes < geometry.bytes || target_bytes % 512 != 0 { + return Err(bad("target is too small or not sector aligned")); + } + if source.metadata()?.len() != geometry.bytes + || image::digest(source, geometry.bytes, |n| progress("checking", n))? != sha256 + || inspect(source, geometry.bytes)? != *geometry + { + return Err(Error( + "Image changed after preview; target untouched".into(), + )); + } + let mut head = geometry.head.clone(); + let mut tail = geometry.tail.clone(); + let sectors = target_bytes / 512; + put32(&mut head, 458, (sectors - 1).min(u32::MAX as u64) as u32); + for (header, lba, other, entries) in [ + (&mut head[512..1024], 1, sectors - 1, 2), + (&mut tail[TABLE_BYTES..], sectors - 1, 1, sectors - 33), + ] { + put64(header, 24, lba); + put64(header, 32, other); + put64(header, 48, sectors - 34); + put64(header, 72, entries); + put32(header, 16, 0); + let crc = crc32(&header[..92]); + put32(header, 16, crc); + } + let old_tail = vec![0; tail.len()]; + let patch = |buffer: &mut [u8], offset| { + overlay(buffer, offset, &head, 0); + if target_bytes != geometry.bytes { + overlay( + buffer, + offset, + &old_tail, + geometry.bytes - old_tail.len() as u64, + ); + } + overlay(buffer, offset, &tail, target_bytes - tail.len() as u64); + }; + let mut original = vec![0; 1024 * 1024]; + let mut written = vec![0; original.len()]; + for phase in ["writing", "verifying"] { + progress(phase, 0)?; + let mut hash = Sha256::new(); + let mut offset = 0; + while offset < geometry.bytes { + let n = original.len().min((geometry.bytes - offset) as usize); + source.read_exact_at(&mut original[..n], offset)?; + hash.update(&original[..n]); + patch(&mut original[..n], offset); + if phase == "writing" { + target.write_all_at(&original[..n], offset)?; + } else { + target.read_exact_at(&mut written[..n], offset)?; + if written[..n] != original[..n] { + return Err(Error("Disk readback mismatch; flash failed".into())); + } + } + offset += n as u64; + if offset % (64 * 1024 * 1024) == 0 || offset == geometry.bytes { + progress(phase, offset)?; + } + } + if image::hex(&hash.finalize()) != sha256 || source.metadata()?.len() != geometry.bytes { + return Err(Error( + "Source changed during transfer; target is incomplete".into(), + )); + } + if phase == "writing" { + target.write_all_at(&tail, target_bytes - tail.len() as u64)?; + target.sync_all()?; + if target.metadata()?.file_type().is_block_device() { + if unsafe { libc::ioctl(target.as_raw_fd(), 0x1261 as libc::Ioctl) } < 0 { + return Err(std::io::Error::last_os_error().into()); + } + } else { + let rc = unsafe { + libc::posix_fadvise(target.as_raw_fd(), 0, 0, libc::POSIX_FADV_DONTNEED) + }; + if rc != 0 { + return Err(std::io::Error::from_raw_os_error(rc).into()); + } + } + } + } + let mut actual_tail = vec![0; tail.len()]; + target.read_exact_at(&mut actual_tail, target_bytes - tail.len() as u64)?; + let observed = inspect(target, target_bytes)?; + if actual_tail != tail + || observed.partitions != geometry.partitions + || observed.disk_uuid != geometry.disk_uuid + || observed.kind != geometry.kind + { + return Err(Error( + "Written GPT or backup metadata failed verification".into(), + )); + } + target.sync_all()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use fds_common::manifest::Class; + use std::os::fd::FromRawFd; + + fn memory() -> File { + let fd = unsafe { libc::memfd_create(c"fds-flash-test".as_ptr(), libc::MFD_CLOEXEC) }; + assert!(fd >= 0); + unsafe { File::from_raw_fd(fd) } + } + fn fixture() -> (File, Geometry, String) { + let source = memory(); + let layout = + image::Layout::new(Class::System, 1024 * 1024, None, [1; 16], [2; 16]).unwrap(); + layout.write(&source).unwrap(); + source + .write_all_at(&[0xe2, 0xe1, 0xf5, 0xe0], 1024 * 1024 + 1024) + .unwrap(); + let geometry = inspect(&source, layout.bytes).unwrap(); + let hash = image::digest(&source, layout.bytes, |_| Ok(())).unwrap(); + (source, geometry, hash) + } + #[test] + fn exact_larger_and_overlapping_backup_locations() { + let (source, geometry, hash) = fixture(); + for extra in [0, 512, 4 * 1024 * 1024] { + let target = memory(); + let size = geometry.bytes + extra; + target.set_len(size).unwrap(); + transfer(&source, &target, &geometry, &hash, size, |_, _| Ok(())).unwrap(); + assert_eq!( + inspect(&target, size).unwrap().partitions, + geometry.partitions + ); + } + assert_eq!( + image::digest(&source, geometry.bytes, |_| Ok(())).unwrap(), + hash + ); + } + #[test] + fn changed_source_is_rejected_before_any_write() { + let (source, geometry, hash) = fixture(); + let target = memory(); + target.set_len(geometry.bytes).unwrap(); + let before = image::digest(&target, geometry.bytes, |_| Ok(())).unwrap(); + source.write_all_at(b"changed", 1024 * 1024 + 8192).unwrap(); + assert!( + transfer( + &source, + &target, + &geometry, + &hash, + geometry.bytes, + |_, _| Ok(()) + ) + .is_err() + ); + assert_eq!( + image::digest(&target, geometry.bytes, |_| Ok(())).unwrap(), + before + ); + } + #[test] + fn readback_corruption_and_io_failure_never_succeed() { + let (source, geometry, hash) = fixture(); + let target = memory(); + target.set_len(geometry.bytes).unwrap(); + let failure = transfer( + &source, + &target, + &geometry, + &hash, + geometry.bytes, + |phase, n| { + if phase == "verifying" && n == 0 { + target.write_all_at(b"corrupt", 1024 * 1024 + 8192)?; + } + Ok(()) + }, + ) + .unwrap_err(); + assert!(failure.to_string().contains("readback mismatch")); + let readonly = File::open(format!("/proc/self/fd/{}", target.as_raw_fd())).unwrap(); + assert!( + transfer( + &source, + &readonly, + &geometry, + &hash, + geometry.bytes, + |_, _| Ok(()) + ) + .is_err() + ); + let interrupted = transfer( + &source, + &target, + &geometry, + &hash, + geometry.bytes, + |phase, n| { + if phase == "writing" && n > 0 { + return Err(Error("Target removed".into())); + } + Ok(()) + }, + ) + .unwrap_err(); + assert!(interrupted.to_string().contains("removed")); + } +} diff --git a/rust/fds-workstation/src/flash/mod.rs b/rust/fds-workstation/src/flash/mod.rs new file mode 100644 index 0000000..4b1461e --- /dev/null +++ b/rust/fds-workstation/src/flash/mod.rs @@ -0,0 +1,103 @@ +//! Guided and unattended installation of complete disk images on Linux. +pub mod cli; +pub mod device; +mod image; + +use fds_common::{Error, Result}; +use serde::Serialize; +use std::{ + fs::File, + os::{fd::AsRawFd, unix::fs::MetadataExt}, + path::{Path, PathBuf}, +}; + +#[derive(Serialize)] +pub struct Plan { + pub image_path: PathBuf, + pub image: image::Geometry, + pub sha256: String, + pub target: device::Target, + pub target_id: String, + pub confirmation: String, +} +pub struct Prepared { + pub plan: Plan, + source: File, +} + +pub fn prepare( + image_path: &Path, + device_path: &Path, + file_target: bool, + expected_sha256: Option<&str>, +) -> Result { + let image_path = image_path.canonicalize()?; + let source = crate::cartridge::open_image(&image_path)?; + let geometry = image::inspect(&source, source.metadata()?.len())?; + let sha256 = fds_burn::image::digest(&source, geometry.bytes, |_| Ok(()))?; + if expected_sha256.is_some_and(|expected| expected != sha256) { + return Err(Error( + "Image SHA-256 does not match --sha256; target untouched".into(), + )); + } + if image::inspect(&source, source.metadata()?.len())? != geometry { + return Err(Error("Image geometry changed during inspection".into())); + } + let target = device::inspect(device_path, file_target)?; + let metadata = std::fs::metadata(target.path())?; + if source.metadata()?.dev() == metadata.dev() && source.metadata()?.ino() == metadata.ino() { + return Err(Error("Image and target refer to the same file".into())); + } + target.protect()?; + if target.bytes() < geometry.bytes || target.bytes() % 512 != 0 { + return Err(Error( + "Destination is smaller than the image or is not sector aligned".into(), + )); + } + let target_id = target.id()?; + let confirmation = format!("ERASE {}", target.path().display()); + Ok(Prepared { + plan: Plan { + image_path, + image: geometry, + sha256, + target, + target_id, + confirmation, + }, + source, + }) +} + +impl Prepared { + pub fn write(&self) -> Result<()> { + let target = self.plan.target.open()?; + let source_meta = self.source.metadata()?; + let target_meta = target.metadata()?; + if source_meta.dev() == target_meta.dev() && source_meta.ino() == target_meta.ino() { + return Err(Error("Image and destination are the same open file".into())); + } + image::transfer( + &self.source, + &target, + &self.plan.image, + &self.plan.sha256, + self.plan.target.bytes(), + |phase, bytes| { + self.plan.target.present()?; + eprintln!("{phase}: {bytes}/{} bytes", self.plan.image.bytes); + Ok(()) + }, + )?; + self.plan.target.present()?; + if !self.plan.target.is_file() + && unsafe { libc::ioctl(target.as_raw_fd(), 0x125f as libc::Ioctl) } < 0 + { + return Err(Error(format!( + "Image verified, but the kernel could not reload the partition table: {}. Do not use the disk until it is reconnected and inspected", + std::io::Error::last_os_error() + ))); + } + Ok(()) + } +} diff --git a/rust/fds-workstation/src/lib.rs b/rust/fds-workstation/src/lib.rs index 11ed834..8ec6d57 100644 --- a/rust/fds-workstation/src/lib.rs +++ b/rust/fds-workstation/src/lib.rs @@ -2,6 +2,7 @@ pub mod cartridge; pub mod doctor; pub mod emulator; +pub mod flash; mod qmp; mod serial; pub mod software; diff --git a/tests/integration/arch-package.py b/tests/integration/arch-package.py index dc95039..8b28494 100644 --- a/tests/integration/arch-package.py +++ b/tests/integration/arch-package.py @@ -24,7 +24,7 @@ result=subprocess.run(command,capture_output=True,text=True) (work/'install.log').write_text(result.stdout+result.stderr) assert result.returncode==0,(result.stdout,result.stderr) version=subprocess.check_output([str(project/'tools/version')],text=True).strip() -for tool in ['fds-cartridge','fds-emulator']: +for tool in ['fds-cartridge','fds-emulator','fds-flash']: output=subprocess.check_output([str(root/'usr/bin'/tool),'--version'],text=True).strip() assert output==f'{tool} {version}',output subprocess.run([str(root/'usr/bin'/tool),'--help'],check=True,stdout=subprocess.DEVNULL) @@ -37,6 +37,10 @@ assert not (manual/'.git').exists() assert (root/'usr/share/licenses/fds-tools/RUST-NOTICES.txt').stat().st_size>0 installed=subprocess.check_output(['pacman','--config',str(config),'--root',str(root),'--dbpath',str(root/'var/lib/pacman'),'-Q','fds-tools'],text=True).strip() assert installed==f'fds-tools {version}-1',installed -(work/'acceptance.json').write_text(json.dumps(dict(status='passed',package=str(package),installed=installed,installer='tools/install-workstation',host_install=False),indent=2)+'\n') +# makepkg may set different compiler flags from a direct Cargo build. Exercise +# the installed flasher itself against disposable files, not just its version. +subprocess.run(['python3',str(project/'tests/integration/workstation-flash.py'), + '--cli',str(root/'usr/bin/fds-flash')],check=True) +(work/'acceptance.json').write_text(json.dumps(dict(status='passed',package=str(package),installed=installed,installer='tools/install-workstation',installed_flash_verified=True,host_install=False),indent=2)+'\n') (project/'out/arch-package-current.txt').write_text(str(work)+'\n') -print('PASS: real pacman installation and both installed commands:',work) +print('PASS: real pacman installation and all three installed commands:',work) diff --git a/tests/integration/clean-checks.py b/tests/integration/clean-checks.py index 092fe0d..78feec1 100644 --- a/tests/integration/clean-checks.py +++ b/tests/integration/clean-checks.py @@ -88,6 +88,12 @@ class CleanupTests(unittest.TestCase): self.file('out/manifests/dasung-s6-database.txt', str(self.root / 'out/dasung-s6.new000/compiled')) self.file('out/workstation-images.new000/acceptance.json') self.file('out/workstation-images-current.txt', 'out/workstation-images.new000\n') + self.file('out/workstation-flash.old000/disk.img') + self.file('out/workstation-flash.new000/acceptance.json') + self.file('out/workstation-flash-current.txt', 'out/workstation-flash.new000\n') + self.file('out/workstation-flash-vm.old000/fixture.tar') + self.file('out/workstation-flash-vm.new000/acceptance.json') + self.file('out/workstation-flash-vm-current.txt', 'out/workstation-flash-vm.new000\n') self.file('out/m9-images.old000/root/program.img') self.file('out/m9-images.new000/root/program.img') os.utime(self.root / 'out/m9-images.old000', ns=(1, 1)) @@ -95,7 +101,8 @@ class CleanupTests(unittest.TestCase): self.file('out/m8-vm.other0/disk.img') (self.root / 'out/m8-vm.kept00/dependency').symlink_to('../m8-vm.other0') self.assertEqual(self.selected(), { - 'out/emu-test.old000', 'out/dasung-s6.old000', 'out/m9-images.old000'}) + 'out/emu-test.old000', 'out/dasung-s6.old000', 'out/m9-images.old000', + 'out/workstation-flash.old000', 'out/workstation-flash-vm.old000'}) def test_symlinks_do_not_delete_external_content(self): outside = Path(self.temporary.name) / 'outside' diff --git a/tests/integration/workstation-flash-vm.py b/tests/integration/workstation-flash-vm.py new file mode 100644 index 0000000..cd0e3c7 --- /dev/null +++ b/tests/integration/workstation-flash-vm.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Run the public flasher against disposable NVMe and USB disks in an ARM VM.""" +import io +import json +from pathlib import Path +import shlex +import subprocess +import sys +import tarfile +import tempfile +import time + +project = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(project/'tools')) +from image_formats import digest, gpt, LINUX_FILESYSTEM +from vm_test import VM + +work = Path(tempfile.mkdtemp(prefix='workstation-flash-vm.', dir=project/'out')) +fixtures = Path((project/'out/workstation-flash-current.txt').read_text().strip()) +binary = project/'target/aarch64-unknown-linux-musl/release/fds-flash' +archive = project/'out/rootfs-cli.tar' +for path in (binary, archive, fixtures/'internal.img', fixtures/'system.img'): + assert path.is_file(), path +inputs = [binary, archive, project/'out/kernel/boot/kernel_2712.img', project/'out/fds-initramfs.img'] +(work/'inputs.sha256').write_text(''.join(f'{digest(p)} {p}\n' for p in inputs)) +# This is an isolated fixture with a root serial console and the new workstation +# binary. It does not alter production login, rootfs archives or attached disks. +additions = { + 'usr/libexec/fds/console-session': (b"#!/bin/bash\nexec env HOME=/root PS1='FLASH# ' bash --noprofile --norc\n", 0o755), + 'usr/bin/fds-flash': (binary.read_bytes(), 0o755), + 'usr/share/fds/flash-internal.img': ((fixtures/'internal.img').read_bytes(), 0o644), + 'usr/share/fds/flash-system.img': ((fixtures/'system.img').read_bytes(), 0o644), +} +with tarfile.open(archive) as source, tarfile.open(work/'fixture.tar','w',format=tarfile.PAX_FORMAT) as target: + for member in source: + if member.name not in additions: + target.addfile(member, source.extractfile(member) if member.isfile() else None) + for name,(data,mode) in additions.items(): + member=tarfile.TarInfo(name);member.size=len(data);member.mode=mode + target.addfile(member,io.BytesIO(data)) +(work/'system').mkdir() +with (work/'build.log').open('w') as log: + subprocess.run([str(project/'image/build-system-cartridge'),'--rootfs',str(work/'fixture.tar'), + '--output-directory',str(work/'system')],check=True,stdout=log,stderr=subprocess.STDOUT) +filesystem=work/'mountable.ext4' +with filesystem.open('xb') as stream:stream.truncate(32*1024*1024) +subprocess.run([str(project/'tools/in-image-tools'),'mke2fs','-q','-F','-t','ext4',str(filesystem)],check=True) +disk=work/'nvme.img' +gpt(disk,[('FDS_DATA',LINUX_FILESYSTEM,filesystem)]) +with (work/'4k.img').open('xb') as stream:stream.truncate(32*1024*1024) +extra=['-device','qemu-xhci,id=xhci,addr=05.0', + '-drive',f'file={disk},if=none,id=nvme,format=raw', + '-device','nvme,drive=nvme,serial=FDS-FLASH', + '-drive',f'file={work/"4k.img"},if=none,id=fourk,format=raw', + '-device','nvme,drive=fourk,serial=FDS-4K,logical_block_size=4096,physical_block_size=4096'] + +def wait(read, predicate, timeout=60): + deadline=time.monotonic()+timeout + while True: + result=read() + if predicate(result):return result + assert time.monotonic()/tmp/flash-error',ok=ok) + except AssertionError as error: + raise AssertionError((arguments, vm.capture('cat /tmp/flash-error'))) from error + +def preview(vm,image,path,ok=True): + text=cmd(vm,['--image',image,'--device',path,'--dry-run','--json'],ok) + return json.loads(text)['plan'] if ok else text + +def write(vm,image,path,plan,ok=True): + return cmd(vm,['--image',image,'--device',path,'--unattended', + '--expect-target',plan['target_id'],'--sha256',plan['sha256'],'--json'],ok) + +record = {} +with VM(work,'flash',work/'system/system.img',extra=extra) as vm: + vm.expect(rb'FLASH# ') + vm.capture('s6-rc -b -l /run/s6-rc -d change cartridged') + rows=json.loads(cmd(vm,['list','--json'])) + nvme=next(row for row in rows if row['target'].get('serial')=='FDS-FLASH')['target']['path'] + fourk=next(row for row in rows if row['target'].get('serial')=='FDS-4K')['target']['path'] + source='/usr/share/fds/flash-internal.img' + before=digest(disk) + preview(vm,source,'/dev/vda',False) + assert 'read-only' in vm.capture('cat /tmp/flash-error') + preview(vm,source,fourk,False) + assert '512-byte' in vm.capture('cat /tmp/flash-error') + preview(vm,source,nvme+'p1',False) + assert 'whole disk' in vm.capture('cat /tmp/flash-error') + vm.capture('mkdir /tmp/flash-mounted; mount -o ro,noload '+shlex.quote(nvme+'p1')+' /tmp/flash-mounted') + preview(vm,source,nvme,False) + assert 'mounted' in vm.capture('cat /tmp/flash-error') + vm.capture('umount /tmp/flash-mounted') + assert digest(disk)==before + plan=preview(vm,source,nvme) + assert json.loads(write(vm,source,nvme,plan))['status']=='verified' + # Kernel partition names are independent of the asynchronous udev/blkid + # cache (the deliberately inert payloads are not mountable filesystems). + def labels(): + return vm.capture('cat /sys/class/block/'+Path(nvme).name+"p*/uevent | sed -n 's/^PARTNAME=//p'").split() + labels=wait(labels,lambda names:names==['FDS_BOOT','FDS_RECOVERY','FDS_INTERNAL']) + assert labels==['FDS_BOOT','FDS_RECOVERY','FDS_INTERNAL'],labels + record['nvme_internal_write_and_kernel_partition_reread']=True + # Reinstalling the same offline FDS disk is an intended operation. + source='/usr/share/fds/flash-system.img' + plan=preview(vm,source,nvme) + assert json.loads(write(vm,source,nvme,plan))['status']=='verified' + assert wait(lambda:vm.capture('cat /sys/class/block/'+Path(nvme).name+"p*/uevent | sed -n 's/^PARTNAME=//p'").split(), + lambda names:names==['FDS_SYSTEM'])==['FDS_SYSTEM'] + record['reflash_internal_disk_with_system']=True + record['mounted_partition_readonly_and_4k_refused']=True + # Hotplug only generated files into this VM. No host block paths are used. + def attach(name, rule=None): + path=work/(name+'.img') + with path.open('xb') as stream:stream.truncate(32*1024*1024) + vm.qmp('blockdev-add',{'driver':'raw','node-name':name,'file':{'driver':'file','filename':str(path)}}) + top=name + if rule: + top=name+'-fault' + vm.qmp('blockdev-add',{'driver':'blkdebug','node-name':top,'image':name, + 'inject-error':[{'event':'none','errno':5,'once':False,**rule}]}) + vm.qmp('device_add',{'driver':'usb-storage','id':name,'drive':top,'bus':'xhci.0','port':'2','serial':name}) + def rows():return json.loads(cmd(vm,['list','--json'])) + entries=wait(rows,lambda entries:any(row['target'].get('serial')==name for row in entries)) + return path,next(row['target']['path'] for row in entries if row['target'].get('serial')==name) + def detach(name): + vm.qmp('device_del',{'id':name}) + wait(lambda:json.loads(cmd(vm,['list','--json'])),lambda entries:all(row['target'].get('serial')!=name for row in entries)) + usb,path=attach('usb-first') + plan=preview(vm,source,path) + detach('usb-first') + usb2,path2=attach('usb-replacement') + write(vm,source,path2,plan,False) + assert 'Target identity' in vm.capture('cat /tmp/flash-error') + fresh=preview(vm,source,path2) + assert json.loads(write(vm,source,path2,fresh))['status']=='verified' + record['usb_replacement_refused_and_fresh_write_verified']=True + detach('usb-replacement') + for name,rule in [('write-error',{'iotype':'write','sector':2056}), + ('flush-error',{'iotype':'flush'}), + ('readback-error',{'iotype':'read','sector':2056})]: + target,path=attach(name,rule) + plan=preview(vm,source,path) + output=write(vm,source,path,plan,False) + error=vm.capture('cat /tmp/flash-error') + assert 'os error 5' in error,(name,error) + assert 'verified' not in output + record[name]=True + detach(name) + # The complete wizard must select the intended disk without CLI paths. + rows=json.loads(cmd(vm,['list','--json'])) + selection=next(n+1 for n,row in enumerate(rows) if row['target']['path']==nvme) + vm.send('fds-flash; printf "\\nFLASH_WIZARD_STATUS:%s\\n" "$?"') + vm.expect(rb'Image path: ');vm.send(source) + vm.expect(rb'Disk number \(no default\): ');vm.send(str(selection)) + vm.expect(rb' to proceed: ');vm.send('ERASE '+nvme) + vm.expect(rb'VERIFIED: image written') + vm.expect(rb'FLASH_WIZARD_STATUS:0\r?\n') + record['interactive_image_disk_selection_and_confirmation']=True + vm.capture('s6-rc -b -l /run/s6-rc -u change cartridged') + vm.send('fds poweroff');vm.expect(rb'reboot: Power down');assert vm.child.wait(timeout=20)==0 +# The final successful NVMe write has a valid GPT on the actual backing file. +subprocess.run(['sfdisk','--verify',str(disk)],check=True) +record.update(status='passed',physical_hardware='not tested',source_binary_sha256=digest(binary)) +(work/'acceptance.json').write_text(json.dumps(record,indent=2)+'\n') +(project/'out/workstation-flash-vm-current.txt').write_text(str(work)+'\n') +print('PASS: real virtual NVMe/USB flashing, protected disks, replacement and I/O failures:',work) +print('SKIP: physical Pi, drive/controller flush behavior and power-loss recovery') diff --git a/tests/integration/workstation-flash.py b/tests/integration/workstation-flash.py new file mode 100644 index 0000000..357bec0 --- /dev/null +++ b/tests/integration/workstation-flash.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Exercise the actual interactive and unattended flasher using disposable files.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import pty +import selectors +import shutil +import struct +import subprocess +import sys +import tempfile +import time + +project = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(project / 'tools')) +from image_formats import gpt, LINUX_FILESYSTEM, EFI_SYSTEM, digest + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--cli', type=Path, required=True) +args = parser.parse_args() +cli = args.cli.resolve() +work = Path(tempfile.mkdtemp(prefix='workstation-flash.', dir=project / 'out')) +log = (work / 'commands.log').open('w') + +# Deliberately inert filesystem signatures in small independent GPT fixtures. +# These test the writer and table validation, not filesystem usability or boot. +parts = [] +for name, kind, signature, offset in [ + ('FDS_BOOT', EFI_SYSTEM, b'FAT32 ', 82), + ('FDS_RECOVERY', LINUX_FILESYSTEM, bytes.fromhex('e2e1f5e0'), 1024), + ('FDS_INTERNAL', LINUX_FILESYSTEM, bytes.fromhex('53ef'), 1080), +]: + path = work / (name + '.bin') + with path.open('xb') as stream: + stream.truncate(1024 * 1024) + stream.seek(offset); stream.write(signature) + if name == 'FDS_BOOT': + stream.seek(11); stream.write(b'\0\x02') + stream.seek(510); stream.write(b'\x55\xaa') + stream.seek(8192); stream.write(b'known payload, not a bootable filesystem') + parts.append((name, kind, path)) +internal = work / 'internal.img' +gpt(internal, parts) +system = work / 'system.img' +gpt(system, [('FDS_SYSTEM', LINUX_FILESYSTEM, parts[1][2])]) + + +def invoke(arguments, ok=True): + result = subprocess.run([str(cli), *map(str, arguments)], capture_output=True, text=True, timeout=60) + log.write(repr(arguments) + '\n' + result.stdout + result.stderr); log.flush() + assert (result.returncode == 0) == ok, (arguments, result.returncode, result.stdout, result.stderr) + return result + + +def target(name, size): + path = work / name + with path.open('xb') as stream: stream.truncate(size) + return path + + +def preview(image, disk): + return json.loads(invoke(['--image', image, '--device', disk, '--file-target', '--dry-run', '--json']).stdout)['plan'] + + +def unattended(image, disk, plan, **changes): + return ['--image', image, '--device', disk, '--file-target', '--unattended', + '--expect-target', changes.get('target_id', plan['target_id']), + '--sha256', changes.get('sha256', plan['sha256']), '--json'] + + +def verify(image, disk): + subprocess.run(['sfdisk', '--verify', str(disk)], check=True, stdout=log, stderr=log) + original = json.loads(subprocess.check_output(['sfdisk', '--json', str(image)]))['partitiontable'] + written = json.loads(subprocess.check_output(['sfdisk', '--json', str(disk)]))['partitiontable'] + assert original['id'] == written['id'] + assert written['lastlba'] == disk.stat().st_size // 512 - 34 + with image.open('rb') as source, disk.open('rb') as output: + for a, b in zip(original['partitions'], written['partitions'], strict=True): + for key in ('start', 'size', 'type', 'uuid', 'name'): assert a[key] == b[key], (a,b) + source.seek(a['start']*512); output.seek(b['start']*512) + assert source.read(a['size']*512) == output.read(b['size']*512) + if disk.stat().st_size == image.stat().st_size: assert digest(image) == digest(disk) + + +for image in [internal, system]: + source_hash = digest(image) + for extra in [0, 512, 4*1024*1024]: + disk = target(f'{image.stem}-{extra}.target', image.stat().st_size + extra) + before = digest(disk) + plan = preview(image, disk) + assert digest(disk) == before + result = invoke(unattended(image, disk, plan, target_id='0'*64), False) + assert 'Target identity' in result.stderr and digest(disk) == before + result = invoke(unattended(image, disk, plan, sha256='0'*64), False) + assert 'SHA-256' in result.stderr and digest(disk) == before + assert json.loads(invoke(unattended(image, disk, plan)).stdout)['status'] == 'verified' + verify(image, disk) + invoke(unattended(image, disk, plan), False) # A file target's contents changed. + assert digest(image) == source_hash + +for name, partitions in [ + ('data', [('FDS_DATA', LINUX_FILESYSTEM, parts[2][2])]), + ('environment', [('FDS_ENVIRONMENT', LINUX_FILESYSTEM, parts[1][2])]), + ('program', [('FDS_PROGRAM', LINUX_FILESYSTEM, parts[1][2])]), + ('software', [(label, LINUX_FILESYSTEM, parts[1][2]) for label in ('FDS_METADATA','FDS_PAYLOAD02','FDS_PAYLOAD03')]), +]: + image=work/(name+'.img');gpt(image,partitions) + disk=target(name+'.target',image.stat().st_size+1024*1024) + plan=preview(image,disk) + assert json.loads(invoke(unattended(image,disk,plan)).stdout)['status']=='verified' + verify(image,disk) + +# Invalid source geometry and destinations must fail without writing. +disk = target('protected.target', internal.stat().st_size) +before = digest(disk) +invoke(['--image', internal, '--device', disk, '--unattended'], False) +invoke(['--image', internal, '--device', disk, '--file-target'], False) # No terminal. +invoke(['--image', internal, '--device', disk, '--dry-run'], False) # Not a block device. +small = target('small.target', 1024) +invoke(['--image', internal, '--device', small, '--file-target', '--dry-run'], False) +invoke(['--image', internal, '--device', internal, '--file-target', '--dry-run'], False) +alias = work/'hardlink.img'; os.link(internal, alias) +invoke(['--image', internal, '--device', alias, '--file-target', '--dry-run'], False) +for name, change in [('bad-primary', 512+16), ('bad-backup', internal.stat().st_size-512+16), + ('bad-table', 1024+56), ('bad-filesystem', 2048*512+82)]: + image = work/(name+'.img'); shutil.copyfile(internal,image) + with image.open('r+b') as stream: + stream.seek(change); byte=stream.read(1); stream.seek(change); stream.write(bytes([byte[0]^1])) + invoke(['--image', image, '--device', disk, '--file-target', '--dry-run'], False) +invoke(['--image', parts[0][2], '--device', disk, '--file-target', '--dry-run'], False) +assert digest(disk) == before + +# Drive the real terminal workflow, including cancellation and changed inputs +# after the review is displayed. No shell evaluates the user's paths or input. +def interactive(name, reply, mutate=None, ok=True, include_image=True): + disk = target(name+'.target', system.stat().st_size + 1024*1024) + image = work/(name+'.img'); shutil.copyfile(system,image) + before = digest(disk) + master, slave = pty.openpty() + command = [str(cli), '--device', str(disk), '--file-target', '--json'] + if include_image: command += ['--image', str(image)] + child = subprocess.Popen(command, stdin=slave, stderr=slave, stdout=subprocess.PIPE, text=True) + os.close(slave) + selector=selectors.DefaultSelector();selector.register(master,selectors.EVENT_READ) + output=bytearray() + def until(marker): + deadline=time.monotonic()+20 + while marker not in output: + assert time.monotonic()