diff --git a/AGENTS.md b/AGENTS.md index 4434523..67dc5be 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # FDS/OS implementation scope -Read `docs/master-plan.md` and `docs/build-host.md` before changing the build. +Read `docs/developer/master-plan.md` and `docs/developer/build-host.md` before changing the build. The user has explicitly authorized implementation through M12. Proceed through the remaining milestones in order, keeping the repository buildable and recording actual acceptance evidence. Defer tests that require the physical Raspberry Pi @@ -34,10 +34,12 @@ claim Pi boot or physical power-cycle recovery is verified. # Workstation software and emulator follow-up Read `docs/workstation.md`, `docs/software-format.md` and -`docs/workstation-tooling-plan.md` for this extension. Software builds and new +`docs/developer/workstation-tooling-plan.md` for this extension. Software builds and new PROGRAM cartridge creation run on generic Linux workstations. The public native Clap tools are `fds-cartridge` and `fds-emulator`. Software images have GPT -metadata partition 1 plus m EROFS payload partitions holding xz tarballs. Keep +metadata partition 1 plus m EROFS payload partitions containing installed Void +package trees. Build source templates with xbps-src on the workstation; do not +create new xz software bundles or extract programs at guest launch. Keep legacy reading and base Dasung integration. Do not mutate the frozen 0.1.0 release or treat its historical acceptance as evidence for changed sources. Validate with `make workstation-test` and `make emulator-test` plus relevant guest checks. diff --git a/Cargo.lock b/Cargo.lock index fdb0183..f75937f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -308,6 +308,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "fds-boottrace" version = "0.1.0" @@ -364,6 +374,17 @@ dependencies = [ "toml", ] +[[package]] +name = "fds-control" +version = "0.1.0" +dependencies = [ + "clap", + "fds-common", + "libc", + "serde_json", + "x11rb", +] + [[package]] name = "fds-release" version = "0.1.0" @@ -459,6 +480,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix", + "windows-link", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -520,6 +551,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "memchr" version = "2.8.3" @@ -602,6 +639,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "semver" version = "1.0.28" @@ -833,6 +883,23 @@ dependencies = [ "memchr", ] +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "gethostname", + "rustix", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + [[package]] name = "zeroize" version = "1.9.0" diff --git a/Cargo.toml b/Cargo.toml index f80601c..0e24c79 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "rust/fds-release", "rust/fds-software", "rust/fds-workstation", + "rust/fds-control", "tests/helpers", ] diff --git a/Makefile b/Makefile index 7327fe0..e6f367e 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,14 @@ SHELL := /bin/bash .SHELLFLAGS := -eu -o pipefail -c .DEFAULT_GOAL := help .NOTPARALLEL: -.PHONY: console-vm cartridge-test data-test desktop-test +.PHONY: console-vm cartridge-test data-test desktop-test clean clean-preview clean-test .PHONY: help bootstrap smoke-test check dasung dasung-test rootfs rootfs-test init-test vm tooling tooling-test kernel initramfs system-card boot-volume boot-test console-test performance-test help: @printf '%s\n' 'FDS/OS — static Rust tools and native s6' \ + 'make clean-preview Show obsolete build/test directories and Rust output selected for cleanup' \ + 'make clean Remove that output; keep current images, latest tests, caches and releases' \ + 'make clean-test Test cleanup protections using small disposable fixtures' \ 'make workstation Build native Linux software/cartridge and QEMU command-line tools' \ 'make workstation-test Verify software builds, multi-partition images and file write/readback' \ 'make emulator-test Boot FDS and test cartridge hotplug with the public emulator' \ @@ -51,6 +54,15 @@ help: 'make init-test Boot an ARM VM and verify s6 PID 1 and service control' \ 'make vm Open a temporary root shell in the verified test VM' +clean-preview: + ./tools/clean-builds --dry-run + +clean: + ./tools/clean-builds + +clean-test: + python3 tests/integration/clean-checks.py + bootstrap: @mkdir -p out/logs ./tools/bootstrap-host 2>&1 | tee out/logs/bootstrap.log @@ -81,6 +93,7 @@ smoke-test: check: ./tests/integration/m0-checks cargo fmt --all -- --check + $(MAKE) clean-test dasung: @mkdir -p out/logs @@ -235,7 +248,7 @@ workstation: workstation-test: workstation cargo test --locked --offline --target $$(rustc -vV | sed -n 's/^host: //p') -p fds-common -p fds-burn -p fds-software -p fds-workstation - python3 tests/integration/workstation-images.py --cli out/workstation/fds-cartridge --image-tool-runner tools/in-image-tools --cc "$(CURDIR)/tools/in-void" aarch64-linux-gnu-gcc 2>&1 | tee out/logs/workstation-images.log + python3 tests/integration/workstation-images.py --cli out/workstation/fds-cartridge --image-tool-runner tools/in-image-tools --xbps-tool-runner tools/in-void --xbps-bin "$(CURDIR)/.host/xbps/usr/bin" 2>&1 | tee out/logs/workstation-images.log emulator-test: workstation python3 tests/integration/workstation-emulator.py --cli out/workstation/fds-emulator --qemu-runner tools/in-void 2>&1 | tee out/logs/workstation-emulator.log diff --git a/README.md b/README.md index aa2c50a..1d7700e 100644 --- a/README.md +++ b/README.md @@ -1,293 +1,73 @@ -# FDS/OS — Felis Data Systems Operating System +# FDS/OS -FDS/OS is an operating system project for the **Felis FP-85**, a planned retro -portable computer built around a Raspberry Pi 5, a Dasung Paperlike 13K grayscale E-Ink display, a removable -battery, and 12 USB cartridge bays. +FDS/OS is a Linux operating system for the Felis FP-85, a Raspberry Pi 5 computer +with twelve USB cartridge bays and a Dasung Paperlike 13K display. It combines +Void Linux packages, native s6 services and static Rust system tools with a +simple cartridge workflow: insert software, run it, and eject it when finished. -The idea is to make the computer's software and data removable. A **SYSTEM** -cartridge carries the operating system, a **DATA** cartridge carries the user's -files, and **PROGRAM** or **ENVIRONMENT** cartridges add applications or select -an environment such as WindowMaker. The internal NVMe holds the boot machinery, -recovery environment, and machine configuration. Swapping a SYSTEM cartridge is -the intended update and rollback mechanism. +## How the system fits together -The project aims to combine familiar Linux software with fast, predictable -startup and shutdown. It uses the Void/XBPS package ecosystem for ordinary Linux -applications, uses native s6 service management, and builds FDS-owned system -tools as small, self-contained Rust executables. - -## What you can use today - -**M0–M12 software acceptance is complete, including a signed local FDS/OS 0.1.0 release.** -Current development also adds [Linux workstation software cartridges and a QEMU emulator](docs/workstation.md). These changes are separate from the frozen 0.1.0 release. -Physical Pi testing is deferred; see -the [implementation ledger](docs/implementation-status.md). -The cross-build environment is implemented and verified on x86_64 Arch Linux. You can build and inspect ARM Linux software from your -Arch machine without owning a Pi. - -| Available now | What it gives you | +| Component | Purpose | | --- | --- | -| `out/fds-os-0.1.0/` | Signed local images, preserved source/build inputs and independent-build comparison; start with its `README.md` | -| `make workstation` | Native Linux `fds-cartridge` and `fds-emulator`: build xz bundles, create/verify 1+m GPT images, preview USB writes, and hotplug images into twelve virtual bays | -| `make bootstrap` | Project-local XBPS tools, a Void build container, and the pinned Rust toolchain | -| `make all` / `make packages` | The complete image set, or just the FDS base packages; see [Complete builds](docs/reproducible-builds.md#prepare-the-snapshot) | -| `make smoke-test` | An ARM glibc XBPS package and an ARM static-musl Rust executable, with automated checks | -| `make check` | Checks for invalid binaries, source pin drift, unsafe overlay replacement, and Rust formatting | -| `make tooling` / `make tooling-test` | Static ARM `fds` and stage0 diagnostic tools, with parsing/discovery tests | -| `make dasung` / `make dasung-test` | Static ARM monitor daemon, base XBPS package, s6 definitions, and simulated monitor tests | -| `make rootfs PROFILE=cli` / `make rootfs-test` | Configured ARM rootfs archive, emulated shell access, and archive acceptance checks | -| `make init-test` / `make vm` | Full ARM boot test, native s6 PID 1, service control, and a temporary development shell | -| `make kernel`, `make initramfs`, `make system-card`, `make boot-volume` | Pi kernel, early userspace, GPT/EROFS SYSTEM and FAT32 boot partition; see [Boot images](docs/boot.md) | -| `make boot-test` | Actual stage0 handoff, missing-media insertion, recovery and archive-format checks in an ARM VM | -| `make console-vm`, `make console-test`, `make performance-test` | Ordinary-user FDS console, boot tracing and measured VM optimization comparison; see [M5 evidence](docs/m5-validation.md) | -| `make cartridge-test` | Virtual USB bay mapping, hardware recognition, metadata/mount rejection checks and safe read-only eject | -| `make data-test` | Writable `/data`, managed background programs, sustained writes, safe eject and independent ext4 checks; see [DATA usage](docs/data.md) | -| `make rootfs PROFILE=development` / `make desktop-test` | WindowMaker, grayscale styling, trusted ENVIRONMENT activation, managed PROGRAM execution and on-demand Ethernet; see [Desktop usage](docs/desktop.md) | -| `make media-image-test` / `make media-test` | SYSTEM/DATA/ENVIRONMENT creation, legacy PROGRAM compatibility, confirmed writes, readback verification and boot of a newly written SYSTEM; see [Media tools](docs/media-tools.md) | -| `make recovery` / `make recovery-test` | Separate recovery rootfs and EROFS, local maintenance console, DATA checks/repairs and replacement SYSTEM workflow; software checks passed, see [Recovery](docs/recovery.md) | -| `make internal-image` / `make internal-test` | Complete internal GPT image and persistent bay/hardware settings; see [Internal storage](docs/internal-storage.md) | -| `make development-test` | Native C/C++/Rust builds, debuggers, build systems, Git and Vim inside the ARM development image | -| `make signing` / `make signing-test` | Host and ARM release signing/verification tools, checked independently against OpenSSL | -| `make eeprom` / `make eeprom-test` | Reversible, verified EEPROM configuration files without applying anything to hardware; see [EEPROM](docs/eeprom.md) | -| Source pinning and input records | A fixed Void source commit plus logs and hashes of the actual build inputs | +| Internal NVMe | Holds boot files, recovery and persistent machine settings | +| SYSTEM cartridge | Supplies the read-only operating system; swapping it changes the OS | +| PROGRAM cartridge | Supplies applications and their dependencies, ready to execute | +| DATA cartridge | Stores writable files at `/data` | +| ENVIRONMENT cartridge | Selects an installed environment, such as WindowMaker | -**The kernel → stage0 → SYSTEM → s6 path boots in an ARM virtual machine.** -The filesystem includes native s6 init, runtime mounts, console and device -services, and Dasung supervision. The ordinary-user FDS console and boot tracing are available. M6 adds the cartridge daemon and `fds bays`, `fds bay`, `fds cartridge`, and -`fds eject`; see [Cartridge usage](docs/cartridges.md). The complete internal NVMe image and independent recovery are implemented; -[release signature tooling](docs/releases.md) has passed host and ARM checks. -[Frozen inputs, offline rebuilds and complete release assembly](docs/reproducible-builds.md) -passed two independent offline builds with all 27 artifacts identical. The local -release is at `out/fds-os-0.1.0/`; [M12 validation](docs/m12-validation.md#final-local-release-acceptance) -records the signing identity, verification commands and exact evidence. -M10's `fds poweroff` and -`fds reboot` have passed software validation; see [Shutdown usage](docs/power.md). The `fds` -tool also provides identity, boot tracing and manifest inspection; see [Rust tooling](docs/tooling.md). See the [roadmap](docs/roadmap.md). +The default interface is a console. WindowMaker provides an optional grayscale +retro desktop and an FDS Control panel for inspecting bays, launching programs +and ejecting cartridges. The same theme is used on ordinary screens and E-Ink. +The Dasung controller is part of the base system and starts independently of +the desktop or any ENVIRONMENT cartridge. -## Start here +## Choose a starting point -- **I want to build software cartridges or try virtual insertion/removal:** follow the [workstation walkthrough](docs/workstation.md). -- **I want to understand the project:** read [Architecture](docs/architecture.md). -- **I want to try the working code:** follow [Your first build](docs/getting-started.md). -- **I want to build the OS filesystem:** follow [Rootfs](docs/rootfs.md). -- **I want to boot it and control services:** follow [Native init and ARM VM](docs/init.md). -- **I want to build the Pi boot files and SYSTEM image:** follow [Boot images](docs/boot.md). -- **I want to use the desktop, applications or Ethernet:** follow [Desktop usage](docs/desktop.md). -- **I want to create or write a cartridge:** follow [Media tools](docs/media-tools.md). -- **I want persistent bay settings and an internal NVMe image:** follow [Internal storage](docs/internal-storage.md). -- **SYSTEM will not boot, or DATA needs checking:** follow [Recovery](docs/recovery.md). -- **I have already built it and want to develop:** use the [Development guide](docs/development.md). -- **A command failed:** look up its message in [Troubleshooting](docs/troubleshooting.md). -- **I do not know the terminology:** keep the [Glossary](docs/glossary.md) open. +- **Build and try FDS:** follow [Build and start FDS](docs/getting-started.md). +- **Build an application cartridge:** use the [software and emulator walkthrough](docs/workstation.md). +- **Use the computer:** read [Using cartridges](docs/cartridges.md), [DATA](docs/data.md), and [the desktop guide](docs/desktop.md). +- **Install or maintain a machine:** see [internal storage](docs/internal-storage.md), [release verification](docs/releases.md), and [recovery](docs/recovery.md). +- **Free build space:** run `make clean-preview`, then `make clean`; see [cleanup](docs/cleanup.md). -The [documentation index](docs/README.md) organizes the remaining guides. The -[master plan](docs/master-plan.md) is the full specification, rather than the -recommended starting point for learning to use the repository. +The [user manual](docs/README.md) is organized around tasks. Build internals, +design decisions, hardware procedures and validation records live separately in +[developer notes](docs/developer/README.md). -## Quick start: software cartridges and emulation +## A typical software session -On a Linux workstation, build the native tools with `make workstation`, then -follow [the complete walkthrough](docs/workstation.md) to package the included -hello/report examples, assemble a cartridge, boot FDS and insert it. This tooling -uses normal Linux QEMU, EROFS and xz utilities; it does not run on the Pi. -Building the complete OS images still uses the environment below. - -## Quick start on Arch Linux - -You need an **x86_64 Arch Linux host**, a regular user account with access to sudo -for installing prerequisites, a network connection, and several GB of free disk -space. The kernel must permit unprivileged user namespaces. A Pi, Docker, and a -separate musl C compiler are not required for the current smoketest. - -Use a checkout path without spaces. In the current workspace: +These commands run on FDS after inserting the example PROGRAM cartridge in bay 1: ```sh -cd /home/felis/source/fds -``` - -On another machine, enter your own FDS checkout directory instead. Use an actual -Git checkout so the pinned Void submodule can be initialized for a normal online -build. A complete [frozen-input snapshot](docs/reproducible-builds.md) instead -restores the source and its bundled Void checkout for offline building; a bare -source archive alone does not supply those inputs. This repository does not currently -configure a public Git remote, so no public clone URL is assumed here. - -Install the prerequisites once: - -```sh -sudo pacman -S --needed bash coreutils binutils git curl make file tar xz gzip zstd \ - bubblewrap rustup ca-certificates findutils diffutils grep sed gawk util-linux -``` - -Then run these commands **in order, as your regular user**: - -```sh -make bootstrap -make smoke-test -make check -``` - -`make bootstrap` downloads and prepares the build tools. `make smoke-test` -compiles the two ARM artifacts and verifies their architectures and library -requirements. `make check` exercises the validation failure cases and checks -formatting. If a command fails, stop at that step and use the linked -[troubleshooting guide](docs/troubleshooting.md); a partially completed run is -not a successful build. - -The first run downloads hundreds of MB of compiler and build dependencies and -may take a while on a slow connection. Later runs reuse caches, although XBPS -operations may still access the network. `cargo --offline` applies only to Rust -crate downloads, not to the whole pipeline. - -Successful smoke-test output ends with: - -```text -PASS: aarch64 static ELF -... -PASS: XBPS package hello-2.12.3_1 architecture=aarch64 (glibc) -SKIP: ARM execution (optional qemu-aarch64 not installed); ELF verification passed -PASS: M0 smoke test complete -``` - -The QEMU line is a documented optional skip. If QEMU is installed, a runtime PASS -replaces it. The [first-build guide](docs/getting-started.md) explains how to run -the executable with QEMU and interpret every result. - -To build the current OS filesystem and boot its development VM, continue with: - -```sh -sudo pacman -S --needed python e2fsprogs libarchive lz4 -make rootfs PROFILE=cli -make rootfs-test -make init-test -make vm -``` - -Python must be version 3.14 or newer. Additional emulation tools are installed -inside the project-local build container. `make init-test` boots ARM Linux, -checks native init and service control, and powers off. `make vm` repeats those -checks and opens a temporary root shell; type `exit` to shut it down. Nothing is -flashed to a physical disk. The [VM guide](docs/init.md) explains the console, -service commands, logs, and limits in detail. - -To build the complete operating-system image set and try its ordinary user -console, continue from that prepared workstation: - -The complete images require substantially more space than the initial smoke -test: the offline diagnostic build occupied about 24 GB, before additional VM -test copies. See [build-space observations](docs/reproducible-builds.md#what-is-frozen) -before preparing two independent release builds. - -```sh -make all -make init-test -make boot-test -make console-vm -``` - -`make all` builds CLI and development SYSTEM cartridges, independent recovery -and the internal disk. `make init-test` prepares the full QEMU environment and -checks native init. `make boot-test` checks startup, missing-media recovery and -all initramfs formats. `make console-vm` opens the actual `FDS>` console. -Enter these commands **inside that console**: - -```sh -fds info fds bays -fds boot-profile -fds poweroff +fds bay 01 +hello 'Hello from FDS' +fds run 01 -- demo.report:report +fds eject 01 ``` -This VM has no physical USB devices attached, so empty cartridge bays are -expected. Its home directory is temporary; use a DATA cartridge for persistent -files on the real machine. `fds poweroff` performs an orderly shutdown. The -[boot guide](docs/boot.md#use-the-ordinary-fds-console) explains console controls, -and the [internal-storage guide](docs/internal-storage.md) covers physical -installation when the Pi hardware is ready. +`hello` runs in your current terminal. `fds run` starts a managed background +command. Both forms are tracked for eject. Wait for **SAFE** before removing +media; use `fds poweroff` to shut down the whole computer. -## Dasung monitor support +## Build tools -The existing Rust `dasungd` from the monitor troubleshooting task is integrated -as a required base-system package, with a native s6 boot service. Build it with -`make dasung`, then run `make dasung-test`. The [Dasung guide](docs/dasung.md) -explains the exact monitor profile, dependencies, configuration, and Pi validation -still needed. M2 starts the controller under native s6 in the ARM VM; the VM has no physical monitor. +Run these from the repository root: -## What the build produces - -| Path | Contents | How to use it | -| --- | --- | --- | -| `out/fds-internal.img` | Complete GPT disk with BOOT, independent RECOVERY and persistent machine-settings partitions | Use the [internal installation procedure](docs/internal-storage.md); this belongs on the internal NVMe | -| `out/fds-system-cli.img` | Immutable CLI SYSTEM cartridge, including base Dasung support | Build with `make system-card PROFILE=cli`; use the [media writer](docs/media-tools.md) to write a SYSTEM cartridge | -| `out/fds-system-development.img` | SYSTEM with native compilers, debugging tools and the optional desktop runtime | Build with `make rootfs PROFILE=development` followed by `make system-card PROFILE=development` | -| `out/fds-boot.img` / `out/fds-recovery.img` | Individual FAT32 BOOT and EROFS RECOVERY partition payloads | Normally use the complete internal image; these are component images, not interchangeable whole disks | -| `out/fds-initramfs.img` | Early boot program and base display support | Included in BOOT; four compressed/uncompressed formats are also under `out/initramfs/` | -| `out/rootfs-aarch64.tar` | Configured ARM glibc userspace, including Dasung | Follow [M1 rootfs](docs/rootfs.md) to build, inspect, and run it | -| `out/m2-vm-latest/` | Last successful ARM test disk, serial log, environment and hashes | Use `make vm` to boot it; see [Native init](docs/init.md) | -| `out/fds-smoketest` | Static-musl AArch64 Rust executable | Inspect with `file`/`readelf`; optionally run with `qemu-aarch64` | -| `out/packages/hello-2.12.3_1.aarch64.xbps` | GNU hello packaged for ARM glibc | Query or extract it with the [package guide](docs/packages.md) | -| `out/packages/aarch64-repodata` | Local package index | Lets XBPS query the exported package | -| `out/logs/` | Bootstrap and build logs | Diagnose a failed or apparently stalled step | -| `out/manifests/` | Package inventories and SHA-256 digests | Inspect which inputs and outputs were used | - -`make all` produces the complete image set above in order. It prepares files and -does not write physical disks. Build and test commands share a build container; -run them sequentially. Stable paths point to the latest successful versioned -build directories. See [Offline rebuilds](docs/reproducible-builds.md) for the -separate frozen-input and release-verification workflow. - -The rootfs tar and individual binaries are not flashable Pi images. The separate -VM disk boots with the generic ARM test kernel. The Rust -executable is ARM code and normally cannot run directly on the x86_64 host. -The XBPS package is not an Arch package and is not installed with pacman. - -## How the build relates to the future machine - -```text -Working today: x86_64 Arch workstation - ├─ pinned Void + xbps-src + GNU cross compiler → aarch64 glibc package - └─ pinned Rust + bundled musl + rust-lld → static AArch64 executable - -Pi boot path, software-tested in a VM; physical validation pending: - internal NVMe kernel/initramfs → SYSTEM cartridge → native s6 → FDS console -``` - -The first line supplies conventional Linux packages. The second supplies FDS's -own control tools without a dependency on the installed glibc version. Both -produce ARM code; only their library/linking strategy differs. See -[the architecture guide](docs/architecture.md#two-library-strategies). - -## Where to find things - -| Directory or file | Purpose today | +| Command | Result | | --- | --- | -| `rust/fds-smoketest/`, `rust/dasungd/`, `rust/fds-common/`, `rust/fds-cli/`, `rust/fds-stage0/` | Static-linking smoketest, monitor controller, shared contracts, CLI and early boot | -| `tools/` | Bootstrap, package build, overlay preparation, and ELF checks | -| `config/` | Pinned host tool download and Void build configuration | -| `vendor/void-packages/` | Unmodified upstream source at the recorded commit | -| `tests/integration/m0-checks` | Executable validation and protection checks | -| `docs/` | Tutorials, design explanations, references, and validation evidence | -| `packages/fds-dasungd/`, `s6/source/`, `image/base-packages.list` | Monitor package, native service definitions, and mandatory base inclusion | -| `packages/fds-base/`, `packages/fds-base-files/`, `packages/fds-init/` | Base dependencies, identity, accounts, layout and native init | -| `packages/fds-kernel/`, `packages/fds-cli/`, `image/` | Pi kernel delta, static CLI package and filesystem/boot builders | -| `profiles/` | CLI, development and recovery package selections | -| `rust/fds-release/`, `tools/frozen-inputs`, `tools/assemble-release` | Signature verification, input preservation and local release assembly | +| `make workstation` | Native Linux `fds-cartridge` and `fds-emulator` executables | +| `make bootstrap` | Prepares the complete OS build environment on x86_64 Arch Linux | +| `make rootfs PROFILE=cli` | Builds the ARM base filesystem and its packages | +| `make system-card PROFILE=cli` | Creates a whole-disk SYSTEM cartridge image | +| `make initramfs` | Creates early boot userspace | +| `make all` | Builds the complete CLI/development, recovery and internal image set | +| `make clean` | Removes obsolete generated workspaces and Rust build output | -Generated `.host/`, `target/`, and `out/` contents are ignored by Git. Rustup also -installs the pinned toolchain in your user account's Rustup directory. The -[build reference](docs/build-host.md) details prerequisites, storage, and pins. +New software cartridges are built from metadata and Void source packages using +`xbps-src` on a Linux workstation. Their EROFS partitions contain installed +programs and dependencies directly. FDS does not extract those programs at launch. -## Project status and evidence - -The [M0 validation report](docs/m0-validation.md) records the actual build commands, -results, and initial artifact hashes. QEMU execution and Pi hardware were not -validated in that initial run. Later milestones measured VM boot and shutdown; -physical Pi performance remains unmeasured. Source and toolchain pins are in -place, and the complete frozen-input/offline release workflow is undergoing M12 -acceptance. See [current evidence](docs/m12-validation.md) for the distinction. - -The [M1 guide](docs/rootfs.md) explains rootfs usage; [M1 validation](docs/m1-validation.md) records the results. -M2 usage is in [Native init and ARM VM](docs/init.md), with results in [M2 validation](docs/m2-validation.md). -M3 adds [static-musl FDS tooling](docs/tooling.md). Work continues through M12 -under [AGENTS.md](AGENTS.md), with physical hardware checks deferred. Project-owned -source comments, documentation, diagnostics, and development discussions use English. +The workstation walkthrough includes prerequisites, complete example commands, +expected outputs, emulator lifecycle and the confirmed USB-writing workflow. +Append `--help` to either native tool for its command reference; run `make help` +for the complete build target list. diff --git a/docs/README.md b/docs/README.md index 8b0b682..841fe88 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,86 +1,25 @@ -# Documentation guide +# FDS/OS user manual -FDS/OS builds bootable ARM SYSTEM images, an internal NVMe disk, independent -recovery, and a development environment. Native s6 manages the base system, -including Dasung support; cartridge tools manage DATA, programs and optional -desktop/network activation. M0–M12 software checks and local release -verification have passed. Physical Pi validation is deferred. The guides -below explain how to build, use and test the implemented system. +FDS/OS runs a cartridge-based Linux computer. Start with the overview, then choose +the guide for building software, trying the emulator, or operating the machine. +Commands in workstation guides run on your Linux computer; commands in the +operating guides run at the FDS console unless stated otherwise. -## If this is your first visit - -1. Read the [project overview](../README.md) for the intended computer and current capabilities. -2. Follow [Your first build](getting-started.md) to create and inspect the working artifacts. -3. Read [Architecture](architecture.md) to understand the relationship between the build host, Linux packages, FDS tools, and cartridges. -4. Use the [Development guide](development.md) for subsequent changes and rebuilds. - -The [Glossary](glossary.md) explains terms such as ABI, sysroot, EROFS, and -masterdir. No prior Void or s6 experience is assumed by the first-build guide. - -## Working with the repository today - -| Guide | Question it answers | +| Guide | What you will learn | | --- | --- | -| [Workstation and emulator](workstation.md) | How do I build software, create a 1+m cartridge image, boot FDS, insert/remove it and write USB? | -| [Software cartridge format](software-format.md) | What goes in metadata, payload partitions and xz tarballs, and how does the guest run them? | -| [Workstation validation](workstation-validation.md) | Which host builds, whole-image writes and public QEMU lifecycle tests actually passed? | -| [Your first build](getting-started.md) | What do I install, what do I run, and how do I know it worked? | -| [M1 root filesystem](rootfs.md) | How do I build the OS filesystem, run its ARM shell, inspect packages, and test the archive? | -| [FDS Rust tooling](tooling.md) | How do I build and run the static FDS command and boot-discovery tools? | -| [Clap migration validation](clap-validation.md) | Which parser, complete-image, VM and startup checks passed after standardizing the Rust command lines? | -| [Internal storage](internal-storage.md) | How do I build the full NVMe disk, persist bay settings and save diagnostics? | -| [Release signatures](releases.md) | How do I create a signing key, sign artifacts and verify a download with a trusted key? | -| [Frozen inputs and offline rebuilds](reproducible-builds.md) | How do I preserve inputs, build without network access, compare results and assemble a local release? | -| [Boot images](boot.md) | How do I create the Pi boot partition, initramfs and SYSTEM cartridge, and test insertion/recovery? | -| [Desktop, PROGRAM and Ethernet](desktop.md) | How do I start WindowMaker, run cartridge programs, and enable networking? | -| [Media tools](media-tools.md) | How are images created, inspected and confirmed before cartridge writes? | -| [Pi 5 EEPROM configuration](eeprom.md) | How do I prepare, review and restore bootloader settings without flashing the build host? | -| [Twelve-bay stress tests](stress-testing.md) | How do I run the twelve-device VM checks and prepare physical tests? | -| [Shutdown and reboot](power.md) | How does shutdown protect DATA, and what should I do when it is blocked? | -| [Writable DATA](data.md) | Where do user files live, how do managed programs work, and when is removal safe? | -| [Implementation through M12](implementation-status.md) | What is complete, pending, and deferred to physical hardware? | -| [M12 software evidence](m12-validation.md) | Which checks, offline comparisons and signed local artifacts establish M12 software acceptance? | -| [Native init and ARM VM](init.md) | How do I boot the system, open its shell, and start or stop a service? | -| [M1 validation](m1-validation.md) | What did the original rootfs milestone verify? | -| [M2 validation](m2-validation.md) | Which full ARM boot and service-control checks passed? | -| [M3 validation](m3-validation.md) | Which static tooling and ARM parsing/discovery checks passed? | -| [M4 software validation](m4-validation.md) | Which image, module, stage0, insertion and recovery tests passed? | -| [M5 software validation](m5-validation.md) | Which ordinary-console, trace, emulation and optimization checks passed? | -| [M6 software validation](m6-validation.md) | Which virtual USB, mapping, metadata, mount, IPC and eject checks passed? | -| [M7 DATA validation](m7-validation.md) | Did sustained writes survive safe eject, reinsertion and independent filesystem checks? | -| [M8 desktop/network validation](m8-validation.md) | Did desktop transitions, PROGRAM execution, fonts and Ethernet work in the ARM VM? | -| [M9 media-tool validation](m9-validation.md) | Did ARM image creation, confirmed cartridge writes, verification and boot of a newly written SYSTEM pass? | -| [M10 shutdown validation](m10-validation.md) | Which ordered halt/reboot, busy refusal, crash and writeback-error tests passed? | -| [M11 stress validation](m11-validation.md) | Which twelve-device and media failure tests have passed, and what remains? | -| [Build host reference](build-host.md) | Which versions, dependencies, environment settings, and checks does the pipeline use? | -| [Development](development.md) | How do I rebuild only the component I changed, configure the build, and prepare a change? | -| [Dasung monitor](dasung.md) | How is the existing monitor daemon built, packaged, configured, and tested for the base system? | -| [Dasung validation](dasung-validation.md) | Which builds, simulator cases, and native s6 checks actually passed? | -| [Packages](packages.md) | What is an XBPS package, and how do I inspect the one I built? | -| [Troubleshooting](troubleshooting.md) | Why did a command fail, and what should I check next? | -| [M0 validation](m0-validation.md) | What has actually been built and verified on the development host? | +| [Overview](../README.md) | How the computer, internal storage and cartridges fit together | +| [Build and start FDS](getting-started.md) | Prepare the OS build workstation, create images and boot the emulator | +| [Software and emulator walkthrough](workstation.md) | Build Void source packages, assemble cartridges, insert them and write USB media | +| [Using cartridges](cartridges.md) | Inspect bays, run applications, resolve command names and eject safely | +| [DATA and files](data.md) | Use persistent storage and manage active DATA | +| [Desktop and control panel](desktop.md) | Open WindowMaker, manage cartridges graphically and customize the theme | +| [Software format](software-format.md) | Write source recipes and understand the installed-program image layout | +| [Internal storage](internal-storage.md) | Install boot/recovery storage and manage machine settings | +| [Recovery](recovery.md) | Inspect failed media and check or repair DATA | +| [Release verification](releases.md) | Verify downloaded images before use | +| [Dasung display](dasung.md) | Inspect and control the built-in Paperlike daemon | +| [Cleanup](cleanup.md) | Reclaim obsolete build output without losing current images or personal data | +| [Troubleshooting](troubleshooting.md) | Diagnose build, cartridge, desktop and emulator problems | -## Understanding the operating system - -These pages explain the implemented behavior and distinguish software evidence -from properties that still need physical hardware tests. - -| Guide | Subject | -| --- | --- | -| [Architecture](architecture.md) | Machine model, two library strategies, host/target split, writable state | -| [Boot](boot.md) | From internal NVMe to a removable SYSTEM and the first console | -| [Cartridges](cartridges.md) | Cartridge classes, physical bays, insertion, ejection, and manifests | -| [Services](services.md) | Native s6, service dependencies, readiness, and shutdown | -| [Performance](performance.md) | Timing targets, measurement boundaries, and regression reporting | -| [Recovery](recovery.md) | Build and enter independent recovery, inspect cartridges, check/repair DATA and prepare replacement SYSTEM media | - -## Planning and reference - -- [Roadmap](roadmap.md): current milestone, remaining milestones, and acceptance criteria. -- [Master plan](master-plan.md): the complete 67-section specification; examples describe the final design unless stated otherwise. -- [Project instructions](../AGENTS.md): implementation scope and conventions. - -A `sh` code block in a current-workflow guide is intended to be run from the -repository root unless the surrounding text says otherwise. Planned runtime -examples use `text` blocks and are labeled as future behavior. Run build commands -sequentially; they share one Void build directory. +Implementation plans, dependency rationale, measurements, hardware procedures and +historical test results are maintained separately in [developer notes](developer/README.md). diff --git a/docs/cartridges.md b/docs/cartridges.md index 16d65c4..6ac5e8f 100644 --- a/docs/cartridges.md +++ b/docs/cartridges.md @@ -1,197 +1,98 @@ -# Using cartridges and configuring bays +# Use cartridges and run programs -[Documentation index](README.md) · [Boot images](boot.md) · [Services](services.md) +A bay number identifies a physical slot, from `01` through `12`. It stays tied +to the slot rather than the order in which Linux discovers USB disks. -M6 adds the base `fds-cartridged` service, twelve bay states, USB hardware -recognition, strict metadata inspection, read-only storage mounts and eject. -Its [ARM VM acceptance checks passed](m6-validation.md). [Writable DATA integration](data.md) has passed M7 software acceptance; -[desktop/program activation](desktop.md) passed M8 and [media creation tools](media-tools.md) -passed M9 software acceptance. [M10 ordered shutdown](power.md) has passed software acceptance. +| Cartridge | Role | +| --- | --- | +| SYSTEM | Supplies the read-only operating system used at boot | +| PROGRAM | Supplies applications and their runtime dependencies | +| DATA | Stores your writable files at `/data` | +| ENVIRONMENT | Requests an installed desktop profile, such as WindowMaker | -## Inspect the current machine +## Inspect a bay -At the FDS console: +Insert the cartridge, then run: ```sh fds bays -fds bay 3 -fds cartridge 3 -fds --json bays +fds bay 01 ``` -The ordinary `fds` user can use these commands. `bays` shows all twelve bays, -including empty and unconfigured ones. `cartridge` includes validated metadata -and the current mount path. `--json` exposes a versioned machine-readable report. -`fds rescan` explicitly refreshes inventory; normal insert/remove events trigger -refresh automatically. The console does not wait for the inventory scan. +USB discovery takes place asynchronously. `fds bay` shows its state, identity, +mount location, running process count and software catalogue. `fds --json bay 01` +returns structured details, including command aliases. `fds rescan` refreshes +the inventory. No application starts merely because a PROGRAM cartridge is inserted. | State | Meaning and next action | | --- | --- | -| UNCONFIGURED | No measured controller/port mapping exists for this bay | -| EMPTY | Configured bay with no detected USB device | -| HARDWARE | USB device without storage; catalog name or VID/PID is shown | -| UNRECOGNIZED STORAGE | Storage exists but no named FDS partition is available | -| MOUNTED READ ONLY | Manifest validated; filesystem available at the displayed path | -| MOUNTED READ WRITE | Active DATA is writable at `/data`; eject before removal | -| PROTECTED | The active SYSTEM root; eject is refused | -| AMBIGUOUS | Multiple devices or named partitions match; nothing is selected arbitrarily | -| ERROR | Inspection, mounting or metadata validation failed; read the diagnostic | -| SAFE | The requested unmount succeeded; remove the cartridge | +| EMPTY | No cartridge is detected | +| MOUNTED READ ONLY | Contents are available; PROGRAM commands can run | +| MOUNTED READ WRITE | DATA is active at `/data` | +| SAFE | Storage is released; remove the cartridge | +| PROTECTED | This cartridge supplies the running SYSTEM; shut down before removal | +| UNCONFIGURED | The physical slot needs a bay mapping | +| ERROR / AMBIGUOUS | Read the detail; correct the media or mapping before use | -An unrecognized storage state may briefly appear while the kernel discovers its -partitions. It is not permission to remove a device being used elsewhere. -Errors and ambiguous media never produce SAFE. An additional mount of the same filesystem also blocks managed eject until -it is unmounted. +## Run a software command -## Calibrate the physical bay map - -The shipped `/etc/fds/bays.toml` is deliberately empty: no Pi bay wiring has yet -been measured. This produces UNCONFIGURED states, not invented assignments. - -1. On the assembled machine, insert one known USB device into one physical bay. -2. Run `fds topology`. This diagnostic explicitly shows controller/port paths; - ordinary bay commands omit those implementation details. -3. Record that path and repeat for each bay. Test both USB 2 and USB 3 devices, - because their companion root hubs can have different logical port paths. -4. Put the measured map in your machine configuration directory and build the - [internal image](internal-storage.md), or use `fds machine install DIRECTORY` - from recovery and reboot. Machine settings persist independently of SYSTEM. - The packaged `/etc/fds/` files are fallback defaults for missing/invalid internal - storage; `fds machine status` shows which source is active. -5. Verify all twelve devices together, in different insertion orders and after - reboot. That physical acceptance remains deferred. - -A **syntax example only**, using identities that must be replaced by observations: - -```toml -[front] -hub = "platform/example-controller:usb2/1" -[front.ports] -1 = 1 -2 = 2 -3 = 3 -4 = 4 -5 = 5 -6 = 6 - -[front_superspeed] -hub = "platform/example-controller:usb3/1" -[front_superspeed.ports] -1 = 1 -2 = 2 -3 = 3 -4 = 4 -5 = 5 -6 = 6 -``` - -`hub` names the controller and parent port chain. Its numbered `ports` map the -next downstream port to a bay number from 1 through 12. Additional groups can -map the rear hub. Explicit USB 2/3 aliases may refer to the same bay; duplicate -ports, overlapping parent/child mappings, and duplicate bays within one hub are -rejected. If two devices simultaneously match aliases for one bay, it becomes -AMBIGUOUS. A hub inserted into a cartridge bay with multiple downstream devices -also needs an explicit future composite-device policy; M6 refuses to guess. - -The identity excludes the Linux USB bus number and `/dev/sdX` enumeration order. -It includes the controller's sysfs path, USB protocol generation and port chain. -Moving a hub to another controller or changing wiring requires recalibration. - -## Storage formats and metadata - -| Class | GPT partition name | Filesystem and current behavior | -| --- | --- | --- | -| SYSTEM | FDS_SYSTEM | EROFS; current root is protected, additional media read-only | -| DATA | FDS_DATA | ext4; inspected read-only first, then activated at `/data` under the M7 policy | -| PROGRAM | FDS_METADATA + FDS_PAYLOAD02… | Metadata and xz bundles in `1+m` EROFS partitions; see [software format](software-format.md) | -| Legacy PROGRAM | FDS_PROGRAM | Existing single EROFS application tree; still readable | -| ENVIRONMENT | FDS_ENVIRONMENT | EROFS; declarative selection of a trusted built-in profile | -| UTILITY | FDS_UTILITY | EROFS; inspection only, no automatic actions | -| HARDWARE | None required | VID/PID, optional serial, device/interface class and bay | - -Storage contains `/FDS/CARTRIDGE.TOML`. For example: - -```toml -format = 1 -[cartridge] -id = "fds.windowmaker" -name = "WINDOW SYSTEM" -class = "environment" -version = "0.1" -[media] -writable = false -[activation] -profile = "windowmaker" -``` - -The manifest class must match the GPT partition name. SYSTEM, PROGRAM and -ENVIRONMENT are read-only; DATA declares writable media during the initial read-only inspection. Unknown keys, root commands, unsafe identifiers, control -characters, symlink metadata and nonregular metadata files are rejected. -The parser accepts at most 64 KiB. No `/FDS/autorun.sh` is run. - -Inspect a standalone manifest with `fds inspect /path/to/CARTRIDGE.TOML`. -For mounted storage, `fds cartridge N` reports the daemon's validated copy. -Media is inspected in a root-only staging directory, then published at -`/run/fds/media/NN` after validation, with nodev, nosuid and noexec. Device -major/minor and kernel disk sequence are checked before mounting an opened -block-device descriptor, preventing stale enumeration names from selecting a -replacement disk. - -## Eject a mounted cartridge +For the included hello/report cartridge: ```sh -fds eject 2 +hello 'Hello from my cartridge' +report +fds run 01 -- demo.hello:hello 'Hello from my cartridge' ``` -Leave any shell working directory inside that cartridge and close files first. -SAFE is emitted only after an ordinary unmount succeeds. Busy mounts return an -error; forced or lazy unmount is never used to declare safe removal. Once SAFE, -the daemon keeps that insertion unmounted until removal and reinsertion, including -across a service restart. Its root-owned volatile marker is tied to the kernel -disk sequence, so a replacement disk is not mistaken for ejected media. -The active SYSTEM cannot be ejected. Swap SYSTEM only after shutting down. +Direct commands run in the foreground with your terminal, input/output streams, +arguments, working directory and exit code. Pipes and redirection work normally. +`fds run` starts a managed background program and prints its PID; its output is +written to `/run/log/cartridged/current`. Both forms run as the ordinary FDS user +and are tracked for cartridge removal. -Pulling a mounted cartridge without eject is surprise removal. The daemon clears -its inventory and detaches a vanished read-only mount where needed; that cleanup -is not a successful eject. [M7 DATA handling](data.md) extends ejection to writable DATA, consumer -tracking and syncfs. The temporary `/home/fds` remains usable without DATA and is lost at -power-off. +New cartridges run directly from read-only payload partitions. They require no +program extraction or compilation on FDS. The stable `/run/fds/bin` directory is +already in the console and terminal PATH, so existing shells see inserted commands. -## Hardware recognition +If cartridges export the same command name, the lowest numbered bay wins. +Within a bay, the first software selector in lexical order wins. System commands +appear earlier in PATH and retain their usual meanings. To select a specific +cartridge command, use its qualified alias: -Edit `packages/fds-cartridged/files/hardware-catalog.toml`, then rebuild SYSTEM. -Entries name devices, never executable actions: - -```toml -[[device]] -name = "MY SERIAL ADAPTER" -vendor = "1234" -product = "5678" -# Optional exact restrictions: -serial = "UNIT-1" -class = "02" +```sh +b01:demo.hello:hello 'Explicit bay and software' +fds run 01 -- demo.hello:hello ``` -Replace these example IDs with observed values. Device or interface class may -match `class`. Multiple matching catalog entries produce an error; unknown -hardware remains identified by VID/PID without being mistaken for empty media. -Dasung control remains in the independent base monitor service. +Legacy PROGRAM media uses `b01:COMMAND` for a qualified direct alias. Commands +are removed on eject/unplug; a shared name falls back to the next available bay. +After the final cartridge is removed, a shell may remember the old executable +path; `hash -r` clears Bash's command cache. -## Implementation and diagnostics +## Eject safely -The daemon subscribes to kernel USB/block events before taking its first snapshot. -A receive-buffer overflow causes a fresh snapshot. IPC uses a bounded Unix socket -at `/run/fds/control.sock`, mode 0660 and group `fds`, with peer UID checks for -root and the ordinary FDS user. Slow clients have individual deadlines and do not -block other clients. No asynchronous runtime or new external Rust dependency is -introduced: the existing std, libc, serde, serde_json and toml components suffice. +```sh +cd "$HOME" +fds eject 01 +fds bay 01 +``` -Root diagnostics: `s6-svstat /run/service/cartridged` and -`cat /run/log/cartridged/current`. Logs are bounded and volatile. Restart through -`s6-rc -l /run/s6-rc -d change cartridged`, then the corresponding `-u` command. -Close users of cartridge mounts first; restart cleanup refuses busy leftovers. +Eject stops managed programs, flushes writable storage and releases its mounts. +Remove the cartridge after **SAFE** is reported. If eject fails, close files, +shells or extra mounts using that cartridge and retry. Never treat a timeout as +permission to pull writable media. Use `fds poweroff` to stop the whole computer; +swap the running SYSTEM only after shutdown. -On the build host, `make cartridge-test` runs fixtures and full virtual USB tests. -These exercise actual kernel events and mounts but cannot establish physical -wiring, USB power stability, or Pi port behavior. M11 expands to twelve-device -stress, and physical calibration must be recorded separately. +The [control panel](desktop.md) provides the same safe-eject action. For writable +storage behavior, see [DATA and files](data.md). + +## Configure physical bay numbering + +Run `fds topology` with one identifiable USB device inserted in each slot in +turn. Record the controller/hub topology for both USB 2 and USB 3 connections, +then map those stable identities in the machine's `bays.toml`. Do not map +`/dev/sda` names or enumeration order. The emulator supplies its own known map. + +Use [machine settings](internal-storage.md) to validate, export and install the +configuration. The detailed wiring/calibration procedure is in the +[hardware engineering reference](developer/cartridges.md). diff --git a/docs/cleanup.md b/docs/cleanup.md new file mode 100644 index 0000000..42affff --- /dev/null +++ b/docs/cleanup.md @@ -0,0 +1,90 @@ +# Reclaim build space + +Run these from the repository root after stopping builds and test VMs: + +```sh +make clean-preview +make clean +``` + +The preview lists every selected directory, its combined allocated footprint, +and the generated directories being kept. It does not delete files. `make clean` +recalculates the selection and deletes it without a confirmation prompt. Repeat +it whenever old build/test workspaces accumulate. It needs Python 3, Git and GNU +`du`, already used by the build host; it needs no Rust compilation, downloads, +root privileges or new Python packages. + +## What is removed + +- Old generated rootfs, kernel, initramfs, SYSTEM, boot, recovery and internal + image workspaces that are no longer referenced by published output links. +- Old generated integration-test directories, including their disk images, + disposable test DATA overlays and local diagnostic logs. The current results + referenced by `*-latest` links or workstation current-pointer files are kept. +- Old generated package staging copies and service compilation directories. +- This checkout's `target/` Rust compilation output. Exported executables in + `out/` and `out/workstation/` remain available. The next Rust build recompiles + as needed using the retained Cargo source cache. + +The command recognizes the specific temporary-directory names produced by the +repository's builders and tests. It never treats all of `out/` as disposable. +Unknown names are left alone, including ad hoc diagnostic experiments. + +## What is kept + +- Current rootfs archives for every profile, kernel, initramfs, boot, recovery, + internal disk and SYSTEM images, through their published symlinks. Links in + `out/manifests/` and links within retained workspaces also retain their targets. +- Latest published test runs, the current workstation image/emulator fixtures, + the current Dasung s6 database, and the newest `m9-images.*` fixture needed by + the media tests. These can still occupy tens of GiB. +- Signed releases such as `out/fds-os-0.1.0/`, input snapshots such as + `out/inputs-m12-v5/`, and independent restored trees such as + `out/rebuild-m12-v5-a/`. These remain available for release reproduction. +- User-created cartridge images, installed software trees and personal emulator sessions + such as `out/my-emulator/`, including their persistent DATA overlays. +- `out/logs/`, `out/manifests/`, packages, downloads, `out/cache/`, `.host/`, the + Void checkout/build container, and Cargo/Rustup caches outside this checkout. +- Git-tracked files and any recognized workspace containing a `.fds-keep` entry. + +To retain an older generated workspace for investigation, put a marker in it +before cleaning: + +```sh +touch out/m8-vm.YOUR_RUN/.fds-keep +make clean-preview +``` + +Replace `YOUR_RUN` with the actual directory suffix. Remove that marker when the +workspace is disposable again. Reserve the generated names for the build/test +tools; store personal sessions and images under your own names. + +## Build-space management + +Image builds and VM suites retain workspaces for diagnosis and preserve the last +successful published image. Repeat `make clean-preview` and `make clean` after +completed build batches. Cleanup is explicit; there is no automatic retention +limit during a build. Full image builds require tens of GiB per independent tree. + +## Running safely and rebuilding + +Use cleanup sequentially, just like the existing shared-container builds. It +holds the existing rootfs/base-package/image-tool locks, excludes another +cleanup, and refuses mounted candidate paths or observable active build/VM +processes. These checks are safeguards, not permission to start a separate build +while cleanup is running. It never follows a candidate symlink into another +directory, and refuses a symlinked `out/` entirely. Only the checkout containing +the script is eligible; there is no arbitrary deletion-path option. + +Existing published images and native tools remain usable after cleanup. To +refresh exported binaries, run `make tooling` or `make workstation`. To rebuild +the smoke-test artifacts and rerun build checks: + +```sh +make smoke-test +make check +``` + +After cleanup, run the smoke test before `make check`, since the checks need its +rebuilt executable. Cleanup implementation and validation details are in the +[developer notes](developer/cleanup.md). diff --git a/docs/dasung.md b/docs/dasung.md index d8de682..1ae68a4 100644 --- a/docs/dasung.md +++ b/docs/dasung.md @@ -1,185 +1,47 @@ -# Dasung Paperlike 13K base-system integration +# Dasung Paperlike display -[Documentation index](README.md) · [First build](getting-started.md) · [Services](services.md) +FDS includes `dasungd` for the configured Dasung Paperlike 13K grayscale monitor. +The controller starts during early boot and remains supervised in the base system, +including console-only use. No GUI or ENVIRONMENT cartridge is required. -FDS includes the Rust **dasungd** controller from the project's earlier -**Fix Dasung monitor black screen** task. It targets the user's **Paperlike 13K -grayscale**, serial `L56051794302`, with the confirmed 3200 × 2400 timing at about -37 Hz. This is core display support: its package is required for the base SYSTEM -image, and its service belongs to the base boot bundle, including console-only use. -It does not require an ENVIRONMENT cartridge, WindowMaker, or a graphical session. +The supplied profile uses the dedicated monitor's EDID and USB companion identity, +with raw mode 1 and contrast 4 as startup defaults. It is specific to that monitor; +do not use its identity as a blanket match for other CH340 USB devices. -**Implementation boundary:** the daemon, ARM package, and s6 service definitions -are included by the [rootfs assembler](rootfs.md). [M2 native init](init.md) starts -the controller in the base boot graph, including in the ARM VM with no monitor. -The [M4 boot path](boot.md) starts an early copy before SYSTEM exists, then stops -and reaps it before base s6 starts the normal daemon. Software handoff is tested -in a VM; physical Pi display and monitor recovery remain unverified. The final -s6 database is compiled at image build time. +## Inspect the controller -## Build it on the workstation - -Complete `make bootstrap` as described in [Your first build](getting-started.md), -then run from the repository root, as your regular user: +The controller's socket is restricted to root. From a root maintenance console +on FDS: ```sh -make dasung -make dasung-test -``` - -The first command prepares the additional C cross compiler inside the existing -Void build container, fetches the crates pinned in `Cargo.lock`, and cross-builds -static-musl `dasungd`. It verifies the ELF, builds the `fds-dasungd` XBPS overlay, -indexes the output, and compiles the actual s6 source into a validation database. -The second runs unit tests and a simulated serial monitor, then inspects the -package and compiles the service definitions extracted from that package. It also -runs that s6 graph with a native test binary, a read-only container root, and no -physical device access to check startup, logging, permissions, restart, and stop. - -Python 3 is needed **only on the host for the simulator**. If absent on Arch, -install `python` with pacman. No Python interpreter is shipped for the daemon. -The first additional toolchain download is approximately 243 MB and needs roughly -1 GB of extra container space. Builds still depend on rolling Void binary inputs. -Run these commands sequentially with other FDS builds; they share the container. - -| Output | Purpose | -| --- | --- | -| `out/dasungd` | Static AArch64 executable; no target glibc, musl, libusb, or libudev shared library dependency | -| `out/packages/fds-dasungd-0.1.0_1.aarch64.xbps` | Target package with executable, config, exact EDID, s6 source, udev rule, and licenses | -| `out/manifests/dasung-s6-database.txt` | Path to the service database compiled on the host for validation | -| `out/manifests/dasung-artifacts.sha256` | Executable and package hashes | -| `out/manifests/dasung-build-packages.txt` | Installed build-container input versions | -| `out/logs/dasung-build.log`, `dasung-checks.log` | Build and test evidence | - -The standalone validation database contains only the Dasung subset. The M2 -image assembler merges it with the init package and compiles the full current -graph while building the rootfs. No daemon is started on the workstation by these targets; -the integration test uses its own temporary socket and pseudo-terminal. - -## What goes into the base system - -`image/base-packages.list` makes `fds-dasungd` a mandatory hardware package. -`packages/fds-dasungd/template` is the active FDS-owned XBPS overlay. The upstream -Void tree remains pinned and unchanged. Source is maintained at `rust/dasungd/`; -the original task directory is not required for future builds. - -The package installs: - -- `/usr/bin/dasungd` and `/etc/dasungd.toml`. -- `/usr/lib/firmware/edid/dasung-paperlike13k-37hz.bin`. -- `/etc/s6-rc/source/dasungd`, its runtime-directory and log services, and the - `boot/contents.d/dasungd` membership entry. -- `/usr/lib/udev/rules.d/99-dasung-spi.rules`, adapted to eudev syntax. -- Documentation and the controller/libusb licenses. - -s6 supervises the foreground daemon. A short oneshot creates private writable -runtime/log directories; it does not wait for the display. The daemon starts with -no monitor attached and retries discovery. Its initial service transition does -not wait for a monitor reply, a desktop, the network, or all cartridge bays. -The log service rotates a small log under `/run/log/dasungd`. - -The init package adds a dependency on `runtime-fs` so the controller starts with -its kernel interfaces mounted. The service runs as root because it claims USB interfaces. Its socket -is under a root-owned mode-0750 directory; runtime control is restricted to root. -A future FDS CLI can expose selected operations through an explicit policy. -No world-writable USB rule, systemd unit, or runit service is installed. - -## Confirmed profile and Pi-specific choices - -| Property | Recorded value | -| --- | --- | -| Monitor | Paperlike 13K grayscale; `L56051794302` | -| Resolution | 3200 × 2400 | -| Pixel clock | **304210 kHz** (304.21 MHz) | -| Horizontal active / sync start / sync end / total | 3200 / 3248 / 3280 / 3360 | -| Vertical active / sync start / sync end / total | 2400 / 2423 / 2427 / 2447 | -| Sync polarities | Positive horizontal and vertical | -| EDID SHA-256 | `b6c1e0a8d315d9cf5c2fb83784a177530bcc085c2d0724dc7478a9c12449e4f4` | -| Control transport | USB UART `1a86:7523`, with companion SPI bridge `1a86:5512` | -| Startup parameters | Raw mode 1 and contrast 4, previously read back on this unit | -| Keepalive | `20 01` every two seconds; minimum 150 ms between command writes | - -The video timing was confirmed on the original AMD workstation, **not on a Pi**. -The original USB-C video connection is not a Pi cabling prescription. Pi connector, -cable/adapter, mode acceptance, and picture stability require hardware validation. -Desktop 2× scaling is separate from EDID timing; a console uses its own font size. - -FDS configures `display.enabled=false` and `hotplug="none"`: the daemon handles -USB control without applying the source project's AMD debugfs disconnect/reconnect -procedure. The Pi boot/display integration must supply the appropriate KMS/firmware -EDID configuration for its real connector. The Pi boot-volume builder now supplies -an unqualified firmware EDID override for this dedicated monitor; it does not -assume `DP-2` or a numbered HDMI connector. See the [boot guide](boot.md) for -the exact command line and the remaining physical mode test. - -The daemon requires the configured monitor's DRM EDID and the matching USB -companion topology before claiming a device. Multiple matching UARTs are rejected; -`usb_path` can disambiguate a verified Pi topology. Do not substitute a blanket -match for all CH340 adapters or copy this unit's identity to a different monitor. - -The original monitor's SPI driver interaction caused a dark picture after a power -cycle. `image/kernel/dasung.config` requires `CONFIG_SPI_CH341=n`; the Pi kernel -builder enforces this before boot, including the initramfs environment. The scoped -udev rule is secondary protection: it cannot prevent a built-in or already-loaded -SPI driver from probing. The daemon also reserves the companion interface without -sending SPI data and refuses to take it from an already-bound kernel driver. - -## Runtime usage once installed in an FDS image - -These are target commands, **not commands to run against the workstation's live -monitor during the build**: - -```text dasungd status dasungd query +``` + +`status` reports whether the monitor is connected and responsive, the age of its +last reply and cached parameters. `query` requests current values. To refresh +the display or adjust contrast: + +```sh dasungd refresh dasungd set contrast 4 dasungd set mode 1 --save dasungd forget mode ``` -`status` returns JSON including `connected`, `responsive`, reply age, and cached -parameters. A successful set means a packet was sent; use query/status to check -observed values. It is not proof of physical image quality. Color-model mode names -are not assigned to this grayscale monitor's raw numeric modes. +A successful set means the command was sent; use query/status to read back the +result. The grayscale model uses raw numeric modes. Saved overrides in +`/run/dasungd/settings.json` survive a daemon restart during the same boot, but +not power loss. Image configuration defaults are in `/etc/dasungd.toml`. -The default socket is `/run/dasungd/control.sock`. External programs can use its -newline-delimited JSON protocol; the daemon remains the only USB owner: +## Diagnose a connection problem -```json -{"op":"status"} -{"op":"set","parameter":"contrast","value":4,"save":true} -``` +Inspect `/run/log/dasungd/current` and the reported status. Discovery requires +the configured DRM EDID and matching USB companion topology. An ambiguous match +is rejected. Set `usb_path` only after identifying the intended device topology. -In FDS's current payload, saved overrides live in `/run/dasungd/settings.json`. -They survive a daemon restart **within the same boot**, but not power loss. -Mode 1/contrast 4 are persistent image configuration defaults. Once persistent -machine-state mounts are implemented, point `state_file` at that internal writable -storage to retain user overrides across boots. Do not put it on read-only SYSTEM -or make the base display daemon depend on a removable DATA cartridge. - -## Dependencies and source provenance - -The imported Rust dependencies are retained and pinned by the workspace lockfile: -`rusb` wraps USB access; `libc` supports Linux serial/locking calls; `ctrlc` handles -termination; `clap` provides CLI parsing; `serde`, `serde_json`, and `toml` support -configuration and the local socket; `anyhow` supplies contextual errors. -Transitive versions are recorded in `Cargo.lock`. - -The `vendored` feature builds libusb from the pinned `libusb1-sys` crate. The Void -`cross-aarch64-linux-musl` toolchain supplies C headers/compiler for that library; -Rust's bundled musl target alone was enough for the old pure-Rust smoke-test, but -is insufficient for this C dependency. `LIBUSB_NO_PKG_CONFIG` and -`LIBUDEV_NO_PKG_CONFIG` prevent accidental host/shared-library linkage. libusb uses -its Linux netlink backend. No libusb/libudev shared runtime package is needed. - -The target package depends on GNU coreutils for directory preparation, execline -for service launch, s6/s6-rc for supervision, and eudev for device rules. These -match the planned base userspace. The container's s6-rc is a build-time compiler, -not a target PID 1 running on Arch. - -[Import provenance](../rust/dasungd/IMPORT.md) records the original source hashes. -[Original validation](../rust/dasungd/VALIDATION.md) records the live handoff and -remaining physical test. Protocol and reconnect tests use simulated hardware; -Pi boot, image quality, physical disconnect/reconnect, and corrected cold-power -recovery must still be verified on the actual target. No boot benchmark is claimed. +FDS uses the daemon for USB control. Its configuration disables the original +workstation-specific display hotplug procedure. Video mode and cabling are part +of the machine's boot/display configuration. The detailed protocol, build +rationale and physical display procedures are in the +[developer reference](developer/dasung.md). diff --git a/docs/data.md b/docs/data.md index ef2fbb1..5daa167 100644 --- a/docs/data.md +++ b/docs/data.md @@ -1,138 +1,52 @@ -# Writable DATA and managed programs +# DATA and persistent files -[Documentation index](README.md) · [Cartridges](cartridges.md) · [Services](services.md) - -M7 implements writable DATA. Its sustained-write, safe-eject and surrounding -image regression checks [passed in the ARM VM](m7-validation.md). A DATA -cartridge uses a GPT partition named `FDS_DATA`, ext4, and a validated -`/FDS/CARTRIDGE.TOML` declaring class `data` and `writable = true`. -Use the [M9 media tools](media-tools.md) to create new DATA media. The virtual -tests construct disposable image files and never format a workstation disk. - -## Where your files live - -`/home/fds` remains a temporary home, whether DATA is present or absent. Its -contents disappear at power-off. Persistent user files belong under `/data`. -Separating these paths lets you eject DATA without hiding the active shell's -home directory or substituting another cartridge underneath it. - -After inserting a DATA cartridge into a configured bay: +FDS keeps SYSTEM read-only. Store documents, source code and other persistent +files on a DATA cartridge. With one healthy DATA cartridge inserted, FDS mounts +it at `/data` and gives the `fds` user access to its files. ```sh fds bays -fds cartridge 2 -ls /data +fds bay 02 +mkdir -p /data/projects +printf 'My first FDS file\n' >/data/projects/hello.txt +cat /data/projects/hello.txt ``` -If exactly one valid DATA candidate is available and none is active, the daemon -activates it at `/data`. The filesystem must grant UID/GID 1000 the intended -write permissions; insertion does not recursively change ownership of user files. -M9's DATA formatter supplies that ownership for fresh media. - -An active DATA session stays attached to its current cartridge when another is -inserted. If a discovery snapshot contains multiple candidates with none active, -they remain read-only until you select one: +Confirm that the bay reports `MOUNTED READ WRITE` before writing. Only one DATA +cartridge is active at a time. To select DATA explicitly: ```sh -fds data use 4 +fds data use 02 ``` -Eject the existing active DATA before selecting a replacement. The daemon never -changes `/data` underneath an active session. EMPTY, ERROR and SAFE states retain -the meanings in the [cartridge guide](cartridges.md); MOUNTED READ WRITE means -that the displayed DATA filesystem is available for user writes. +If another DATA is active, eject it first. In recovery, DATA starts read-only and +requires this explicit activation before it can be used for ordinary writes. -## Run a program that participates in eject +## Programs using DATA -Ordinary console commands work as usual. They are not automatically killed when -you request eject: an open file or working directory may make eject fail as busy. -For a background task that should stop when its DATA is ejected, use: +Run a system command under the DATA bay's process tracking when it should stop +as part of ejecting that DATA: ```sh -fds run 2 -- /usr/bin/bash -c 'date > /data/managed-example.txt' +fds run 02 -- /usr/bin/tail -f /data/application.log ``` -This starts a **background** managed program and prints its process ID. It has no -interactive input. Output goes to the bounded, volatile cartridge service log: +This background command uses `/data` as its working directory. Its output goes +to the cartridge service log. A directly launched PROGRAM is tracked under its +PROGRAM bay; close it if it keeps files open on a different DATA cartridge. -```sh -tail /run/log/cartridged/current -fds cartridge 2 -``` - -The executable path must be absolute. The child starts with `/data` as its working -directory, the ordinary FDS UID/GID, no supplementary groups, no effective -capabilities, a small explicit environment, and `no_new_privs`. Programs cannot -gain privileges through setuid executables. The limit is 256 managed processes -per bay. This interface accepts healthy writable DATA; the [desktop guide](desktop.md) -also explains M8's explicit PROGRAM-media launch interface. - -A root-owned Linux cgroup tracks the program and its descendants, including -children that outlive their parent. Cgroups are kernel process tracking, not a -new init system or runtime package. The configured kernel already includes the -required cgroup v2 and process-limit support. The daemon sets up its hierarchy -independently of the console. See the [kernel cgroup interface](https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html). - -## Eject and remove DATA +## Finish a session ```sh cd "$HOME" -fds eject 2 +fds eject 02 ``` -Eject checks for additional mounts, stops managed consumers, flushes the DATA -filesystem with `syncfs`, and performs a normal unmount. Only successful completion -produces SAFE. A retained filesystem descriptor observes writeback errors during -the session. M10 additionally makes the filesystem read-only and checks writeback -again before closing that descriptor for unmount, preventing an error-reporting -gap when a busy mount needs a later retry. -Linux's [syncfs interface](https://man7.org/linux/man-pages/man2/syncfs.2.html) -reports filesystem errors that must not be ignored. +Wait for SAFE, then remove the cartridge. `fds poweroff` performs the storage +shutdown sequence for all active bays. Unexpected power loss or pulling a drive +during a write can damage files; [recovery](recovery.md) explains checking DATA. -Managed programs receive TERM first. The daemon waits on actual cgroup exit -notifications; programs that have not exited after a one-second deadline are -killed as a group, including newly forked descendants. A further bounded exit -check must succeed. These are failure deadlines, not delays imposed on a program -that has already stopped. Requesting eject therefore ends those background jobs; -it does not promise that unfinished application work is completed. - -An unmanaged busy shell/file, extra mount, failed flush or failed unmount prevents -SAFE. Close the reported use and retry. A recorded DATA writeback fault requires -investigation or recovery; a later successful call is not used to erase it and -pretend the failed write succeeded. The daemon never uses lazy/forced unmount to -claim successful eject. A successfully ejected insertion remains unmounted across -a daemon restart until removal/reinsertion. - -M10 records writable sessions and faults outside the daemon process. If it crashes -before verified unmount, the same insertion stays quarantined and read-only on -restart. A later flush cannot reconstruct the lost error history. See -[Shutdown and recovery records](power.md#data-recovery-records) for the current -implementation and its acceptance status. - -Pulling DATA during writes is an error. The daemon stops its managed consumers, -cleans up the vanished mount and logs that writes may have been lost. This path -never produces SAFE and makes no filesystem-cleanliness promise. Use recovery -inspection/repair before trusting media that was removed during I/O. - -## Build and verify on the workstation - -After changing DATA code or image configuration, build fresh inputs in order: - -```sh -make rootfs PROFILE=cli -make initramfs -make system-card -make data-test -``` - -The test boots disposable virtual media, writes continuously, verifies ordinary -user privileges and descendant shutdown, checks busy and additional mounts, -reopens persisted data, tests multiple candidates and simulates surprise removal. -After safe eject it extracts the ext4 partition into an ordinary file, runs -read-only `e2fsck -fn`, and compares the sustained-write payload byte-for-byte. -No host mount or physical USB device is used. - -Run `make cartridge-test`, `make init-test`, `make boot-test` and -`make console-test` for the surrounding regressions. Test logs and VM evidence -are written under `out/`. Real flash-controller caches, battery loss, USB power -and physical Pi eject/shutdown latency remain hardware acceptance work. +In the emulator, writes go to a separate session overlay. Eject preserves that +overlay, and reinserting the original image creates a new overlay. See +[exporting emulator DATA](workstation.md#save-emulator-data) before moving or +deleting a session. diff --git a/docs/desktop.md b/docs/desktop.md index 2e7bac6..769315c 100644 --- a/docs/desktop.md +++ b/docs/desktop.md @@ -1,212 +1,67 @@ -# Desktop, programs, and networking +# WindowMaker and FDS Control -[Documentation index](README.md) · [Cartridge guide](cartridges.md) · [DATA guide](data.md) +WindowMaker provides the FDS desktop. Its default appearance is grayscale on +all displays: white backgrounds, black borders and selected titles, gray widgets, +Terminus text, and no animations. No E-Ink monitor detection or special cartridge +is needed to select this theme. -M8 software checks passed in the ARM virtual machine. The [validation report](m8-validation.md) -records desktop, program and networking evidence. Raspberry Pi video output and -Dasung display quality remain deferred physical checks. +## Open the desktop -## What starts when you boot - -FDS opens its ordinary-user console first. Xorg, WindowMaker, and the DHCP client -are installed in SYSTEM but are absent from the boot bundle. Dasung control -remains in the base boot bundle, including when no desktop is running. - -The default graphical backend is Xorg on virtual terminal 2, using its built-in -modesetting driver. The physical keyboard and pointer use libinput. Console -access remains on the original terminal. There is no display manager. Physical -VC4 output, monitor resolution, and E-Ink appearance still require Pi testing. - -## Start and stop the desktop - -At `FDS>`: +At the FDS console: ```sh -fds profiles fds profile activate windowmaker fds profiles ``` -Activation is asynchronous. The second status command reports the selected -profile and an activation-to-ready duration once WindowMaker is ready. This -measurement starts at the activation request, so time spent deciding when to -start the desktop is excluded. `fds boot-profile` also records the first desktop -readiness event for the boot. Neither measure includes firmware or power-on. +FDS opens a terminal and **FDS Control**. You can reopen the panel from the +WindowMaker root menu's **FDS Control** item, or run `fds-control` in a terminal. +An ENVIRONMENT cartridge requesting the `windowmaker` profile can also start +the desktop. The Dasung controller runs independently of this session. -The desktop opens an FDS terminal. Right-click its white background to open the -menu, launch another terminal, enable/disable Ethernet, or return to the console. -To stop it from either a terminal or the console: +## Manage cartridges in the panel + +![FDS Control with a PROGRAM cartridge and its terminal output](images/fds-control.png) + +1. Select a bay in the left column. The right column shows its identity, state, + running process count and available software commands. +2. Select a command, then choose **Run in terminal**. The terminal stays open + after the command exits so you can read its output; close it when finished. +3. Choose **Safe eject** to stop that bay's programs and release its storage. + Wait for **SAFE** before removing the cartridge. +4. Use **Rescan** to refresh discovery. The panel also checks for changes + automatically and updates the display when the visible state changes. + +For an inactive DATA cartridge, the primary button becomes **Use DATA**. Activate +it there to make it available at `/data`; eject another active DATA first. + +Errors appear in the status area. Actions are unavailable when their prerequisites +are missing, such as running an application from an empty bay or ejecting the +active SYSTEM. The panel uses the same cartridge service as `fds`. + +Keyboard controls: **Tab** moves focus between bays, programs and buttons; +**Up/Down** changes the selected bay or program; **Enter** activates the focused +control; **R** rescans; **Esc** closes the panel. Closing the panel does not eject cartridges or close +application terminals; use Safe eject when you want to release a cartridge. + +## Return to the console + +Choose **Return to console** in the root menu, or run: ```sh fds profile deactivate ``` -Stopping the desktop closes its terminal and all processes descended from its -session. Save work first. Files in `/home/fds` survive desktop restarts during the -same boot; the home directory is still temporary and disappears at shutdown. -Use [DATA](data.md) for persistent files. Removing an active DATA cartridge also -stops the desktop before flushing DATA, because desktop applications may use it. +This closes the desktop session. It does not shut down FDS. Use `fds poweroff` +when you want to turn the computer off. -`fds-profile status`, `fds-profile activate windowmaker`, and -`fds-profile deactivate` provide the standalone static helper interface. +## Customize the appearance -## ENVIRONMENT cartridges +System defaults are installed in `/etc/WindowMaker/`. Your files in +`$HOME/GNUstep/Defaults/` override those defaults. The session copies the FDS +settings into a new home once and preserves existing preferences. Use +WindowMaker's settings tools or edit those preference files to change fonts, +colors and behavior. Plain `wmaker` also reads the FDS system defaults. -An ENVIRONMENT cartridge is a read-only EROFS partition named `FDS_ENVIRONMENT`. -Its `FDS/CARTRIDGE.TOML` uses the [documented manifest format](cartridges.md), with: - -```toml -[activation] -profile = "windowmaker" -``` - -When exactly one eligible ENVIRONMENT cartridge is present, the daemon requests -WindowMaker after console readiness. The graphical runtime comes from SYSTEM; -the cartridge provides a declarative request, never a privileged executable. -Unknown profile names do not execute anything. If several eligible cartridges -arrive together, select a profile explicitly instead of relying on bay ordering. - -Eject its bay with `fds eject N`. The desktop stops before SAFE is reported. -Surprise removal also stops its associated desktop. Manually deactivating while -the cartridge remains inserted suppresses automatic reactivation until removal -and reinsertion. A manually started desktop is independent of ENVIRONMENT media. -A cartridge-service restart reevaluates mounted media and retires old optional -sessions; this is a maintenance operation, not a way to preserve desktop jobs. - -## E-Ink defaults - -`fds-eink` installs the WindowMaker policy and Terminus fonts: a white background, -black text and borders, grayscale controls, outline movement/resizing, no -animations, no blinking decorations, no app-icon bounce, no dock, no compositor, -and no periodically updating clock. X11 compositing is disabled. The terminal -uses black on white and a nonblinking cursor. - -Defaults are copied into the ordinary user's `~/GNUstep/Defaults` at the first -session. Later sessions preserve edits there. They are temporary unless you -explicitly store a copy on DATA. The authoritative image defaults live in -`/usr/share/fds/eink`; edit their package sources and rebuild SYSTEM to change the -machine-wide defaults. - -Font caches and X11 font indexes are generated while building the image. -An active X server's keyboard map is session state, generated only on desktop -activation. That is separate from global boot-time cache generation. - -## Launch a software cartridge - -Build current software cartridges on a [Linux workstation](workstation.md). They -contain metadata plus xz software bundles in payload partitions. In FDS: - -```sh -fds bay 4 -fds run 4 -- demo.hello:hello -fds eject 4 -``` - -Use the software id and command listed by `fds bay`. The guest verifies and -extracts the bundle into a read-only temporary cache, then starts it as UID 1000. -There are no guest software build hooks. - -### Legacy PROGRAM compatibility - -Existing PROGRAM media contains a read-only EROFS partition named `FDS_PROGRAM`, metadata -in `FDS/CARTRIDGE.TOML`, and its self-contained application files: - -```text -app/bin/editor -app/lib/ -app/share/ -``` - -FDS validates and mounts it at `/run/fds/apps/`. Insertion does not -run anything, and the mount initially disallows execution. To launch an executable -named `editor` from bay 4: - -```sh -fds cartridge 4 -fds run 4 -- editor -``` - -Use a simple executable name from `app/bin`, followed by that program's arguments. -The daemon enables read-only execution and starts it as UID 1000, with no added -privileges. Executable paths must remain within the cartridge's `app` directory. -Programs receive `FDS_APP`, a PATH including `app/bin`, `LD_LIBRARY_PATH` including -`app/lib`, and `XDG_DATA_DIRS` including `app/share`. Bundled libraries can also use -an executable-relative RPATH. FDS does not resolve dependencies across cartridges. - -Programs are background jobs. Graphical programs can use the active authenticated -X display. Standard output/error goes to the bounded, temporary cartridge log: - -```sh -tail /run/log/cartridged/current -fds bay 4 -fds eject 4 -``` - -The bay status reports managed processes. Eject stops the whole process group, -including forked descendants, before unmounting. The existing DATA form remains -`fds run N -- /absolute/system/command arguments`, using `/data` as its working -directory. DATA itself remains mounted with execution disabled. - -## Ethernet on demand - -External networking is off until explicitly requested or a mapped USB Ethernet cartridge -produces a real network interface. Interfaces are associated with their USB -ancestors; a descriptive label alone does not start networking. Wi-Fi setup is -not implemented by this Ethernet policy. The local loopback interface is always -up for applications on the same machine; it does not enable external traffic. - -```sh -fds network on -fds profiles -ip -brief address -ip route -fds network off -``` - -Enabling networking starts DHCP without waiting for a lease. An absent server -cannot delay `FDS>`. Explicit activation selects available Ethernet interfaces; -automatic activation selects interfaces under mapped cartridge USB devices. -Stopping networking stops DHCP, brings the managed interfaces down, and clears -temporary DNS settings. This initial profile uses IPv4 DHCP; it disables kernel -IPv6 autoconfiguration on those interfaces. After an explicit stop, an inserted Ethernet -cartridge remains suppressed until it is removed, or you enable networking again. - -Leases, DNS settings, and logs live under `/run`; nothing makes SYSTEM writable. -`/etc/resolv.conf` points to `/run/fds/resolv.conf`. Inspect -`/run/log/network/current` when an address is missing. - -## Development and troubleshooting - -The production console image includes the real desktop stack. The development -image adds Xvfb and X11 diagnostics for a virtual display without a connected GPU: - -```sh -make rootfs PROFILE=development -make desktop-test -``` - -The test creates its own disposable image selecting Xvfb; the normal development -image still selects Xorg. Never mistake an Xvfb pass for Pi graphics verification. -Changing the SYSTEM image profile requires building that rootfs first; the image -builder rejects a profile label that disagrees with its embedded identity. - -If desktop activation fails, `fds profiles` reports it. Read -`/run/log/xserver/current` and `/run/log/desktop/current`, then deactivate before -retrying. Xorg's detailed log is `/run/log/xserver/Xorg.0.log`. X access requires -an authority cookie readable only by root and the FDS group; do not use `xhost +` -or disable access control as a workaround. - -Dependency rationale: Xorg supplies the display server and modesetting driver; -libinput supplies keyboard/pointer input; WindowMaker supplies window management; -Terminus and fontconfig supply the typography; xterm supplies a terminal; the static FDS helper -observes X11 property events for readiness without polling delays; xset/xsetroot apply the -static session settings. Xvfb, xdotool, xwd, xwininfo, xdpyinfo, xprop, and xauth are -optional development diagnostics. `fds-dhcpcd` builds the pinned upstream DHCP -client with privilege separation and volatile state paths, omitting its runit -service files. All FDS control helpers remain static Rust/musl binaries. - -The readiness observer uses the [X11 core protocol](https://xorg.freedesktop.org/archive/X11R7.7/doc/xproto/x11protocol.html) -and establishes its property subscription before launching the window manager. -Font caches are generated with the image's normalized timestamp, following -[fontconfig's reproducible-cache support](https://fontconfig.pages.freedesktop.org/fontconfig/fontconfig-user.html), -so exporting the filesystem does not invalidate their directory timestamps. +The root menu includes Ethernet on/off controls. Equivalent console commands +are `fds network on`, `fds network off`, and `fds profiles` to inspect the result. diff --git a/docs/developer/README.md b/docs/developer/README.md new file mode 100644 index 0000000..40ef332 --- /dev/null +++ b/docs/developer/README.md @@ -0,0 +1,80 @@ +# Developer notes + +This is the engineering side of FDS: how the pieces are built, why choices were +made, and what we actually measured. The [user manual](../README.md) describes +how to use the current interfaces without the milestone history. + +Start with [the active cartridge/desktop revision](current-revision.md). Its +acceptance is separate from the frozen 0.1.0 release. Old test reports describe +the inputs named in those reports; a changed source tree needs fresh checks. + +## Design and build references + +- [Master plan](master-plan.md) +- [Build host and dependencies](build-host.md) +- [Development workflow](development.md) +- [Workstation tooling plan](workstation-tooling-plan.md) +- [Architecture](architecture.md), [packages](packages.md), [rootfs](rootfs.md), [init](init.md), [services](services.md) +- [Reproducible builds and frozen releases](reproducible-builds.md) +- [Dasung integration and physical monitor procedure](dasung.md) +- [Physical bay calibration and stress procedures](stress-testing.md) +- [Implementation ledger](implementation-status.md) and [roadmap](roadmap.md) + +## Acceptance history + +- [clap-validation](clap-validation.md) +- [dasung-validation](dasung-validation.md) +- [m0-validation](m0-validation.md) +- [m1-validation](m1-validation.md) +- [m10-validation](m10-validation.md) +- [m11-validation](m11-validation.md) +- [m12-validation](m12-validation.md) +- [m2-validation](m2-validation.md) +- [m3-validation](m3-validation.md) +- [m4-validation](m4-validation.md) +- [m5-validation](m5-validation.md) +- [m6-validation](m6-validation.md) +- [m7-validation](m7-validation.md) +- [m8-validation](m8-validation.md) +- [m9-validation](m9-validation.md) +- [workstation-validation](workstation-validation.md) + +## Earlier guides and working notes + +These retain useful implementation details and the historical interfaces that +went with them. Use the user manual for current commands, especially software +cartridge creation: new payloads contain installed Void package trees. + +- [architecture](architecture.md) +- [boot](boot.md) +- [build-host](build-host.md) +- [cartridges](cartridges.md) +- [cleanup](cleanup.md) +- [dasung](dasung.md) +- [data](data.md) +- [desktop](desktop.md) +- [development](development.md) +- [eeprom](eeprom.md) +- [getting-started](getting-started.md) +- [glossary](glossary.md) +- [implementation-status](implementation-status.md) +- [init](init.md) +- [internal-storage](internal-storage.md) +- [m4-work](m4-work.md) +- [master-plan](master-plan.md) +- [media-tools](media-tools.md) +- [packages](packages.md) +- [performance](performance.md) +- [power](power.md) +- [recovery](recovery.md) +- [releases](releases.md) +- [reproducible-builds](reproducible-builds.md) +- [roadmap](roadmap.md) +- [rootfs](rootfs.md) +- [services](services.md) +- [software-format](software-format.md) +- [stress-testing](stress-testing.md) +- [tooling](tooling.md) +- [troubleshooting](troubleshooting.md) +- [workstation-tooling-plan](workstation-tooling-plan.md) +- [workstation](workstation.md) diff --git a/docs/architecture.md b/docs/developer/architecture.md similarity index 97% rename from docs/architecture.md rename to docs/developer/architecture.md index afc66b4..484af5d 100644 --- a/docs/architecture.md +++ b/docs/developer/architecture.md @@ -1,5 +1,8 @@ # Understanding FDS/OS +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [Roadmap](roadmap.md) · [Glossary](glossary.md) FDS/OS is designed for a Raspberry Pi 5 portable computer whose operating system, diff --git a/docs/boot.md b/docs/developer/boot.md similarity index 97% rename from docs/boot.md rename to docs/developer/boot.md index aba4785..ecf0fdf 100644 --- a/docs/boot.md +++ b/docs/developer/boot.md @@ -1,5 +1,8 @@ # Boot images and stage0 +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [First build](getting-started.md) · [Implementation status](implementation-status.md) FDS keeps the machine's boot files on internal NVMe and its operating system on a diff --git a/docs/build-host.md b/docs/developer/build-host.md similarity index 90% rename from docs/build-host.md rename to docs/developer/build-host.md index 89c9cc6..a3bab75 100644 --- a/docs/build-host.md +++ b/docs/developer/build-host.md @@ -1,5 +1,8 @@ # M0: Arch Linux x86_64 build host +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [First build](getting-started.md) · [Troubleshooting](troubleshooting.md) This is the technical reference for the build foundation used by later milestones. @@ -41,15 +44,20 @@ explicitly use `XBPS_ARCH=x86_64` for the glibc build container, then run `./xbps-src -a aarch64 pkg hello` to build the ARM glibc package. Local ARM package indexing also explicitly sets `XBPS_ARCH=aarch64`. -## Separate generic Linux workstation tools +## Generic Linux workstation tools and desktop additions -`make workstation` builds native `fds-cartridge` and `fds-emulator` without the -Void/Arch bootstrap. Their runtime dependencies are QEMU (`qemu-system-aarch64` -and `qemu-img`), erofs-utils, xz and bubblewrap. A software recipe chooses its -cross compiler. The shared archive reader adds Rust `tar` 0.4.46 and its -`filetime` dependency; both are locked. No new daemon or host build tools are -installed in the Pi base image. Follow [the workstation guide](workstation.md). -The full OS build prerequisites below retain their existing scope. +`make workstation` builds native `fds-cartridge` and `fds-emulator`. Runtime +requirements are QEMU, erofs-utils, bubblewrap and native XBPS with a prepared +Void source checkout for software creation. `fds-cartridge` invokes xbps-src, +installs runtime dependencies and creates direct EROFS trees. The Rust tar reader +and xz utility remain for legacy cartridges and package/source archives. See +[the current workstation guide](../workstation.md). + +The target `fds-control` panel uses x11rb 0.13.2's pure-Rust core X11 connection. +It adds no shared GUI toolkit or background service. Its protocol, hostname and +OS-access dependencies are locked and included in Rust license notices. Both +the panel and foreground launcher are packaged in fds-cli as static ARM binaries. +See [the revision rationale](current-revision.md). ## Host dependencies @@ -173,6 +181,11 @@ runtime or Pi boot test. ## Inputs, caches, and overlays +Use `make clean-preview` followed by `make clean` when repeated builds and VM +tests accumulate. [Cleanup](cleanup.md) explains the retained current images, +release/input archives and caches. Full image builds and VM suites need tens of +GiB per working tree, beyond the initial M0 smoke test's requirements. + - `.host/xbps/`: project-local host tools, not installed into `/usr`. - `vendor/void-packages/masterdir-x86_64/`: glibc build container. - `vendor/void-packages/hostdir/`: download caches, cross packages, and build output. diff --git a/docs/developer/cartridges.md b/docs/developer/cartridges.md new file mode 100644 index 0000000..8e39c79 --- /dev/null +++ b/docs/developer/cartridges.md @@ -0,0 +1,200 @@ +# Using cartridges and configuring bays + +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + +[Documentation index](README.md) · [Boot images](boot.md) · [Services](services.md) + +M6 adds the base `fds-cartridged` service, twelve bay states, USB hardware +recognition, strict metadata inspection, read-only storage mounts and eject. +Its [ARM VM acceptance checks passed](m6-validation.md). [Writable DATA integration](data.md) has passed M7 software acceptance; +[desktop/program activation](desktop.md) passed M8 and [media creation tools](media-tools.md) +passed M9 software acceptance. [M10 ordered shutdown](power.md) has passed software acceptance. + +## Inspect the current machine + +At the FDS console: + +```sh +fds bays +fds bay 3 +fds cartridge 3 +fds --json bays +``` + +The ordinary `fds` user can use these commands. `bays` shows all twelve bays, +including empty and unconfigured ones. `cartridge` includes validated metadata +and the current mount path. `--json` exposes a versioned machine-readable report. +`fds rescan` explicitly refreshes inventory; normal insert/remove events trigger +refresh automatically. The console does not wait for the inventory scan. + +| State | Meaning and next action | +| --- | --- | +| UNCONFIGURED | No measured controller/port mapping exists for this bay | +| EMPTY | Configured bay with no detected USB device | +| HARDWARE | USB device without storage; catalog name or VID/PID is shown | +| UNRECOGNIZED STORAGE | Storage exists but no named FDS partition is available | +| MOUNTED READ ONLY | Manifest validated; filesystem available at the displayed path | +| MOUNTED READ WRITE | Active DATA is writable at `/data`; eject before removal | +| PROTECTED | The active SYSTEM root; eject is refused | +| AMBIGUOUS | Multiple devices or named partitions match; nothing is selected arbitrarily | +| ERROR | Inspection, mounting or metadata validation failed; read the diagnostic | +| SAFE | The requested unmount succeeded; remove the cartridge | + +An unrecognized storage state may briefly appear while the kernel discovers its +partitions. It is not permission to remove a device being used elsewhere. +Errors and ambiguous media never produce SAFE. An additional mount of the same filesystem also blocks managed eject until +it is unmounted. + +## Calibrate the physical bay map + +The shipped `/etc/fds/bays.toml` is deliberately empty: no Pi bay wiring has yet +been measured. This produces UNCONFIGURED states, not invented assignments. + +1. On the assembled machine, insert one known USB device into one physical bay. +2. Run `fds topology`. This diagnostic explicitly shows controller/port paths; + ordinary bay commands omit those implementation details. +3. Record that path and repeat for each bay. Test both USB 2 and USB 3 devices, + because their companion root hubs can have different logical port paths. +4. Put the measured map in your machine configuration directory and build the + [internal image](internal-storage.md), or use `fds machine install DIRECTORY` + from recovery and reboot. Machine settings persist independently of SYSTEM. + The packaged `/etc/fds/` files are fallback defaults for missing/invalid internal + storage; `fds machine status` shows which source is active. +5. Verify all twelve devices together, in different insertion orders and after + reboot. That physical acceptance remains deferred. + +A **syntax example only**, using identities that must be replaced by observations: + +```toml +[front] +hub = "platform/example-controller:usb2/1" +[front.ports] +1 = 1 +2 = 2 +3 = 3 +4 = 4 +5 = 5 +6 = 6 + +[front_superspeed] +hub = "platform/example-controller:usb3/1" +[front_superspeed.ports] +1 = 1 +2 = 2 +3 = 3 +4 = 4 +5 = 5 +6 = 6 +``` + +`hub` names the controller and parent port chain. Its numbered `ports` map the +next downstream port to a bay number from 1 through 12. Additional groups can +map the rear hub. Explicit USB 2/3 aliases may refer to the same bay; duplicate +ports, overlapping parent/child mappings, and duplicate bays within one hub are +rejected. If two devices simultaneously match aliases for one bay, it becomes +AMBIGUOUS. A hub inserted into a cartridge bay with multiple downstream devices +also needs an explicit future composite-device policy; M6 refuses to guess. + +The identity excludes the Linux USB bus number and `/dev/sdX` enumeration order. +It includes the controller's sysfs path, USB protocol generation and port chain. +Moving a hub to another controller or changing wiring requires recalibration. + +## Storage formats and metadata + +| Class | GPT partition name | Filesystem and current behavior | +| --- | --- | --- | +| SYSTEM | FDS_SYSTEM | EROFS; current root is protected, additional media read-only | +| DATA | FDS_DATA | ext4; inspected read-only first, then activated at `/data` under the M7 policy | +| PROGRAM | FDS_METADATA + FDS_PAYLOAD02… | Metadata and xz bundles in `1+m` EROFS partitions; see [software format](software-format.md) | +| Legacy PROGRAM | FDS_PROGRAM | Existing single EROFS application tree; still readable | +| ENVIRONMENT | FDS_ENVIRONMENT | EROFS; declarative selection of a trusted built-in profile | +| UTILITY | FDS_UTILITY | EROFS; inspection only, no automatic actions | +| HARDWARE | None required | VID/PID, optional serial, device/interface class and bay | + +Storage contains `/FDS/CARTRIDGE.TOML`. For example: + +```toml +format = 1 +[cartridge] +id = "fds.windowmaker" +name = "WINDOW SYSTEM" +class = "environment" +version = "0.1" +[media] +writable = false +[activation] +profile = "windowmaker" +``` + +The manifest class must match the GPT partition name. SYSTEM, PROGRAM and +ENVIRONMENT are read-only; DATA declares writable media during the initial read-only inspection. Unknown keys, root commands, unsafe identifiers, control +characters, symlink metadata and nonregular metadata files are rejected. +The parser accepts at most 64 KiB. No `/FDS/autorun.sh` is run. + +Inspect a standalone manifest with `fds inspect /path/to/CARTRIDGE.TOML`. +For mounted storage, `fds cartridge N` reports the daemon's validated copy. +Media is inspected in a root-only staging directory, then published at +`/run/fds/media/NN` after validation, with nodev, nosuid and noexec. Device +major/minor and kernel disk sequence are checked before mounting an opened +block-device descriptor, preventing stale enumeration names from selecting a +replacement disk. + +## Eject a mounted cartridge + +```sh +fds eject 2 +``` + +Leave any shell working directory inside that cartridge and close files first. +SAFE is emitted only after an ordinary unmount succeeds. Busy mounts return an +error; forced or lazy unmount is never used to declare safe removal. Once SAFE, +the daemon keeps that insertion unmounted until removal and reinsertion, including +across a service restart. Its root-owned volatile marker is tied to the kernel +disk sequence, so a replacement disk is not mistaken for ejected media. +The active SYSTEM cannot be ejected. Swap SYSTEM only after shutting down. + +Pulling a mounted cartridge without eject is surprise removal. The daemon clears +its inventory and detaches a vanished read-only mount where needed; that cleanup +is not a successful eject. [M7 DATA handling](data.md) extends ejection to writable DATA, consumer +tracking and syncfs. The temporary `/home/fds` remains usable without DATA and is lost at +power-off. + +## Hardware recognition + +Edit `packages/fds-cartridged/files/hardware-catalog.toml`, then rebuild SYSTEM. +Entries name devices, never executable actions: + +```toml +[[device]] +name = "MY SERIAL ADAPTER" +vendor = "1234" +product = "5678" +# Optional exact restrictions: +serial = "UNIT-1" +class = "02" +``` + +Replace these example IDs with observed values. Device or interface class may +match `class`. Multiple matching catalog entries produce an error; unknown +hardware remains identified by VID/PID without being mistaken for empty media. +Dasung control remains in the independent base monitor service. + +## Implementation and diagnostics + +The daemon subscribes to kernel USB/block events before taking its first snapshot. +A receive-buffer overflow causes a fresh snapshot. IPC uses a bounded Unix socket +at `/run/fds/control.sock`, mode 0660 and group `fds`, with peer UID checks for +root and the ordinary FDS user. Slow clients have individual deadlines and do not +block other clients. No asynchronous runtime or new external Rust dependency is +introduced: the existing std, libc, serde, serde_json and toml components suffice. + +Root diagnostics: `s6-svstat /run/service/cartridged` and +`cat /run/log/cartridged/current`. Logs are bounded and volatile. Restart through +`s6-rc -l /run/s6-rc -d change cartridged`, then the corresponding `-u` command. +Close users of cartridge mounts first; restart cleanup refuses busy leftovers. + +On the build host, `make cartridge-test` runs fixtures and full virtual USB tests. +These exercise actual kernel events and mounts but cannot establish physical +wiring, USB power stability, or Pi port behavior. M11 expands to twelve-device +stress, and physical calibration must be recorded separately. diff --git a/docs/clap-validation.md b/docs/developer/clap-validation.md similarity index 97% rename from docs/clap-validation.md rename to docs/developer/clap-validation.md index 596b696..0d65037 100644 --- a/docs/clap-validation.md +++ b/docs/developer/clap-validation.md @@ -1,5 +1,8 @@ # Clap command-line migration validation +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [Command usage](tooling.md) · [M12 evidence](m12-validation.md) Software acceptance completed on 2026-09-21. Every Rust command-line interface diff --git a/docs/developer/cleanup.md b/docs/developer/cleanup.md new file mode 100644 index 0000000..772e0ff --- /dev/null +++ b/docs/developer/cleanup.md @@ -0,0 +1,131 @@ +# Reclaim build space + +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + +Run these from the repository root after stopping builds and test VMs: + +```sh +make clean-preview +make clean +``` + +The preview lists every selected directory, its combined allocated footprint, +and the generated directories being kept. It does not delete files. `make clean` +recalculates the selection and deletes it without a confirmation prompt. Repeat +it whenever old build/test workspaces accumulate. It needs Python 3, Git and GNU +`du`, already used by the build host; it needs no Rust compilation, downloads, +root privileges or new Python packages. + +## What is removed + +- Old generated rootfs, kernel, initramfs, SYSTEM, boot, recovery and internal + image workspaces that are no longer referenced by published output links. +- Old generated integration-test directories, including their disk images, + disposable test DATA overlays and local diagnostic logs. The current results + referenced by `*-latest` links or workstation current-pointer files are kept. +- Old generated package staging copies and service compilation directories. +- This checkout's `target/` Rust compilation output. Exported executables in + `out/` and `out/workstation/` remain available. The next Rust build recompiles + as needed using the retained Cargo source cache. + +The command recognizes the specific temporary-directory names produced by the +repository's builders and tests. It never treats all of `out/` as disposable. +Unknown names are left alone, including ad hoc diagnostic experiments. + +## What is kept + +- Current rootfs archives for every profile, kernel, initramfs, boot, recovery, + internal disk and SYSTEM images, through their published symlinks. Links in + `out/manifests/` and links within retained workspaces also retain their targets. +- Latest published test runs, the current workstation image/emulator fixtures, + the current Dasung s6 database, and the newest `m9-images.*` fixture needed by + the media tests. These can still occupy tens of GiB. +- Signed releases such as `out/fds-os-0.1.0/`, input snapshots such as + `out/inputs-m12-v5/`, and independent restored trees such as + `out/rebuild-m12-v5-a/`. These remain available for release reproduction. +- User-created cartridge images, software bundles and personal emulator sessions + such as `out/my-emulator/`, including their persistent DATA overlays. +- `out/logs/`, `out/manifests/`, packages, downloads, `out/cache/`, `.host/`, the + Void checkout/build container, and Cargo/Rustup caches outside this checkout. +- Git-tracked files and any recognized workspace containing a `.fds-keep` entry. + +To retain an older generated workspace for investigation, put a marker in it +before cleaning: + +```sh +touch out/m8-vm.YOUR_RUN/.fds-keep +make clean-preview +``` + +Replace `YOUR_RUN` with the actual directory suffix. Remove that marker when the +workspace is disposable again. Reserve the generated names for the build/test +tools; store personal sessions and images under your own names. + +Cleanup leaves historical acceptance reports intact. Reports mentioning deleted +older workspaces are historical records; their files cannot be rechecked until +rebuilt, and they never prove that changed source code passed acceptance. + +## Why the directory became so large + +Image builders and integration tests deliberately retain their working +directories so failed runs can be inspected and a failed build cannot replace a +good published image. Repeated suites therefore accumulate complete rootfs +trees, archives and several disk images per case. Before this target existed, +this checkout's `out/` measured approximately **900 GiB** of allocated file +blocks after repeated M0–M12 and workstation validation runs. That is accumulated +history, not the size of a single OS image. + +`make clean` removes superseded workspaces; it does not impose automatic +retention during a build. Use the preview periodically. For complete builds, +allow tens of GiB per independent tree plus space for VM tests and saved inputs; +the small initial smoke test's space requirement does not cover the full suite. + +The preview uses allocated blocks, rather than adding the apparent capacities +of sparse VM disks. Hardlinks are counted once within the selection. Reflinks, +filesystem snapshots and hardlinks into retained directories can reduce the +physical space actually recovered. The command also prints the observed change +in filesystem free space; other processes and delayed filesystem accounting can +affect that number. + +## Running safely and rebuilding + +Use cleanup sequentially, just like the existing shared-container builds. It +holds the existing rootfs/base-package/image-tool locks, excludes another +cleanup, and refuses mounted candidate paths or observable active build/VM +processes. These checks are safeguards, not permission to start a separate build +while cleanup is running. It never follows a candidate symlink into another +directory, and refuses a symlinked `out/` entirely. Only the checkout containing +the script is eligible; there is no arbitrary deletion-path option. + +Existing published images and native tools remain usable after cleanup. To +refresh exported binaries, run `make tooling` or `make workstation`. To rebuild +the smoke-test artifacts and rerun build checks: + +```sh +make smoke-test +make check +``` + +`make check` needs the smoke-test binary in `target/`, so run the smoke test first +after cleaning. `make clean-test` independently exercises cleanup selection, +preserved releases/data, repeated runs, symlinks, tracked files, read-only guest +directories, hardlinks, locks, live processes and mount rejection using tiny +temporary repositories. It does not build OS images or boot VMs. + +## Recorded cleanup validation + +On the development checkout, cleanup removed 576 disposable directories with +a 638.00 GiB allocated footprint. The filesystem reported 605.86 GiB more free +space immediately afterward; `out/` dropped from about 900 GiB to 268 GiB. +The retained space includes independent release rebuilds, input snapshots and +current image/test workspaces. A second preview selected zero directories. + +All 52 recorded current image/tool and frozen release file hashes remained +identical, and all 33 published links remained intact. `make bootstrap`, +`make smoke-test`, `make check` and all nine cleanup fixture tests passed. +The smoke test skipped its optional ARM execution because host `qemu-aarch64` +was unavailable; its ELF checks passed. This validates host cleanup and does +not add any physical Pi or monitor acceptance evidence. The local command log +is `out/logs/cleanup.log`, with a summary in +`out/manifests/cleanup-acceptance.json`. diff --git a/docs/developer/current-revision.md b/docs/developer/current-revision.md new file mode 100644 index 0000000..ae6e909 --- /dev/null +++ b/docs/developer/current-revision.md @@ -0,0 +1,122 @@ +# Cartridge and desktop revision + +This work replaces the custom software bundle workflow and separates the user +manual from development history. The accepted 0.1.0 release stays unchanged. + +## Required outcomes + +1. Professional user documentation contains installation, usage and reference + material. Milestone history, measurements, test gaps and implementation notes + live separately under `docs/developer/`. +2. `fds-cartridge` builds Void source packages with `xbps-src`, installs their + runtime dependencies on the workstation, and writes installed program trees + directly into EROFS payloads. New cartridges require no guest extraction. + Existing cartridge reading stays available. +3. The grayscale E-Ink WindowMaker appearance is the default desktop theme on + every display, with normal user customization still possible. +4. Validated cartridge commands become available in existing shells through a + stable PATH directory. Direct launches retain terminal behavior, arguments, + exit status, dependency paths and managed eject/unplug cleanup. Command name + collisions have deterministic, documented behavior. +5. A native Rust X11 control panel with a matching retro appearance manages bay + status, cartridge details, program launch and safe eject from WindowMaker. + +## Implementation and acceptance + +- Replace creation of custom xz bundles with typed Clap source-package options, + installed-tree metadata, deterministic integrity checks and direct mounts. +- Verify real Void builds, dependency installation and resulting image contents; + retain corruption, filesystem and write/readback rejection coverage. +- Add managed PATH launch registration and exercise foreground commands, + collisions, eject, surprise removal and service restart in the ARM guest. +- Package and integrate the X11 panel and defaults. Exercise the actual window, + input actions and rendered appearance in the guest desktop. +- Rebuild CLI and development images. Run workstation/emulator, rootfs/init and + relevant cartridge/DATA/desktop/shutdown regressions against changed sources. +- Rewrite and relocate documentation after the final interfaces are verified. + Keep hardware-specific validation procedures and limits in developer notes. +- Use `make clean-preview` and `make clean` between completed validation batches + to avoid retaining hundreds of GiB of obsolete images again. + +Current status: all five changes are implemented and the fresh host/ARM VM +acceptance below has passed. Physical checks remain a separate hardware task. + +## Fresh acceptance + +The validation logs are retained under `out/logs/cartridge-desktop-revision/`. +`out/manifests/cartridge-desktop-revision.json` records the actual working-tree +source hashes, artifact hashes, fixture identities and log hashes. The source +revision is uncommitted; HEAD alone does not identify these changes. +These checks ran against this revision, independently of the frozen release. + +| Check | Evidence and scope | +| --- | --- | +| Build host | `make bootstrap`, `make smoke-test`, `make check`; all passed. The optional host-native `qemu-aarch64` shortcut was skipped; ARM execution uses the prepared Void environment. | +| Root filesystems | `make rootfs PROFILE=cli` and `PROFILE=development`, with `make rootfs-test` for each: passed. Archives `rootfs-build.9f2sNj` and `rootfs-build.PIumlA` contain byte-identical copies of the exported FDS tools and Dasung daemon. | +| Boot / tools | `make init-test`, `out/m2-vm.ATiMG0`: native s6 PID 1, service control and orderly poweroff passed. `make tooling-test` passed shared contracts and real ARM diagnostic execution. Both SYSTEM images and initramfs were rebuilt. | +| Rust | Host tests for common contracts, software, workstation tools, cartridge service, CLI and X11 panel; static-musl ARM compilation passed. | +| Software creation | `make workstation-test`, `out/workstation-images.f6tq1mya`: actual xbps-src builds, installed dependencies, EROFS trees, deterministic image bytes, malformed-content rejection and confirmed file write/readback. | +| Emulator / foreground commands | `out/emu-test.tqxiaglf`: direct PATH and qualified names, literal arguments, pipes, exit status, cwd permissions, interactive input, Ctrl-C, suspend/resume, terminal restoration, collisions, unplug, daemon restart and legacy format-1 archive compatibility passed. | +| Cartridge control | `make cartridge-test`, `out/m6-vm.e5j5n3q5`: active SYSTEM protection, unprivileged IPC, bounded clients, mounts, busy/safe eject, hotplug, removal and restart passed. | +| DATA | `make data-test`, `out/m7-vm.oiam2j2z`: sustained writes, privilege drop, descendant cleanup, sync/unmount, persistence, multiple candidates, interrupted media and surprise removal passed. | +| Desktop | `make desktop-test`, `out/m8-vm.zgk_yiyn`: real ARM Xvfb/WindowMaker, authentication, UID 1000 panel, global grayscale defaults, mouse launch of legacy and Void programs, keyboard rescan, safe eject and session cleanup passed. ENVIRONMENT and isolated DHCP regressions also passed. | +| Shutdown | `make power-test`, `out/m10-vm.rx6kz1zd`: ordered poweroff/reboot, writeback, active preparation, quarantine and refusal on unsafe storage passed. | +| Documentation | User instructions and development history are separated; all 66 Markdown documents had balanced code fences and valid relative file links. The actual panel screenshot was visually reviewed. | + +The [desktop screenshot](../images/fds-control.png) comes from the ARM guest, +not a mockup. Additional shell/where commands in its example cartridge belong +to the test fixture; the ordinary example publishes hello and report. + +## Validation limits + +The host used for acceptance is x86_64 Arch Linux with the prepared Void runner. +The native tool interface supports other Linux workstations with the documented +prerequisites, but this run does not establish coverage of every distribution. + +No physical Raspberry Pi boot, USB bay calibration, flash power-loss durability, +Dasung power-cycle recovery, E-Ink refresh quality or hardware latency was tested. +Use the existing hardware procedures for those checks. VM timings are VM +observations only. The independent s6/Dasung base integration remains mandatory. +The saved 0.1.0 release and its frozen inputs were not modified. + +## Implementation notes + +The X11 panel uses `x11rb` 0.13.2 and its pure-Rust connection, with core X11 +requests and no optional extensions or C GUI toolkit. The protocol companion +crate generates requests; `gethostname` and `rustix` support X11 authentication +and OS access. Versions are locked in Cargo.lock, and their licenses are included +by the existing Rust notice generator. This keeps the GUI compatible with the +static-musl target. Reference: https://docs.rs/x11rb/0.13.2/x11rb/. + +Foreground commands pass their already-open stdio descriptors to the privileged +service. The service creates the child, joins its root-owned bay cgroup, drops to +UID/GID 1000, and only then resolves the requested working directory. A pidfd and +exit status return to the launcher; interactive sessions use a PTY proxy. No +writable cgroup descriptor or permission to migrate arbitrary PIDs is exposed +to a client. This matters because Linux cgroup writes use open-file credentials. +Reference: Linux 6.12 kernel/cgroup/cgroup.c, cgroup_procs_write_start(). + +The first ARM compilation exposed musl/glibc differences in msghdr field widths; +portable field casts fixed those. Final acceptance uses rebuilt binaries and +images, rather than the intermediate builds that preceded that correction. + +The first emulator run found that Clap's multicall implementation uses +`Path::file_stem()`, truncating dotted aliases such as `b01:demo.report:shell`. +The launcher now lets Clap parse the invocation path as a typed positional and +uses a second typed external-subcommand parser for explicit `fds-program` calls. +Application help/version flags and dotted names have a regression test. This +keeps the no-manual-argument-parser rule while preserving the actual alias. + +The initial X11 click test raced the desktop terminal appearing above the panel. +The test now waits for that terminal and raises the panel before each real mouse +click. Actual program output, consumer tracking and SAFE state are still checked; +window existence alone is not treated as successful launch. + +The shutdown suite now publishes `out/m10-vm-latest` after successful completion. +That pointer was also installed for the successful run above. Cleanup retains +that evidence using the same published-link rule as the other runtime suites. + +After validation, `make clean` removed 52 obsolete workspaces and Rust build +output. The filesystem reported 62.17 GiB of newly available space. Current +images, latest runtime fixtures, caches, personal outputs and frozen release +inputs were retained; the complete cleanup log accompanies this revision. diff --git a/docs/dasung-validation.md b/docs/developer/dasung-validation.md similarity index 86% rename from docs/dasung-validation.md rename to docs/developer/dasung-validation.md index 9fbb0e4..2e23bd7 100644 --- a/docs/dasung-validation.md +++ b/docs/developer/dasung-validation.md @@ -1,10 +1,13 @@ # Dasung integration validation +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [Integration and usage](dasung.md) Validated on 2026-09-20 on the x86_64 Arch development host. This records the FDS port of the existing local `dasungd`; it is separate from the historical live -monitor validation in [the imported source](../rust/dasungd/VALIDATION.md). +monitor validation in [the imported source](../../rust/dasungd/VALIDATION.md). This is the original integration record. The subsequent rootfs build, emulated ARM execution, and integration rerun are recorded in [M1 validation](m1-validation.md). @@ -54,9 +57,9 @@ Package dependencies are coreutils, execline, s6, s6-rc, and eudev. The static executable does not add a shared musl, libusb, or libudev runtime dependency. The packet keepalive/pacing and confirmed 304210 kHz EDID are retained. -Local generated evidence: [build log](../out/logs/dasung-build-final.log), -[test log](../out/logs/dasung-test-final.log), and -[artifact manifest](../out/manifests/dasung-artifacts.sha256). These outputs are +Local generated evidence: [build log](../../out/logs/dasung-build-final.log), +[test log](../../out/logs/dasung-test-final.log), and +[artifact manifest](../../out/manifests/dasung-artifacts.sha256). These outputs are ignored by Git and must be regenerated in a fresh checkout. ## Not verified diff --git a/docs/developer/dasung.md b/docs/developer/dasung.md new file mode 100644 index 0000000..14f377c --- /dev/null +++ b/docs/developer/dasung.md @@ -0,0 +1,188 @@ +# Dasung Paperlike 13K base-system integration + +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + +[Documentation index](README.md) · [First build](getting-started.md) · [Services](services.md) + +FDS includes the Rust **dasungd** controller from the project's earlier +**Fix Dasung monitor black screen** task. It targets the user's **Paperlike 13K +grayscale**, serial `L56051794302`, with the confirmed 3200 × 2400 timing at about +37 Hz. This is core display support: its package is required for the base SYSTEM +image, and its service belongs to the base boot bundle, including console-only use. +It does not require an ENVIRONMENT cartridge, WindowMaker, or a graphical session. + +**Implementation boundary:** the daemon, ARM package, and s6 service definitions +are included by the [rootfs assembler](rootfs.md). [M2 native init](init.md) starts +the controller in the base boot graph, including in the ARM VM with no monitor. +The [M4 boot path](boot.md) starts an early copy before SYSTEM exists, then stops +and reaps it before base s6 starts the normal daemon. Software handoff is tested +in a VM; physical Pi display and monitor recovery remain unverified. The final +s6 database is compiled at image build time. + +## Build it on the workstation + +Complete `make bootstrap` as described in [Your first build](getting-started.md), +then run from the repository root, as your regular user: + +```sh +make dasung +make dasung-test +``` + +The first command prepares the additional C cross compiler inside the existing +Void build container, fetches the crates pinned in `Cargo.lock`, and cross-builds +static-musl `dasungd`. It verifies the ELF, builds the `fds-dasungd` XBPS overlay, +indexes the output, and compiles the actual s6 source into a validation database. +The second runs unit tests and a simulated serial monitor, then inspects the +package and compiles the service definitions extracted from that package. It also +runs that s6 graph with a native test binary, a read-only container root, and no +physical device access to check startup, logging, permissions, restart, and stop. + +Python 3 is needed **only on the host for the simulator**. If absent on Arch, +install `python` with pacman. No Python interpreter is shipped for the daemon. +The first additional toolchain download is approximately 243 MB and needs roughly +1 GB of extra container space. Builds still depend on rolling Void binary inputs. +Run these commands sequentially with other FDS builds; they share the container. + +| Output | Purpose | +| --- | --- | +| `out/dasungd` | Static AArch64 executable; no target glibc, musl, libusb, or libudev shared library dependency | +| `out/packages/fds-dasungd-0.1.0_1.aarch64.xbps` | Target package with executable, config, exact EDID, s6 source, udev rule, and licenses | +| `out/manifests/dasung-s6-database.txt` | Path to the service database compiled on the host for validation | +| `out/manifests/dasung-artifacts.sha256` | Executable and package hashes | +| `out/manifests/dasung-build-packages.txt` | Installed build-container input versions | +| `out/logs/dasung-build.log`, `dasung-checks.log` | Build and test evidence | + +The standalone validation database contains only the Dasung subset. The M2 +image assembler merges it with the init package and compiles the full current +graph while building the rootfs. No daemon is started on the workstation by these targets; +the integration test uses its own temporary socket and pseudo-terminal. + +## What goes into the base system + +`image/base-packages.list` makes `fds-dasungd` a mandatory hardware package. +`packages/fds-dasungd/template` is the active FDS-owned XBPS overlay. The upstream +Void tree remains pinned and unchanged. Source is maintained at `rust/dasungd/`; +the original task directory is not required for future builds. + +The package installs: + +- `/usr/bin/dasungd` and `/etc/dasungd.toml`. +- `/usr/lib/firmware/edid/dasung-paperlike13k-37hz.bin`. +- `/etc/s6-rc/source/dasungd`, its runtime-directory and log services, and the + `boot/contents.d/dasungd` membership entry. +- `/usr/lib/udev/rules.d/99-dasung-spi.rules`, adapted to eudev syntax. +- Documentation and the controller/libusb licenses. + +s6 supervises the foreground daemon. A short oneshot creates private writable +runtime/log directories; it does not wait for the display. The daemon starts with +no monitor attached and retries discovery. Its initial service transition does +not wait for a monitor reply, a desktop, the network, or all cartridge bays. +The log service rotates a small log under `/run/log/dasungd`. + +The init package adds a dependency on `runtime-fs` so the controller starts with +its kernel interfaces mounted. The service runs as root because it claims USB interfaces. Its socket +is under a root-owned mode-0750 directory; runtime control is restricted to root. +A future FDS CLI can expose selected operations through an explicit policy. +No world-writable USB rule, systemd unit, or runit service is installed. + +## Confirmed profile and Pi-specific choices + +| Property | Recorded value | +| --- | --- | +| Monitor | Paperlike 13K grayscale; `L56051794302` | +| Resolution | 3200 × 2400 | +| Pixel clock | **304210 kHz** (304.21 MHz) | +| Horizontal active / sync start / sync end / total | 3200 / 3248 / 3280 / 3360 | +| Vertical active / sync start / sync end / total | 2400 / 2423 / 2427 / 2447 | +| Sync polarities | Positive horizontal and vertical | +| EDID SHA-256 | `b6c1e0a8d315d9cf5c2fb83784a177530bcc085c2d0724dc7478a9c12449e4f4` | +| Control transport | USB UART `1a86:7523`, with companion SPI bridge `1a86:5512` | +| Startup parameters | Raw mode 1 and contrast 4, previously read back on this unit | +| Keepalive | `20 01` every two seconds; minimum 150 ms between command writes | + +The video timing was confirmed on the original AMD workstation, **not on a Pi**. +The original USB-C video connection is not a Pi cabling prescription. Pi connector, +cable/adapter, mode acceptance, and picture stability require hardware validation. +Desktop 2× scaling is separate from EDID timing; a console uses its own font size. + +FDS configures `display.enabled=false` and `hotplug="none"`: the daemon handles +USB control without applying the source project's AMD debugfs disconnect/reconnect +procedure. The Pi boot/display integration must supply the appropriate KMS/firmware +EDID configuration for its real connector. The Pi boot-volume builder now supplies +an unqualified firmware EDID override for this dedicated monitor; it does not +assume `DP-2` or a numbered HDMI connector. See the [boot guide](boot.md) for +the exact command line and the remaining physical mode test. + +The daemon requires the configured monitor's DRM EDID and the matching USB +companion topology before claiming a device. Multiple matching UARTs are rejected; +`usb_path` can disambiguate a verified Pi topology. Do not substitute a blanket +match for all CH340 adapters or copy this unit's identity to a different monitor. + +The original monitor's SPI driver interaction caused a dark picture after a power +cycle. `image/kernel/dasung.config` requires `CONFIG_SPI_CH341=n`; the Pi kernel +builder enforces this before boot, including the initramfs environment. The scoped +udev rule is secondary protection: it cannot prevent a built-in or already-loaded +SPI driver from probing. The daemon also reserves the companion interface without +sending SPI data and refuses to take it from an already-bound kernel driver. + +## Runtime usage once installed in an FDS image + +These are target commands, **not commands to run against the workstation's live +monitor during the build**: + +```text +dasungd status +dasungd query +dasungd refresh +dasungd set contrast 4 +dasungd set mode 1 --save +dasungd forget mode +``` + +`status` returns JSON including `connected`, `responsive`, reply age, and cached +parameters. A successful set means a packet was sent; use query/status to check +observed values. It is not proof of physical image quality. Color-model mode names +are not assigned to this grayscale monitor's raw numeric modes. + +The default socket is `/run/dasungd/control.sock`. External programs can use its +newline-delimited JSON protocol; the daemon remains the only USB owner: + +```json +{"op":"status"} +{"op":"set","parameter":"contrast","value":4,"save":true} +``` + +In FDS's current payload, saved overrides live in `/run/dasungd/settings.json`. +They survive a daemon restart **within the same boot**, but not power loss. +Mode 1/contrast 4 are persistent image configuration defaults. Once persistent +machine-state mounts are implemented, point `state_file` at that internal writable +storage to retain user overrides across boots. Do not put it on read-only SYSTEM +or make the base display daemon depend on a removable DATA cartridge. + +## Dependencies and source provenance + +The imported Rust dependencies are retained and pinned by the workspace lockfile: +`rusb` wraps USB access; `libc` supports Linux serial/locking calls; `ctrlc` handles +termination; `clap` provides CLI parsing; `serde`, `serde_json`, and `toml` support +configuration and the local socket; `anyhow` supplies contextual errors. +Transitive versions are recorded in `Cargo.lock`. + +The `vendored` feature builds libusb from the pinned `libusb1-sys` crate. The Void +`cross-aarch64-linux-musl` toolchain supplies C headers/compiler for that library; +Rust's bundled musl target alone was enough for the old pure-Rust smoke-test, but +is insufficient for this C dependency. `LIBUSB_NO_PKG_CONFIG` and +`LIBUDEV_NO_PKG_CONFIG` prevent accidental host/shared-library linkage. libusb uses +its Linux netlink backend. No libusb/libudev shared runtime package is needed. + +The target package depends on GNU coreutils for directory preparation, execline +for service launch, s6/s6-rc for supervision, and eudev for device rules. These +match the planned base userspace. The container's s6-rc is a build-time compiler, +not a target PID 1 running on Arch. + +[Import provenance](../../rust/dasungd/IMPORT.md) records the original source hashes. +[Original validation](../../rust/dasungd/VALIDATION.md) records the live handoff and +remaining physical test. Protocol and reconnect tests use simulated hardware; +Pi boot, image quality, physical disconnect/reconnect, and corrected cold-power +recovery must still be verified on the actual target. No boot benchmark is claimed. diff --git a/docs/developer/data.md b/docs/developer/data.md new file mode 100644 index 0000000..1f6d99d --- /dev/null +++ b/docs/developer/data.md @@ -0,0 +1,141 @@ +# Writable DATA and managed programs + +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + +[Documentation index](README.md) · [Cartridges](cartridges.md) · [Services](services.md) + +M7 implements writable DATA. Its sustained-write, safe-eject and surrounding +image regression checks [passed in the ARM VM](m7-validation.md). A DATA +cartridge uses a GPT partition named `FDS_DATA`, ext4, and a validated +`/FDS/CARTRIDGE.TOML` declaring class `data` and `writable = true`. +Use the [M9 media tools](media-tools.md) to create new DATA media. The virtual +tests construct disposable image files and never format a workstation disk. + +## Where your files live + +`/home/fds` remains a temporary home, whether DATA is present or absent. Its +contents disappear at power-off. Persistent user files belong under `/data`. +Separating these paths lets you eject DATA without hiding the active shell's +home directory or substituting another cartridge underneath it. + +After inserting a DATA cartridge into a configured bay: + +```sh +fds bays +fds cartridge 2 +ls /data +``` + +If exactly one valid DATA candidate is available and none is active, the daemon +activates it at `/data`. The filesystem must grant UID/GID 1000 the intended +write permissions; insertion does not recursively change ownership of user files. +M9's DATA formatter supplies that ownership for fresh media. + +An active DATA session stays attached to its current cartridge when another is +inserted. If a discovery snapshot contains multiple candidates with none active, +they remain read-only until you select one: + +```sh +fds data use 4 +``` + +Eject the existing active DATA before selecting a replacement. The daemon never +changes `/data` underneath an active session. EMPTY, ERROR and SAFE states retain +the meanings in the [cartridge guide](cartridges.md); MOUNTED READ WRITE means +that the displayed DATA filesystem is available for user writes. + +## Run a program that participates in eject + +Ordinary console commands work as usual. They are not automatically killed when +you request eject: an open file or working directory may make eject fail as busy. +For a background task that should stop when its DATA is ejected, use: + +```sh +fds run 2 -- /usr/bin/bash -c 'date > /data/managed-example.txt' +``` + +This starts a **background** managed program and prints its process ID. It has no +interactive input. Output goes to the bounded, volatile cartridge service log: + +```sh +tail /run/log/cartridged/current +fds cartridge 2 +``` + +The executable path must be absolute. The child starts with `/data` as its working +directory, the ordinary FDS UID/GID, no supplementary groups, no effective +capabilities, a small explicit environment, and `no_new_privs`. Programs cannot +gain privileges through setuid executables. The limit is 256 managed processes +per bay. This interface accepts healthy writable DATA; the [desktop guide](desktop.md) +also explains M8's explicit PROGRAM-media launch interface. + +A root-owned Linux cgroup tracks the program and its descendants, including +children that outlive their parent. Cgroups are kernel process tracking, not a +new init system or runtime package. The configured kernel already includes the +required cgroup v2 and process-limit support. The daemon sets up its hierarchy +independently of the console. See the [kernel cgroup interface](https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html). + +## Eject and remove DATA + +```sh +cd "$HOME" +fds eject 2 +``` + +Eject checks for additional mounts, stops managed consumers, flushes the DATA +filesystem with `syncfs`, and performs a normal unmount. Only successful completion +produces SAFE. A retained filesystem descriptor observes writeback errors during +the session. M10 additionally makes the filesystem read-only and checks writeback +again before closing that descriptor for unmount, preventing an error-reporting +gap when a busy mount needs a later retry. +Linux's [syncfs interface](https://man7.org/linux/man-pages/man2/syncfs.2.html) +reports filesystem errors that must not be ignored. + +Managed programs receive TERM first. The daemon waits on actual cgroup exit +notifications; programs that have not exited after a one-second deadline are +killed as a group, including newly forked descendants. A further bounded exit +check must succeed. These are failure deadlines, not delays imposed on a program +that has already stopped. Requesting eject therefore ends those background jobs; +it does not promise that unfinished application work is completed. + +An unmanaged busy shell/file, extra mount, failed flush or failed unmount prevents +SAFE. Close the reported use and retry. A recorded DATA writeback fault requires +investigation or recovery; a later successful call is not used to erase it and +pretend the failed write succeeded. The daemon never uses lazy/forced unmount to +claim successful eject. A successfully ejected insertion remains unmounted across +a daemon restart until removal/reinsertion. + +M10 records writable sessions and faults outside the daemon process. If it crashes +before verified unmount, the same insertion stays quarantined and read-only on +restart. A later flush cannot reconstruct the lost error history. See +[Shutdown and recovery records](power.md#data-recovery-records) for the current +implementation and its acceptance status. + +Pulling DATA during writes is an error. The daemon stops its managed consumers, +cleans up the vanished mount and logs that writes may have been lost. This path +never produces SAFE and makes no filesystem-cleanliness promise. Use recovery +inspection/repair before trusting media that was removed during I/O. + +## Build and verify on the workstation + +After changing DATA code or image configuration, build fresh inputs in order: + +```sh +make rootfs PROFILE=cli +make initramfs +make system-card +make data-test +``` + +The test boots disposable virtual media, writes continuously, verifies ordinary +user privileges and descendant shutdown, checks busy and additional mounts, +reopens persisted data, tests multiple candidates and simulates surprise removal. +After safe eject it extracts the ext4 partition into an ordinary file, runs +read-only `e2fsck -fn`, and compares the sustained-write payload byte-for-byte. +No host mount or physical USB device is used. + +Run `make cartridge-test`, `make init-test`, `make boot-test` and +`make console-test` for the surrounding regressions. Test logs and VM evidence +are written under `out/`. Real flash-controller caches, battery loss, USB power +and physical Pi eject/shutdown latency remain hardware acceptance work. diff --git a/docs/developer/desktop.md b/docs/developer/desktop.md new file mode 100644 index 0000000..4b78bb9 --- /dev/null +++ b/docs/developer/desktop.md @@ -0,0 +1,215 @@ +# Desktop, programs, and networking + +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + +[Documentation index](README.md) · [Cartridge guide](cartridges.md) · [DATA guide](data.md) + +M8 software checks passed in the ARM virtual machine. The [validation report](m8-validation.md) +records desktop, program and networking evidence. Raspberry Pi video output and +Dasung display quality remain deferred physical checks. + +## What starts when you boot + +FDS opens its ordinary-user console first. Xorg, WindowMaker, and the DHCP client +are installed in SYSTEM but are absent from the boot bundle. Dasung control +remains in the base boot bundle, including when no desktop is running. + +The default graphical backend is Xorg on virtual terminal 2, using its built-in +modesetting driver. The physical keyboard and pointer use libinput. Console +access remains on the original terminal. There is no display manager. Physical +VC4 output, monitor resolution, and E-Ink appearance still require Pi testing. + +## Start and stop the desktop + +At `FDS>`: + +```sh +fds profiles +fds profile activate windowmaker +fds profiles +``` + +Activation is asynchronous. The second status command reports the selected +profile and an activation-to-ready duration once WindowMaker is ready. This +measurement starts at the activation request, so time spent deciding when to +start the desktop is excluded. `fds boot-profile` also records the first desktop +readiness event for the boot. Neither measure includes firmware or power-on. + +The desktop opens an FDS terminal. Right-click its white background to open the +menu, launch another terminal, enable/disable Ethernet, or return to the console. +To stop it from either a terminal or the console: + +```sh +fds profile deactivate +``` + +Stopping the desktop closes its terminal and all processes descended from its +session. Save work first. Files in `/home/fds` survive desktop restarts during the +same boot; the home directory is still temporary and disappears at shutdown. +Use [DATA](data.md) for persistent files. Removing an active DATA cartridge also +stops the desktop before flushing DATA, because desktop applications may use it. + +`fds-profile status`, `fds-profile activate windowmaker`, and +`fds-profile deactivate` provide the standalone static helper interface. + +## ENVIRONMENT cartridges + +An ENVIRONMENT cartridge is a read-only EROFS partition named `FDS_ENVIRONMENT`. +Its `FDS/CARTRIDGE.TOML` uses the [documented manifest format](cartridges.md), with: + +```toml +[activation] +profile = "windowmaker" +``` + +When exactly one eligible ENVIRONMENT cartridge is present, the daemon requests +WindowMaker after console readiness. The graphical runtime comes from SYSTEM; +the cartridge provides a declarative request, never a privileged executable. +Unknown profile names do not execute anything. If several eligible cartridges +arrive together, select a profile explicitly instead of relying on bay ordering. + +Eject its bay with `fds eject N`. The desktop stops before SAFE is reported. +Surprise removal also stops its associated desktop. Manually deactivating while +the cartridge remains inserted suppresses automatic reactivation until removal +and reinsertion. A manually started desktop is independent of ENVIRONMENT media. +A cartridge-service restart reevaluates mounted media and retires old optional +sessions; this is a maintenance operation, not a way to preserve desktop jobs. + +## E-Ink defaults + +`fds-eink` installs the WindowMaker policy and Terminus fonts: a white background, +black text and borders, grayscale controls, outline movement/resizing, no +animations, no blinking decorations, no app-icon bounce, no dock, no compositor, +and no periodically updating clock. X11 compositing is disabled. The terminal +uses black on white and a nonblinking cursor. + +Defaults are copied into the ordinary user's `~/GNUstep/Defaults` at the first +session. Later sessions preserve edits there. They are temporary unless you +explicitly store a copy on DATA. The authoritative image defaults live in +`/usr/share/fds/eink`; edit their package sources and rebuild SYSTEM to change the +machine-wide defaults. + +Font caches and X11 font indexes are generated while building the image. +An active X server's keyboard map is session state, generated only on desktop +activation. That is separate from global boot-time cache generation. + +## Launch a software cartridge + +Build current software cartridges on a [Linux workstation](workstation.md). They +contain metadata plus xz software bundles in payload partitions. In FDS: + +```sh +fds bay 4 +fds run 4 -- demo.hello:hello +fds eject 4 +``` + +Use the software id and command listed by `fds bay`. The guest verifies and +extracts the bundle into a read-only temporary cache, then starts it as UID 1000. +There are no guest software build hooks. + +### Legacy PROGRAM compatibility + +Existing PROGRAM media contains a read-only EROFS partition named `FDS_PROGRAM`, metadata +in `FDS/CARTRIDGE.TOML`, and its self-contained application files: + +```text +app/bin/editor +app/lib/ +app/share/ +``` + +FDS validates and mounts it at `/run/fds/apps/`. Insertion does not +run anything, and the mount initially disallows execution. To launch an executable +named `editor` from bay 4: + +```sh +fds cartridge 4 +fds run 4 -- editor +``` + +Use a simple executable name from `app/bin`, followed by that program's arguments. +The daemon enables read-only execution and starts it as UID 1000, with no added +privileges. Executable paths must remain within the cartridge's `app` directory. +Programs receive `FDS_APP`, a PATH including `app/bin`, `LD_LIBRARY_PATH` including +`app/lib`, and `XDG_DATA_DIRS` including `app/share`. Bundled libraries can also use +an executable-relative RPATH. FDS does not resolve dependencies across cartridges. + +Programs are background jobs. Graphical programs can use the active authenticated +X display. Standard output/error goes to the bounded, temporary cartridge log: + +```sh +tail /run/log/cartridged/current +fds bay 4 +fds eject 4 +``` + +The bay status reports managed processes. Eject stops the whole process group, +including forked descendants, before unmounting. The existing DATA form remains +`fds run N -- /absolute/system/command arguments`, using `/data` as its working +directory. DATA itself remains mounted with execution disabled. + +## Ethernet on demand + +External networking is off until explicitly requested or a mapped USB Ethernet cartridge +produces a real network interface. Interfaces are associated with their USB +ancestors; a descriptive label alone does not start networking. Wi-Fi setup is +not implemented by this Ethernet policy. The local loopback interface is always +up for applications on the same machine; it does not enable external traffic. + +```sh +fds network on +fds profiles +ip -brief address +ip route +fds network off +``` + +Enabling networking starts DHCP without waiting for a lease. An absent server +cannot delay `FDS>`. Explicit activation selects available Ethernet interfaces; +automatic activation selects interfaces under mapped cartridge USB devices. +Stopping networking stops DHCP, brings the managed interfaces down, and clears +temporary DNS settings. This initial profile uses IPv4 DHCP; it disables kernel +IPv6 autoconfiguration on those interfaces. After an explicit stop, an inserted Ethernet +cartridge remains suppressed until it is removed, or you enable networking again. + +Leases, DNS settings, and logs live under `/run`; nothing makes SYSTEM writable. +`/etc/resolv.conf` points to `/run/fds/resolv.conf`. Inspect +`/run/log/network/current` when an address is missing. + +## Development and troubleshooting + +The production console image includes the real desktop stack. The development +image adds Xvfb and X11 diagnostics for a virtual display without a connected GPU: + +```sh +make rootfs PROFILE=development +make desktop-test +``` + +The test creates its own disposable image selecting Xvfb; the normal development +image still selects Xorg. Never mistake an Xvfb pass for Pi graphics verification. +Changing the SYSTEM image profile requires building that rootfs first; the image +builder rejects a profile label that disagrees with its embedded identity. + +If desktop activation fails, `fds profiles` reports it. Read +`/run/log/xserver/current` and `/run/log/desktop/current`, then deactivate before +retrying. Xorg's detailed log is `/run/log/xserver/Xorg.0.log`. X access requires +an authority cookie readable only by root and the FDS group; do not use `xhost +` +or disable access control as a workaround. + +Dependency rationale: Xorg supplies the display server and modesetting driver; +libinput supplies keyboard/pointer input; WindowMaker supplies window management; +Terminus and fontconfig supply the typography; xterm supplies a terminal; the static FDS helper +observes X11 property events for readiness without polling delays; xset/xsetroot apply the +static session settings. Xvfb, xdotool, xwd, xwininfo, xdpyinfo, xprop, and xauth are +optional development diagnostics. `fds-dhcpcd` builds the pinned upstream DHCP +client with privilege separation and volatile state paths, omitting its runit +service files. All FDS control helpers remain static Rust/musl binaries. + +The readiness observer uses the [X11 core protocol](https://xorg.freedesktop.org/archive/X11R7.7/doc/xproto/x11protocol.html) +and establishes its property subscription before launching the window manager. +Font caches are generated with the image's normalized timestamp, following +[fontconfig's reproducible-cache support](https://fontconfig.pages.freedesktop.org/fontconfig/fontconfig-user.html), +so exporting the filesystem does not invalidate their directory timestamps. diff --git a/docs/development.md b/docs/developer/development.md similarity index 94% rename from docs/development.md rename to docs/developer/development.md index bed244a..c21e81c 100644 --- a/docs/development.md +++ b/docs/developer/development.md @@ -1,10 +1,13 @@ # Working on FDS/OS +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [First build](getting-started.md) · [Troubleshooting](troubleshooting.md) This guide is for someone who has completed the first build and wants to change or inspect the working code. Run commands from the repository root, as a normal -user. M0–M12 software acceptance is complete; consult [AGENTS.md](../AGENTS.md) +user. M0–M12 software acceptance is complete; consult [AGENTS.md](../../AGENTS.md) and the [roadmap](roadmap.md) before adding components. ## Choose the smallest useful build @@ -33,12 +36,15 @@ Run builds sequentially. The scripts share one Void masterdir and package cache; starting bootstrap and package builds concurrently is not a supported workflow. `make packages` builds every FDS base package. `make all` builds the complete artifact set; two independent frozen offline builds passed with matching output. See -[Frozen inputs and offline rebuilds](reproducible-builds.md). There is no -destructive `make clean` or generic `make test` target. +[Frozen inputs and offline rebuilds](reproducible-builds.md). Use `make clean-preview` +and `make clean` to remove obsolete workspaces and Rust compilation output while +keeping published images, latest test fixtures, saved releases, inputs and +caches. See [Cleanup](cleanup.md) for retention rules and rebuilding afterward. +There is no generic `make test` target. ## Edit and rebuild the Rust program -The active program is [rust/fds-smoketest/src/main.rs](../rust/fds-smoketest/src/main.rs). +The active program is [rust/fds-smoketest/src/main.rs](../../rust/fds-smoketest/src/main.rs). It is deliberately tiny: it establishes that the intended target and static linking work. `rust/dasungd/`, `rust/fds-common/`, `rust/fds-cli/`, and `rust/fds-stage0/`, `rust/fds-boottrace/`, and `rust/fds-cartridged/` are also active. Directories absent from the workspace member @@ -85,9 +91,9 @@ part of the same change. The crate also rejects the wrong architecture, libc, or missing static CRT at compile time. Do not bypass those checks to make an x86_64 build appear to pass. -The release profile and target configuration live in [Cargo.toml](../Cargo.toml) -and [.cargo/config.toml](../.cargo/config.toml). Rust and target versions are -selected through [rust-toolchain.toml](../rust-toolchain.toml). This M0 crate has +The release profile and target configuration live in [Cargo.toml](../../Cargo.toml) +and [.cargo/config.toml](../../.cargo/config.toml). Rust and target versions are +selected through [rust-toolchain.toml](../../rust-toolchain.toml). This M0 crate has no external crate dependencies. `--offline` works after bootstrap has installed the toolchain; it is not a promise that the complete build pipeline is offline. @@ -126,7 +132,7 @@ an ARM package into the x86_64 build container. ## Change build configuration deliberately -The source of truth is [config/xbps-src.conf](../config/xbps-src.conf). +The source of truth is [config/xbps-src.conf](../../config/xbps-src.conf). `XBPS_MAKEJOBS` controls package build parallelism; the current value is 4. `XBPS_CHROOT_CMD=bwrap` selects the supported container backend. diff --git a/docs/developer/eeprom.md b/docs/developer/eeprom.md new file mode 100644 index 0000000..6f0bd13 --- /dev/null +++ b/docs/developer/eeprom.md @@ -0,0 +1,152 @@ +# Pi 5 EEPROM configuration + +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + +[Boot images](boot.md) · [Recovery](recovery.md) · [Implementation ledger](implementation-status.md) + +M12 now provides an offline configuration workflow using the pinned official +Raspberry Pi tool and a real Pi 5 firmware image. The host checks passed; +application to a physical Pi, boot order, PMIC behavior and timing remain deferred. +`tools/configure-pi-eeprom` only creates files. It never reads or writes a hardware +EEPROM, invokes a firmware updater, or reboots a machine. + +## Preview a profile on the build workstation + +From the repository root: + +```sh +./tools/configure-pi-eeprom --profile production +./tools/configure-pi-eeprom --profile development +make eeprom-test +``` + +The first invocation downloads three checksum-pinned inputs into +`.host/eeprom/`: the official configuration script, the Pi 5 preview firmware and +its upstream license. Later calls verify and reuse them. `FDS_OFFLINE=1` requires +all inputs to exist and refuses a network download. + +The command prints its new `out/eeprom-PROFILE.*` directory. It contains: + +| File | Purpose | +| --- | --- | +| `base.bin` | Exact input firmware retained unchanged | +| `original.conf` | Configuration to restore | +| `configured.conf` | Reviewed profile merged with the supplied settings | +| `configured.bin` | Real EEPROM image with the new configuration | +| `rollback.bin` | Same base firmware with `original.conf` restored | +| `review.diff` | Human-readable configuration changes | +| `manifest.json` | Input identity, selected settings and output SHA-256 hashes | +| `LICENSE` | Complete upstream tool and firmware notices | + +Default inputs are **preview defaults, not a backup of your Pi**. A rollback image +restores the configuration supplied to this command on the supplied base firmware. +It cannot reconstruct a different firmware version or settings that were never +provided. Retain the actual machine's configuration and matching firmware input +before applying anything to that machine. + +## Profile behavior + +| Setting | Production | Development | +| --- | --- | --- | +| `BOOT_ORDER` | `0xf6`: NVMe, repeat | `0xf16`: NVMe, SD, repeat | +| `BOOT_UART` | `0` | `1` | +| `NET_INSTALL_ENABLED` | `0` | `0` | +| `NET_INSTALL_AT_POWER_ON` | `0` | `0` | +| `POWER_OFF_ON_HALT` | `1` | `1` | +| `WAIT_FOR_POWER_BUTTON` | `0` | `0` | + +Boot order is read from the right. Both profiles avoid scanning the twelve USB +cartridges for firmware boot. Development retains an SD rescue path. Both disable +network-install keyboard detection; Raspberry Pi documents that this detection +adds USB initialization and enumeration work. These settings do not establish a +measured FDS boot improvement. [Official bootloader configuration](https://www.raspberrypi.com/documentation/computers/raspberry-pi.html#bootloader-configuration). + +On Pi 5, the power-off setting requests PMIC standby on halt; the dedicated power +button remains the wake mechanism. The wait setting leaves cold power-on boot +enabled. Whether the assembled computer and attached hardware behave as intended +still needs a physical test. [Official power settings](https://www.raspberrypi.com/documentation/computers/raspberry-pi.html#POWER_OFF_ON_HALT). + +The source profiles are [production.conf](../../config/eeprom/production.conf) and +[development.conf](../../config/eeprom/development.conf). No GPIO wake option or +unmeasured HDMI tuning is added. + +## Preserve the machine's settings + +In a Raspberry Pi maintenance environment with the official EEPROM utilities, +collect its configuration and version information before changing firmware: + +```sh +rpi-eeprom-config > pi-current.conf +rpi-eeprom-update > pi-eeprom-status.txt +``` + +Retain the matching original firmware image separately. The status report helps +identify it; it is not a binary backup. Copy the saved inputs to the workstation, +then prepare a new directory: + +```sh +./tools/configure-pi-eeprom \ + --profile production \ + --base-image /absolute/path/to/original-pi5-firmware.bin \ + --current-config /absolute/path/to/pi-current.conf \ + --output-directory /absolute/path/to/new-eeprom-review +``` + +The base must be a regular 2 MiB Pi 5 image, not a device node. The output directory +must not already exist. The helper preserves unrelated configuration lines and +conditional sections. It replaces all occurrences of the six managed settings, +including conditional overrides, with the selected profile's final `[all]` +settings. Review that change in `review.diff`; the exact original conditional +configuration remains in `original.conf` and `rollback.bin`. + +The upstream parser reads the generated images back before success is reported. +This checks file construction and configuration roundtrip, not hardware +compatibility. No secure-boot key, fuse, customer signature or OTP setting is +modified by this helper. + +## Apply and roll back during physical testing + +This step is deliberately outside the host/VM acceptance run. Use the Pi's +maintenance environment and the official installed EEPROM utilities. Once the +reviewed files and saved original inputs are available there, the upstream +configuration interface is: + +```sh +sudo rpi-eeprom-config --apply ./configured.conf ./base.bin +``` + +The updater may program EEPROM immediately or schedule an update depending on +its platform and update method. Follow its output and verify the resulting +configuration/version after the required restart. To restore the supplied +configuration on the same supplied firmware: + +```sh +sudo rpi-eeprom-config --apply ./original.conf ./base.bin +``` + +If the firmware version itself changed, use the saved matching original firmware +for the rollback operation. Keep the maintenance SD and an external EEPROM rescue +route available; internal recovery cannot repair an EEPROM that prevents internal +boot. The official [EEPROM update and recovery guide](https://www.raspberrypi.com/documentation/computers/raspberry-pi.html#raspberry-pi-boot-eeprom) +describes that physical workflow. FDS has not yet tested it on your Pi. + +## Inputs, dependencies and software evidence + +The input lock is [inputs.json](../../config/eeprom/inputs.json), pinned to +`raspberrypi/rpi-eeprom` commit `2fee426f27b6c54d3f5b6f36efd9a2fe1286a45d` and +Pi 5 preview firmware `pieeprom-2026-09-12.bin`. Every download is SHA-256 checked; +a changed cached input is rejected. The +[official source tool](https://github.com/raspberrypi/rpi-eeprom/blob/2fee426f27b6c54d3f5b6f36efd9a2fe1286a45d/rpi-eeprom-config) +is stored unchanged in the cache. Void's tracked source remains unchanged. + +The host workflow uses existing Python and curl. It does not add a target daemon, +package or Rust dependency. The upstream parser's optional signing dependencies +are not needed for configuration-only operations. + +`make eeprom-test` passed production/development roundtrips, unchanged firmware +payloads, repeatable binary output, refusal to overwrite an existing directory, +preservation of custom settings, exact rollback of conditional settings, and +rejection of malformed images, device nodes and oversized configuration files. +Initial evidence is `out/m12-eeprom.zi144zib/`, with log +`out/logs/m12-eeprom-check.log`. The complete local release acceptance is recorded in [M12 validation](m12-validation.md). diff --git a/docs/developer/getting-started.md b/docs/developer/getting-started.md new file mode 100644 index 0000000..9dd4653 --- /dev/null +++ b/docs/developer/getting-started.md @@ -0,0 +1,235 @@ +# Your first build + +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + +[Documentation index](README.md) · [Troubleshooting](troubleshooting.md) + +This walkthrough takes you from an FDS Git checkout to two verified ARM Linux +artifacts. It explains the output as you go. You do not need a Raspberry Pi. + +At the end you will have a small Rust program built with static musl and GNU hello +packaged for aarch64 glibc. This proves that the workstation can build the two +kinds of software FDS/OS needs. It does not produce a bootable OS image. + +## 1. Check the machine and checkout + +Use an x86_64 Arch Linux workstation as a normal user. Check it with: + +```sh +uname -m +cat /etc/os-release +id -u +``` + +Expect `x86_64`, `ID=arch`, and a user ID other than `0`. The bootstrap script +currently rejects other host platforms and root execution. + +Enter your FDS checkout. For the existing development workspace: + +```sh +cd /home/felis/source/fds +pwd +git status --short +``` + +On another workstation, substitute its checkout directory. It must contain +`Makefile`, `Cargo.toml`, `tools/`, `.gitmodules`, and Git metadata. Avoid spaces +in the path. There is no configured public remote in this workspace; obtain a +Git checkout from the project owner rather than using an invented clone URL. + +You need HTTPS access to GitHub, Void repositories, Rust distribution servers, +and package source hosts. Initial downloads total hundreds of MB. Plan for +several GB of free disk space; the verified workstation used roughly 1.6 GB for +the Void masterdir and 455 MB for its hostdir cache, before Rustup and other +outputs. These are observations, not fixed requirements or a build-time promise. + +```sh +df -h . +``` + +## 2. Install host prerequisites + +On an otherwise maintained Arch installation, install the required packages: + +```sh +sudo pacman -S --needed bash coreutils binutils git curl make file tar xz gzip zstd \ + bubblewrap rustup ca-certificates findutils diffutils grep sed gawk util-linux +``` + +This is the host package installation step. Subsequent build commands run as your +normal user. The alternative `./tools/bootstrap-host --install-deps` runs the same +package installation and then continues bootstrap; choose one route. + +The [dependency reference](build-host.md#host-dependencies) explains why each +package is needed. This project uses Rustup to select its pinned Rust compiler, +not an arbitrary system Rust version. If pacman reports an existing Rust package +conflict, resolve the host package choice before continuing; do not force file +replacement. + +Check that bubblewrap can create a user namespace: + +```sh +bwrap --ro-bind / / --unshare-user --uid 0 --gid 0 true +``` + +Success produces no output and exits normally. If it fails, use +[Troubleshooting](troubleshooting.md#bubblewrap-or-user-namespace-failure). +This is a host capability requirement, not something a Pi can fix. + +## 3. Prepare the build environment + +From the repository root: + +```sh +make bootstrap +``` + +This performs these steps: + +1. Initializes `vendor/void-packages` at the recorded Git commit. +2. Downloads the pinned static XBPS archive, verifies its SHA-256, and extracts + the host tools into `.host/xbps/`. +3. Copies the checked-in Void build configuration into the local upstream checkout. +4. Installs Rust 1.98.0, rustfmt, and `aarch64-unknown-linux-musl` through Rustup. +5. Fetches the locked workspace crates needed for Cargo to resolve the Dasung member, + including when building the dependency-free smoketest offline. +6. Creates the x86_64 glibc Void build container and verifies that GCC runs inside it. + +The Void build container supplies compilers and package tools; it is not the +future FDS root filesystem. Rustup changes your user toolchain installation. +XBPS host tools stay inside this repository, and no Pi storage is written. + +The final success marker is: + +```text +PASS: M0 host bootstrap; run make smoke-test to build both aarch64 artifacts +``` + +The command saves its output in `out/logs/bootstrap.log`. Re-running bootstrap +reuses the checkout and downloads, but still checks their pins and integrity. +If it exits nonzero, stop here and resolve that failure before running smoke-test. + +## 4. Cross-build both artifacts + +```sh +make smoke-test +``` + +First Cargo builds `rust/fds-smoketest` for ARM, using the static musl libraries +and linker bundled with the Rust toolchain. Then xbps-src cross-compiles GNU +hello and packages it for aarch64 glibc. The first package build downloads the +cross compiler, target libraries, and build dependencies such as texinfo. + +The script inspects actual ELF files and local package metadata. It checks: + +| Artifact | Required result | +| --- | --- | +| Rust smoketest | AArch64, static linkage, no dynamic interpreter or shared library requirement | +| GNU hello in the XBPS package | AArch64, glibc loader and `libc.so.6` dependency | + +A successful run contains these messages; other compiler output appears between them: + +```text +PASS: aarch64 static ELF +PASS: aarch64 glibc ELF +PASS: XBPS package hello-2.12.3_1 architecture=aarch64 (glibc) +PASS: M0 smoke test complete +``` + +Without QEMU, it also reports: + +```text +SKIP: ARM execution (optional qemu-aarch64 not installed); ELF verification passed +``` + +That skip means the ARM program was compiled and inspected, but not executed. +It does not hide a failed compiler or ELF check. If QEMU is installed, execution +must pass; an emulator failure is a real smoke-test failure. + +## 5. Run the protection and formatting checks + +```sh +make check +``` + +Run this after smoke-test: it expects the built Rust executable to exist. It +checks that bad inputs are rejected, that the Void pin and upstream files are +protected, that overlays cannot replace upstream packages, and that Rust source +is formatted. Several `PASS: rejects ...` lines are expected; they mean the +negative tests worked. The final commands include: + +```text +PASS: M0 guardrail checks complete +cargo fmt --all -- --check +``` + +Rustfmt is normally silent on success. `make check` does not rebuild the ARM +program. After a source change, rebuild first, as described in +[Development](development.md#edit-and-rebuild-the-rust-program). + +## 6. Inspect what you built + +```sh +ls -lh out/fds-smoketest out/packages/ +file out/fds-smoketest +./tools/verify-elf out/fds-smoketest aarch64 static +sha256sum -c out/manifests/artifacts.sha256 +``` + +`file` should include `ARM aarch64` and `statically linked` (or `static-pie linked`). +The checksum command should report `OK` for both artifacts. Hashes verify the +files against this run's manifest; they are not release signatures. + +For ELF details, use: + +```sh +readelf -hW out/fds-smoketest +readelf -lW out/fds-smoketest +readelf -dW out/fds-smoketest +``` + +The header names AArch64. There should be no `INTERP` segment or `NEEDED` library. +An x86_64 host's `ldd` may say `not a dynamic executable` and return 1; that message +is expected here, but is insufficient by itself to prove a foreign ELF is static. +The [package guide](packages.md#inspect-the-built-package) walks through the +corresponding inspection of GNU hello. + +## 7. Optionally execute the ARM Rust program + +The [Arch qemu-user package](https://archlinux.org/packages/extra/x86_64/qemu-user/) +provides user-mode emulation. Install it if you want to execute the ARM program +on the x86_64 workstation: + +```sh +sudo pacman -S --needed qemu-user +qemu-aarch64 out/fds-smoketest +``` + +Expected output: + +```text +FDS/OS M0: aarch64 static-musl OK +``` + +Because this executable is static, this command needs no ARM rootfs or glibc +sysroot. Explicitly invoking `qemu-aarch64` also avoids needing automatic binfmt +registration. Re-running `make smoke-test` records its own QEMU check when the +emulator is on PATH. + +This executes one Linux userspace program. It does not emulate the Pi's firmware, +USB bays, display, battery, or operating-system boot. QEMU execution was not +performed during the initial M0 validation; see the [recorded results](m0-validation.md). + +## 8. Decide what to do next + +- To build the configured OS filesystem and try its ARM programs, follow [Rootfs](rootfs.md): `make rootfs PROFILE=cli`, then `make rootfs-test`. +- To boot native s6 and open a development console, follow [Native init and ARM VM](init.md): `make init-test`, then `make vm`. +- To build the included Dasung monitor package, follow [Dasung](dasung.md). +- To edit the existing Rust program, follow [Development](development.md). +- To understand the `.xbps` artifact, follow [Packages](packages.md). +- To understand the intended removable operating system, read [Architecture](architecture.md) and [Cartridges](cartridges.md). +- To see when bootable images become possible, read [Roadmap](roadmap.md). + +Bootstrap, smoke-test, and check complete the M0 foundation. Continue with the +M1 guide for a configured rootfs archive. There is still no flashing step. diff --git a/docs/glossary.md b/docs/developer/glossary.md similarity index 96% rename from docs/glossary.md rename to docs/developer/glossary.md index 710d48f..d74319b 100644 --- a/docs/glossary.md +++ b/docs/developer/glossary.md @@ -1,5 +1,8 @@ # Glossary +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) | Term | Meaning in this repository | diff --git a/docs/implementation-status.md b/docs/developer/implementation-status.md similarity index 97% rename from docs/implementation-status.md rename to docs/developer/implementation-status.md index 99e8d49..9976f75 100644 --- a/docs/implementation-status.md +++ b/docs/developer/implementation-status.md @@ -1,5 +1,8 @@ # Implementation through M12 +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + This is the software acceptance ledger for the 67 sections of the [master plan](master-plan.md). The user authorized implementation through M12 and deferred tests requiring the physical Raspberry Pi or attached hardware. @@ -37,7 +40,7 @@ the commands, artifacts, failures that led to fixes, and measured results. | Section | Implementation and evidence | Physical or later boundary | | --- | --- | --- | -| 1. Project objective | Pi 5 images, all cartridge classes, native s6, Dasung base support and 12-device virtual stress; [overview](../README.md), [stress evidence](m11-validation.md) | Assembled FP-85, battery, display and all physical bays deferred | +| 1. Project objective | Pi 5 images, all cartridge classes, native s6, Dasung base support and 12-device virtual stress; [overview](../../README.md), [stress evidence](m11-validation.md) | Assembled FP-85, battery, display and all physical bays deferred | | 2. ABI strategy | Every profile uses aarch64 glibc packages; all eight production Rust programs are static ARM musl; rootfs and ELF audits pass | Actual Pi execution deferred | | 3. Limited musl scope | No target musl runtime package or second general-purpose package repository; glibc compiler/desktop acceptance passes | None in software scope | | 4. Native init | s6-linux-init hands PID 1 to s6-svscan; s6-rc database compiled during image construction; [native init evidence](m2-validation.md) | Pi startup timing deferred | diff --git a/docs/init.md b/docs/developer/init.md similarity index 97% rename from docs/init.md rename to docs/developer/init.md index 677f0b1..505bdfc 100644 --- a/docs/init.md +++ b/docs/developer/init.md @@ -1,5 +1,8 @@ # M2: native s6 init and the ARM development VM +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [Rootfs build](rootfs.md) · [Services](services.md) M2 adds the system's real init path. The kernel enters the generated execline diff --git a/docs/developer/internal-storage.md b/docs/developer/internal-storage.md new file mode 100644 index 0000000..a76f711 --- /dev/null +++ b/docs/developer/internal-storage.md @@ -0,0 +1,276 @@ +# Internal storage and machine settings + +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + +[Documentation index](README.md) · [Boot images](boot.md) · [Recovery](recovery.md) + +The internal NVMe supplies the Pi's firmware boot files, independent recovery, +and machine settings. SYSTEM remains a separate removable cartridge. Ordinary +user files belong on DATA, not internal storage. No physical disk is written by +any build command below; Pi/NVMe boot and power-loss tests remain deferred. + +## Build the complete disk image + +Build the component images first, in this order: + +```sh +make kernel +make initramfs +make boot-volume BOOT_MODE=production +make recovery +make internal-image +``` + +The result is `out/fds-internal.img`, a **complete GPT disk image**. Its versioned +build directory contains `layout.json`, source payload checksums, and readback +verification. This differs from `out/fds-boot.img` (a raw FAT partition) and +`out/fds-recovery.img` (a raw EROFS partition). + +| Partition | Format | Default allocation | Purpose | +| --- | --- | --- | --- | +| 1: FDS_BOOT | FAT32, EFI system type | 512 MiB | Pi firmware, kernel, DTBs, configuration and initramfs | +| 2: FDS_RECOVERY | EROFS | 1 GiB | Complete independent maintenance system | +| 3: FDS_INTERNAL | ext4 | 256 MiB | Machine settings and explicitly saved diagnostics | + +The builder pads the recovery partition without changing its EROFS contents, +aligns partitions to 1 MiB, writes both GPT copies, verifies each payload, and +checks the table independently with `sfdisk`. Internal ext4 is fully initialized +before deployment. It contains root-owned `config/` and private `diagnostics/`. +Unused capacity beyond this initial layout is not automatically expanded. + +For a 2 GiB recovery allocation or larger settings partition: + +```sh +./image/build-internal --recovery-mib 2048 --internal-mib 512 +``` + +Output directories supplied with `--output-directory` must already exist and be +empty. The builder accepts ordinary image files, never a host block-device +output. Final hardware provisioning and identity checks belong to the physical +acceptance procedure; do not confuse a partition payload with the complete disk. + +## Install the internal disk when hardware is available + +This is a **deferred physical procedure**. The image and virtual NVMe workflow +are software-tested; writing and booting the user's actual NVMe still require +the hardware. Installation erases the selected disk, including any existing +machine settings. Retain backups before replacing an existing installation. + +Use an NVMe enclosure or another Linux machine that can access the target drive +while the Pi is off. First verify a downloaded release using a separately trusted +public key as described in [Release signatures](releases.md). Its complete disk +is named `fds-internal-0.1.0.img`; a local build uses `out/fds-internal.img`. +Use the complete disk image, not the separate BOOT or RECOVERY payload. + +List disks before and after connecting the intended drive: + +```sh +lsblk -d -o NAME,PATH,MODEL,SERIAL,SIZE,TRAN,LOG-SEC +ls -l /dev/disk/by-id/ +``` + +Match the physical model, serial and capacity. Choose a persistent **whole-disk** +`/dev/disk/by-id/` path without a `-partN` suffix; never guess a `/dev/sdX` name. +Unmount every target partition and disable any swap on it. Do not select the +workstation's system disk. The current images require 512-byte logical sectors +and a disk at least as large as the image; 4 KiB logical-sector media is not +supported by this layout. + +Open Bash (`bash`), replace both paths below, then run the block. It checks the +selected disk again, requires a typed confirmation, writes and reads back the +complete image, and relocates the backup GPT when the disk is larger than the +image. It does not expand any partition or filesystem. + +```bash +( + set -euo pipefail + fds_image="$PWD/out/fds-internal.img" + fds_disk=/dev/disk/by-id/REPLACE_WITH_THE_TARGET_DISK + [[ -f "$fds_image" && -s "$fds_image" && -b "$fds_disk" ]] + [[ $(lsblk -dnro TYPE "$fds_disk") == disk ]] + [[ $(sudo blockdev --getss "$fds_disk") == 512 ]] + fds_bytes=$(stat -Lc %s "$fds_image") + (( $(sudo blockdev --getsize64 "$fds_disk") >= fds_bytes )) + lsblk -p -o NAME,TYPE,MODEL,SERIAL,SIZE,MOUNTPOINTS "$fds_disk" + if lsblk -nrpo MOUNTPOINTS "$fds_disk" | grep '[^[:space:]]' >/dev/null; then + echo 'Target has mounted filesystems or swap; stop and release them first.' >&2 + exit 1 + fi + if lsblk -nrpo TYPE "$fds_disk" | grep -Ev '^(disk|part)$' >/dev/null; then + echo 'Target has device-mapper or other active descendants; stop.' >&2 + exit 1 + fi + read -r -p "Type ERASE $fds_disk to erase this disk: " fds_confirmation + [[ "$fds_confirmation" == "ERASE $fds_disk" ]] + sudo dd if="$fds_image" of="$fds_disk" bs=4M conv=fsync status=progress + sudo blockdev --flushbufs "$fds_disk" + sudo cmp -n "$fds_bytes" "$fds_image" "$fds_disk" + sudo sfdisk --lock=yes --relocate gpt-bak-std "$fds_disk" + sudo sfdisk --verify "$fds_disk" + sudo blockdev --flushbufs "$fds_disk" +) +``` + +Every command must succeed. The byte comparison occurs before relocating GPT, +because relocation intentionally changes disk-table headers. GNU `dd`'s `fsync` +flushes output before it returns; `sfdisk`'s `gpt-bak-std` moves the backup header +to the end of the target. See the [GNU dd manual](https://www.gnu.org/s/coreutils/manual/html_node/dd-invocation.html) +and [sfdisk manual](https://man7.org/linux/man-pages/man8/sfdisk.8.html). +These commands use the already documented coreutils, diffutils and util-linux +host tools; they do not add a target daemon. + +Confirm that the disk shows `FDS_BOOT`, `FDS_RECOVERY` and `FDS_INTERNAL` in that +order using `lsblk -o NAME,PARTLABEL,FSTYPE,MOUNTPOINTS`. Unmount anything the +desktop automatically mounted, then safely disconnect the enclosure and install +the NVMe in the powered-off Pi. Follow [EEPROM preparation](eeprom.md) for the +separately reviewed NVMe boot settings. + +Prepare the **first SYSTEM cartridge on the workstation** using the same guarded +write block, with its two path assignments changed to the chosen SYSTEM image +and a different, empty USB cartridge disk. For a local build select +`out/fds-system-cli.img` or `out/fds-system-development.img`; release files are +`fds-system-cli-0.1.0.img` and `fds-system-development-0.1.0.img`. They are +alternative complete GPT images, each containing one `FDS_SYSTEM` partition. +Readback and backup-GPT relocation apply to this disk too. Check its partition +label, safely disconnect it, and insert exactly one SYSTEM cartridge in the Pi +before normal boot. Subsequent cartridge creation and updates can use FDS's +confirmed [media workflow](media-tools.md) after bay calibration. + +On the first physical boot, check `fds info`, `fds machine status`, and +`fds bays`. The supplied bay map is empty until calibration. Follow +[physical acceptance](stress-testing.md) to measure ports, test the Dasung +display, and record actual boot/shutdown behavior before relying on the machine. + +## Configure the machine before building + +Copy `config/machine/` to your own directory. It contains three files: + +- `machine.toml`: `format = 1` and a short human-readable `name`. +- `bays.toml`: the measured controller/port map described in + [Cartridges and bay calibration](cartridges.md). +- `hardware-catalog.toml`: optional USB identification names using the same schema + as the base catalog. Entries are data and cannot run commands. + +The supplied bay map is deliberately empty because the physical wiring has not +been measured. Do not invent Pi USB paths. USB 2 and USB 3 companion ports need +explicit aliases for the same bay. + +```sh +cp -a config/machine out/my-machine +# Edit the three files in out/my-machine using your editor. +make internal-image MACHINE_CONFIG=out/my-machine +``` + +The builder compiles a native host copy of `fds` and uses the same strict parser +as the target system. You can also validate or pack settings yourself: + +```sh +cargo build --locked --offline --release --target x86_64-unknown-linux-gnu -p fds-cli +./target/x86_64-unknown-linux-gnu/release/fds machine validate out/my-machine +./target/x86_64-unknown-linux-gnu/release/fds machine pack out/my-machine out/my-machine.json +``` + +The three source files become one atomic `config/machine.json` document on +FDS_INTERNAL. Names, lengths, bay aliases, catalog fields and unknown keys are +validated before use. Do not put passwords or private keys in this configuration: +its active snapshot is readable by the local FDS user. + +## What happens at boot + +The `machine-config` native s6 oneshot precedes `cartridged`. It does **not** +precede the console or Dasung controller. It accepts exactly one non-removable +NVMe disk with the three named partitions in the order above. USB lookalikes are +ignored; multiple eligible NVMe disks are rejected instead of choosing by name. + +The settings partition must be clean ext4 with the expected label. Loading uses +`ro,noload,nosuid,nodev,noexec` in a private mount namespace, validates the entire +settings bundle, copies it into `/run/fds/machine/`, and unmounts. No filesystem +repair, journal replay, cache compilation or persistent write occurs during +normal boot. The cartridge service uses that snapshot throughout this boot, +including after a service restart. + +If internal storage is missing, unclean, invalid or ambiguous, the service records +an explanation and uses the immutable image's `/etc/fds/` defaults. The console +still opens. Check the actual source before treating bays as calibrated: + +```sh +fds machine status +fds --json machine status +fds machine export /tmp/current-machine +``` + +Export creates a new directory with the three editable source files. It never +overwrites an existing directory. The status source is `internal_nvme` or +`image_defaults`. A temporarily unavailable NVMe is not adopted later in the same +boot; resolve the issue and reboot to load its settings. This prevents changing +bay identities underneath active cartridge operations. + +## Update settings from recovery + +Bring the edited source directory on a DATA cartridge. At the local root +`RECOVERY#` console, identify its bay with `fds bays`; healthy DATA is mounted +read-only under `/run/fds/media/NN`. For example, if it is in BAY 02: + +```sh +fds machine validate /run/fds/media/02/my-machine +fds machine install /run/fds/media/02/my-machine +fds reboot +``` + +Installation is restricted to root in the recovery image. It writes the complete +validated bundle atomically, saves the old bytes as `config/previous.json`, +flushes and unmounts the internal filesystem, and reports success only after +those steps. A reboot activates the new settings. The currently running bay map +is unchanged, so active media does not move to a different bay mid-operation. +Keep a copy of your previous source directory on DATA or the build host to +reinstall it if the new calibration is wrong. + +An invalid existing JSON document can be replaced this way. Wrong ownership, +symlinks, an unclean filesystem or a damaged directory require offline filesystem +maintenance first. Recovery does not automatically repair internal NVMe; its +`fds recovery repair` command is deliberately limited to DATA cartridges. + +## Save and retrieve diagnostics + +Logs and boot records stay in RAM by default. Root may explicitly save a file +of up to 16 MiB. Saved names cannot contain paths, and existing names are refused. +These operations mount internal ext4 only for the operation and then unmount it. +For example, from recovery: + +```sh +fds --json boot-profile >/tmp/boot.json +fds machine store boot-first.json /tmp/boot.json +fds --json bays >/tmp/cartridge-inventory.json +fds machine store cartridges-first.json /tmp/cartridge-inventory.json +bash /usr/share/fds/capture-hardware /tmp/hardware-capture +tar -C /tmp -czf /tmp/hardware-capture.tar.gz hardware-capture +fds machine store hardware-first.tar.gz /tmp/hardware-capture.tar.gz +fds machine fetch boot-first.json /tmp/retrieved-boot.json +``` + +Use distinct names for subsequent sessions. A failed flush or unmount is an +error, not a successful save. These are machine diagnostics; do not use this +facility as ordinary user storage. The saved cartridge inventory is a persistent +diagnostic snapshot, including metadata already inspected during this boot. +Retrieve it with `fds machine fetch cartridges-first.json /tmp/saved-inventory.json`. +The daemon's live metadata cache remains volatile and is rebuilt from currently +attached devices; saved snapshots are never used to authorize media actions. +Together, explicit boot reports, inventory snapshots and hardware captures provide +the boot history, cached metadata and diagnostics assigned to FDS_INTERNAL in +master-plan section 12, without adding internal writes to startup or shutdown. + +## Dependencies and validation + +No new target package or Rust crate is required. The host image-tool prefix adds +`e2fsprogs` for ext4 creation, inspection and validation; it already supplies FAT +and EROFS tools. A private unprivileged user namespace gives created files root +ownership without requiring a root build session. + +`make internal-test` exercises the actual packaged runtime in ARM VMs with +virtual NVMe. Acceptance evidence belongs in [M12 validation](m12-validation.md); +a passing VM does not verify the Pi EEPROM, PCIe path or physical flash durability. + +The Linux [ext4 mount documentation](https://www.kernel.org/doc/html/latest/admin-guide/ext4.html) +explains why read-only loading also disables journal replay. Filesystem creation +options follow the upstream [mke2fs manual](https://man7.org/linux/man-pages/man8/mke2fs.8.html). diff --git a/docs/m0-validation.md b/docs/developer/m0-validation.md similarity index 91% rename from docs/m0-validation.md rename to docs/developer/m0-validation.md index 949d184..0d33810 100644 --- a/docs/m0-validation.md +++ b/docs/developer/m0-validation.md @@ -1,5 +1,8 @@ # M0 validation report +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [First build](getting-started.md) · [Roadmap](roadmap.md) This is the initial acceptance record, not a setup tutorial. Later documentation @@ -97,13 +100,13 @@ smoke-test then passed. Actual logs and inventories: -- [Bootstrap log](../out/logs/bootstrap.log) -- [Smoke-test log](../out/logs/smoke-test.log) -- [Package build log](../out/logs/xbps-hello.log) -- [Guardrail and formatting checks](../out/logs/check.log) -- [Build package inventory](../out/manifests/void-build-packages.txt) -- [Cached package input digests](../out/manifests/void-package-inputs.sha256) -- [Artifact digests](../out/manifests/artifacts.sha256) +- [Bootstrap log](../../out/logs/bootstrap.log) +- [Smoke-test log](../../out/logs/smoke-test.log) +- [Package build log](../../out/logs/xbps-hello.log) +- [Guardrail and formatting checks](../../out/logs/check.log) +- [Build package inventory](../../out/manifests/void-build-packages.txt) +- [Cached package input digests](../../out/manifests/void-package-inputs.sha256) +- [Artifact digests](../../out/manifests/artifacts.sha256) Generated output is ignored by Git. These links refer to this validated checkout; rerun the commands to recreate the output in another checkout. diff --git a/docs/m1-validation.md b/docs/developer/m1-validation.md similarity index 96% rename from docs/m1-validation.md rename to docs/developer/m1-validation.md index 6243e1a..c7615b1 100644 --- a/docs/m1-validation.md +++ b/docs/developer/m1-validation.md @@ -1,5 +1,8 @@ # M1 root filesystem validation +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [Build and usage](rootfs.md) · [Roadmap](roadmap.md) Validated on **2026-09-21 (Asia/Shanghai)** on the x86_64 Arch workstation. diff --git a/docs/m10-validation.md b/docs/developer/m10-validation.md similarity index 97% rename from docs/m10-validation.md rename to docs/developer/m10-validation.md index 65ff840..e2c3449 100644 --- a/docs/m10-validation.md +++ b/docs/developer/m10-validation.md @@ -1,5 +1,8 @@ # M10 shutdown validation +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Shutdown usage](power.md) · [Implementation ledger](implementation-status.md) M10 software acceptance passed on 2026-09-21 in isolated ARM virtual machines. diff --git a/docs/m11-validation.md b/docs/developer/m11-validation.md similarity index 96% rename from docs/m11-validation.md rename to docs/developer/m11-validation.md index de34aba..727c901 100644 --- a/docs/m11-validation.md +++ b/docs/developer/m11-validation.md @@ -1,5 +1,8 @@ # M11 stress validation +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Stress-test usage and physical procedure](stress-testing.md) · [Implementation ledger](implementation-status.md) M11 software acceptance passed on 2026-09-21. The expanded twelve-device suite diff --git a/docs/m12-validation.md b/docs/developer/m12-validation.md similarity index 98% rename from docs/m12-validation.md rename to docs/developer/m12-validation.md index 8f4b219..94fefae 100644 --- a/docs/m12-validation.md +++ b/docs/developer/m12-validation.md @@ -1,5 +1,8 @@ # M12 production preparation: software evidence +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Implementation ledger](implementation-status.md) · [EEPROM](eeprom.md) · [Recovery](recovery.md) M12 **software acceptance is complete**, including the signed local FDS/OS @@ -10,7 +13,7 @@ the development history; the final acceptance identifies the released bytes. ## Final local release acceptance Completed on 2026-09-21. The local release is -[`out/fds-os-0.1.0/README.md`](../out/fds-os-0.1.0/README.md). +[`out/fds-os-0.1.0/README.md`](../../out/fds-os-0.1.0/README.md). It contains 38 signed manifest entries, versioned images and packages, profile inventories, source and complete build-input archives, guides and reproducibility evidence. It has not been published or installed on hardware. @@ -27,11 +30,11 @@ Both started without FDS build outputs. The complete 27-artifact comparison passed, covering all five installable images, three rootfs tars, four initramfs formats, kernel/DTB, four EEPROM preview files and all nine FDS packages. Logs: `out/logs/m12-offline-v5-{a,b}.log`; comparison: -[`out/m12-reproducibility-v5.json`](../out/m12-reproducibility-v5.json). +[`out/m12-reproducibility-v5.json`](../../out/m12-reproducibility-v5.json). Every one of those artifacts also matches the normal build used by the complete [Clap runtime acceptance](clap-validation.md). The identity proof is -[`out/m12-tested-to-release-v5.json`](../out/m12-tested-to-release-v5.json), +[`out/m12-tested-to-release-v5.json`](../../out/m12-tested-to-release-v5.json), and `out/m12-clap-acceptance-index.json` records the 21 acceptance logs and hashes. This establishes exact-byte coverage of the released images without repeating the same VM suite against identical images. Physical tests remain deferred. @@ -48,7 +51,7 @@ Verification against the separately retained public key checked every manifest entry. OpenSSL independently verified the actual release signature. Both actual release archives were extracted with mode-preserving flags; the complete input lock and standalone source identity passed. Evidence: -[`out/m12-release-acceptance-v5.json`](../out/m12-release-acceptance-v5.json), +[`out/m12-release-acceptance-v5.json`](../../out/m12-release-acceptance-v5.json), with logs in `out/logs/m12-v5-*.log` and extracted files under `out/m12-release-acceptance.mzkfsvev`. diff --git a/docs/m2-validation.md b/docs/developer/m2-validation.md similarity index 97% rename from docs/m2-validation.md rename to docs/developer/m2-validation.md index 58c4293..a53caf5 100644 --- a/docs/m2-validation.md +++ b/docs/developer/m2-validation.md @@ -1,5 +1,8 @@ # M2 native init validation +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [Boot and usage](init.md) · [Roadmap](roadmap.md) Validated on **2026-09-21 (Asia/Shanghai)** on the x86_64 Arch workstation. diff --git a/docs/m3-validation.md b/docs/developer/m3-validation.md similarity index 90% rename from docs/m3-validation.md rename to docs/developer/m3-validation.md index ab64250..4462eea 100644 --- a/docs/m3-validation.md +++ b/docs/developer/m3-validation.md @@ -1,5 +1,8 @@ # M3 static Rust tooling validation +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + Validated on 2026-09-21 (Asia/Shanghai), x86_64 Arch Linux with pinned Rust 1.98.0. This records the M3 diagnostic foundation before M4 adds the real boot handoff. See [tooling usage](tooling.md) and the [through-M12 ledger](implementation-status.md). diff --git a/docs/m4-validation.md b/docs/developer/m4-validation.md similarity index 95% rename from docs/m4-validation.md rename to docs/developer/m4-validation.md index 18acba2..132c347 100644 --- a/docs/m4-validation.md +++ b/docs/developer/m4-validation.md @@ -1,5 +1,8 @@ # M4 software boot validation +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + Validated on the x86_64 Arch workstation on 2026-09-21 (Asia/Shanghai). Physical Pi testing is deferred by the user. This report establishes the software boot machinery, not the physical NVMe/RP1/display path or performance diff --git a/docs/m4-work.md b/docs/developer/m4-work.md similarity index 94% rename from docs/m4-work.md rename to docs/developer/m4-work.md index 27e996e..4c83af3 100644 --- a/docs/m4-work.md +++ b/docs/developer/m4-work.md @@ -1,5 +1,8 @@ # M4 boot integration development record +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + The user has authorized the remaining software through M12. This page records M4 development decisions. Software validation is complete; see [M4 evidence](m4-validation.md). Physical Pi validation remains deferred. diff --git a/docs/m5-validation.md b/docs/developer/m5-validation.md similarity index 95% rename from docs/m5-validation.md rename to docs/developer/m5-validation.md index 72c70d2..b82e292 100644 --- a/docs/m5-validation.md +++ b/docs/developer/m5-validation.md @@ -1,5 +1,8 @@ # M5 software validation +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + M5's ordinary-user console and software timing tools passed on the x86_64 Arch build host using the actual AArch64 kernel, initramfs and SYSTEM image in QEMU 11.1.1. Physical Pi boot, power-on, display and timing acceptance is deferred. diff --git a/docs/m6-validation.md b/docs/developer/m6-validation.md similarity index 95% rename from docs/m6-validation.md rename to docs/developer/m6-validation.md index dde5b90..d3f7f5f 100644 --- a/docs/m6-validation.md +++ b/docs/developer/m6-validation.md @@ -1,5 +1,8 @@ # M6 cartridge software validation +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + Verified on 2026-09-21 on the x86_64 Arch build host. These results use actual AArch64 binaries, kernel USB events, s6 services and filesystem mounts in QEMU 11.1.1. Physical Pi wiring, bay calibration, simultaneous physical devices and diff --git a/docs/m7-validation.md b/docs/developer/m7-validation.md similarity index 95% rename from docs/m7-validation.md rename to docs/developer/m7-validation.md index 1495cc6..a0f32c3 100644 --- a/docs/m7-validation.md +++ b/docs/developer/m7-validation.md @@ -1,5 +1,8 @@ # M7 writable DATA validation +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + Software acceptance passed on 2026-09-21 using the x86_64 Arch build workstation, QEMU 11.1.1, the actual AArch64 Pi kernel and static FDS tools. Physical Pi, removable-media power loss and real shutdown/eject latency remain deferred. diff --git a/docs/m8-validation.md b/docs/developer/m8-validation.md similarity index 97% rename from docs/m8-validation.md rename to docs/developer/m8-validation.md index 220ba65..74331f2 100644 --- a/docs/m8-validation.md +++ b/docs/developer/m8-validation.md @@ -1,5 +1,8 @@ # M8 desktop and network validation +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Usage guide](desktop.md) · [Implementation ledger](implementation-status.md) M8 software acceptance passed on 2026-09-21, including corrected CLI and diff --git a/docs/m9-validation.md b/docs/developer/m9-validation.md similarity index 97% rename from docs/m9-validation.md rename to docs/developer/m9-validation.md index e6a3385..8d5106e 100644 --- a/docs/m9-validation.md +++ b/docs/developer/m9-validation.md @@ -1,5 +1,8 @@ # M9 media-tool validation +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Media-tool usage](media-tools.md) · [Implementation ledger](implementation-status.md) M9 software acceptance passed on 2026-09-21. Both packaged profiles, ARM image diff --git a/docs/master-plan.md b/docs/developer/master-plan.md similarity index 94% rename from docs/master-plan.md rename to docs/developer/master-plan.md index ce31aac..7765497 100644 --- a/docs/master-plan.md +++ b/docs/developer/master-plan.md @@ -6,7 +6,7 @@ integration and the signed local 0.1.0 release.** Tests requiring physical Pi hardware are deferred; software and VM checks remain in scope. See the [implementation ledger](implementation-status.md). A command appearing here is not evidence that it exists in the checkout. Start with the -[project overview](../README.md) and [first-build walkthrough](getting-started.md) +[project overview](../../README.md) and [first-build walkthrough](getting-started.md) for current usage, or the [roadmap](roadmap.md) for implementation status. The 67 numbered sections below retain the full project requirements. @@ -1326,6 +1326,10 @@ Activating a GUI profile starts the user's graphical session directly. # 31. E-Ink profile +Use the grayscale profile as the default WindowMaker appearance on every +display. Install it globally and seed new user preferences without replacing +existing customization. The native FDS Control panel follows the same style. + The monitor controller is base hardware support, independent of the desktop profile. Include the existing Rust `dasungd` for the user's Paperlike 13K grayscale in every SYSTEM image and the native s6 boot bundle. Maintain the confirmed @@ -1374,40 +1378,26 @@ Do not update the clock every second. # 32. PROGRAM cartridge -Do not design a complex dependency resolver across cartridges in version 1. +Create software cartridges on a generic Linux workstation using `fds-cartridge`. +Each format-2 software recipe identifies a Void source package and its commands. +The tool builds with `xbps-src`, installs the package and runtime dependencies +into a private tree, and creates GPT images with metadata partition 1 plus one +or more EROFS program partitions. -New PROGRAM software cartridges use GPT with exactly `1 + m` read-only EROFS -partitions. Partition 1 (`FDS_METADATA`) contains `FDS/CARTRIDGE.TOML` and -`FDS/SOFTWARE.TOML`. The catalogue describes every software bundle in the next -`m` partitions, named `FDS_PAYLOAD02` onward. Each software bundle is an -xz-compressed tarball; multiple bundles may share a payload partition. +Payloads contain `programs/SOFTWARE-ID/` installed roots, including executable +files and libraries. Verify tree integrity and architecture on insertion; run +new software directly from the read-only EROFS mount without guest extraction. +Retain reading of legacy PROGRAM media and catalogue-format-1 archives. -Software compilation, archive packaging and complete cartridge image creation -run on a generic Linux workstation. The confirmed USB writer writes and verifies -that complete disk image. QEMU workstation tooling boots the real FDS kernel, -initramfs and SYSTEM and inserts/removes these images as virtual USB cartridges. +Publish commands through `/run/fds/bin` using a managed foreground launcher. +Keep `fds run BAY -- SOFTWARE-ID:COMMAND` for managed background execution. +Both paths run as the ordinary user and participate in safe eject and unplug +cleanup. Resolve names by lowest bay, then lexical selector; expose qualified +`bBAY:SOFTWARE-ID:COMMAND` aliases without overriding base system commands. -The guest validates metadata and mounts payloads read-only, then verifies and -extracts a selected bundle to a bounded temporary read-only cache. It executes -only an explicitly requested `SOFTWARE-ID:COMMAND` as the ordinary user. It never -runs software build recipes or installation hooks. Eject, physical removal, -service restart and shutdown release caches and stop managed consumers. - -Legacy single-partition `FDS_PROGRAM` media with `app/bin`, `app/lib`, and -`app/share` remain readable. The metadata-first format supersedes that layout -for newly built software cartridges. See [Workstation usage](workstation.md) and -[the format contract](software-format.md). - -Where necessary, use: - -```text -RPATH -wrapper environment -``` - -to resolve shared libraries. - -Do not modify the SYSTEM root. +Provide `fds-control`, a native static Rust X11 panel styled to match the default +grayscale WindowMaker desktop. It displays bays and software, opens commands in +a terminal, requests rescans and safe eject, and reports service errors. --- @@ -1466,10 +1456,10 @@ FDS> fds bays # 34. fds-burn The target media client remains static-musl Rust. New software creation and -writing use native Linux workstation `fds-cartridge`: build bundles, construct +writing use native Linux workstation `fds-cartridge`: build Void source packages, construct and verify the complete metadata-first GPT image, preview the USB target, then write and verify that full image. Never assemble software partitions piecemeal -on the destination drive. See [the workstation workflow](workstation.md). +on the destination drive. See [the workstation workflow](../workstation.md). Responsible for: @@ -1512,7 +1502,7 @@ Use XBPS for: ```text building images resolving package dependencies -resolving libraries for workstation-built PROGRAM bundles +resolving libraries for installed PROGRAM package trees development profile ``` diff --git a/docs/media-tools.md b/docs/developer/media-tools.md similarity index 97% rename from docs/media-tools.md rename to docs/developer/media-tools.md index a751ace..250be1b 100644 --- a/docs/media-tools.md +++ b/docs/developer/media-tools.md @@ -1,5 +1,8 @@ # Cartridge image tools +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [Cartridge usage](cartridges.md) For new software cartridges, use the [Linux workstation workflow](workstation.md): diff --git a/docs/packages.md b/docs/developer/packages.md similarity index 97% rename from docs/packages.md rename to docs/developer/packages.md index 4de8aba..5ef3c08 100644 --- a/docs/packages.md +++ b/docs/developer/packages.md @@ -1,5 +1,8 @@ # Building and inspecting packages +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [Development](development.md) · [Glossary](glossary.md) FDS uses Void's XBPS package format and xbps-src build machinery for ordinary diff --git a/docs/performance.md b/docs/developer/performance.md similarity index 97% rename from docs/performance.md rename to docs/developer/performance.md index 0a79c89..c3cc949 100644 --- a/docs/performance.md +++ b/docs/developer/performance.md @@ -1,5 +1,8 @@ # Performance targets and measurement +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [Boot](boot.md) · [Services](services.md) **Physical Pi performance remains unmeasured.** The target budgets below are diff --git a/docs/power.md b/docs/developer/power.md similarity index 97% rename from docs/power.md rename to docs/developer/power.md index 0c2e96d..3779f69 100644 --- a/docs/power.md +++ b/docs/developer/power.md @@ -1,5 +1,8 @@ # Shutdown and reboot +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [DATA usage](data.md) · [Media tools](media-tools.md) M10 software acceptance passed; see the [validation report](m10-validation.md). diff --git a/docs/developer/recovery.md b/docs/developer/recovery.md new file mode 100644 index 0000000..57ddd62 --- /dev/null +++ b/docs/developer/recovery.md @@ -0,0 +1,210 @@ +# Recovery and rollback + +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + +[Documentation index](README.md) · [Boot images](boot.md) · [Media tools](media-tools.md) · [EEPROM](eeprom.md) + +Recovery is a separate, read-only FDS image. It contains native s6, Bash, GNU +utilities, XBPS, filesystem tools, the static FDS tools, and the base Dasung +controller. It does not need files or programs from a SYSTEM cartridge. +The software workflow is covered by `make recovery-test`; physical Pi recovery remains deferred. +The [complete internal disk](internal-storage.md) includes this recovery image; +its guide separates verified image construction from deferred physical +installation. The build commands below create ordinary files and do not write a +host disk. + +## Build the recovery image + +Use the prepared x86_64 build host described in [Your first build](getting-started.md). +From the repository root, run these commands sequentially: + +```sh +make recovery +make rootfs-test +make recovery-test +``` + +The full VM test also needs the Pi kernel/initramfs and a known-good CLI SYSTEM +image. On a fresh checkout, build those first with `make rootfs PROFILE=cli`, +`make rootfs-test`, `make initramfs`, and `make system-card PROFILE=cli`, then +run the recovery sequence above. Run builds and VM tests sequentially because +they share the same project-local Void environment. + +`make recovery` builds the `recovery` rootfs profile independently, configures all +packages and caches at image construction time, then creates EROFS. Its outputs +are: + +| File | Meaning | +| --- | --- | +| `out/rootfs-recovery.tar` | Complete recovery rootfs, including package database and compiled s6 configuration | +| `out/fds-recovery.img` | Raw EROFS payload for the internal GPT partition named `FDS_RECOVERY` | +| `out/recovery-build.*/manifest.json` | Rootfs hash, filesystem hash, profile and size | +| `out/manifests/rootfs-latest/` | Exact selected package archives and build input records | + +The latest-rootfs pointers also select this recovery build. Profile-specific +rootfs pointers preserve earlier CLI/development builds. A recovery EROFS payload +is not a whole-disk image and is not a SYSTEM cartridge. The SYSTEM builder +rejects a recovery rootfs to prevent confusing those roles. + +## Enter recovery + +Stage0 can enter recovery in two ways: + +1. At its missing-SYSTEM or ambiguous-SYSTEM console, type `recovery` and Enter. +2. For a deliberate maintenance boot, use `fds.boot=recovery` in the boot + partition's single-line `cmdline.txt`, replacing `fds.boot=normal` if present. + Restore normal mode when maintenance is complete. + +Stage0 requires exactly one readable `FDS_RECOVERY` partition and mounts it +read-only. It does not silently choose between duplicate partitions. The ARM VM +suite supplies disposable virtual partitions; physical firmware/NVMe/display +behavior must still be checked on the Pi. + +The local prompt is: + +```text +FDS RECOVERY — LOCAL MAINTENANCE CONSOLE +RECOVERY# +``` + +This is an explicit **root maintenance console**. It reads startup files from the +immutable recovery image and uses a temporary home under `/run/fds`. The root +password remains locked; no SSH or network login service is started. Normal CLI +and development images continue to use the ordinary `fds` account and `FDS>`. + +Recovery inspects cartridges read-only. It does not automatically mount DATA +writable, activate an ENVIRONMENT desktop, or enable Ethernet. Those actions +require explicit commands. The Dasung controller remains in the base boot bundle. + +## Inspect a failed system + +```sh +fds info +fds bays +fds bay 2 +fds inspect BAY02 +fds topology +fds boot-profile +lsusb -t +lsblk -o NAME,MAJ:MIN,MODEL,SERIAL,SIZE,RO,TYPE,FSTYPE,PARTLABEL,MOUNTPOINTS +dmesg +``` + +Use the calibrated bay number and the displayed model/serial to identify a +cartridge. Do not infer a bay from `/dev/sda` or from discovery order. A valid +inactive SYSTEM mounts read-only under `/run/fds/media/NN`; `fds bay N` reports +the actual path. A bad filesystem or manifest produces an error instead of +running anything from the cartridge. + +An empty bay configuration produces `UNCONFIGURED`, not guessed bay numbers. +See [Cartridges](cartridges.md) for calibration. Permanent machine configuration +on `FDS_INTERNAL` is part of the remaining M12 work. + +## Check and repair DATA + +Start with a read-only check. For example, for DATA in bay 2: + +```sh +fds recovery check BAY02 +``` + +This command is available only to root in the recovery image. It identifies one +USB disk and one `FDS_DATA` partition, unmounts its read-only view, reserves the +disk exclusively, verifies GPT and the kernel partition identity, and invokes +`e2fsck -f -n`. It does not repair the filesystem. A clean result leaves DATA +unmounted and reports `SAFE TO REMOVE`. + +If DATA was explicitly activated writable, eject that session first. Close any +shell whose current directory is on DATA and any other reader before checking; +an ordinary busy-unmount failure is reported, never bypassed with lazy unmount. + +If the check reports problems, review its log and preserve a backup where +possible. Preview the repair with: + +```sh +fds recovery repair BAY02 +``` + +The preview displays the model, serial, capacity and a confirmation command. +Copy that exact command only after checking the intended cartridge. Its token +binds the bay, kernel insertion number and current boot. A token from another +insertion or boot is rejected. The preview makes no filesystem changes. + +Confirmed repair runs `e2fsck -f -p`, which performs conservative automatic +repairs and stops when manual judgement is required. It then flushes the device, +invalidates its block cache, and runs a second `e2fsck -f -n`. Only a successful +verification produces `DATA REPAIRED AND VERIFIED` and `SAFE TO REMOVE`. + +The checker uses a temporary kernel loop device backed by the already verified +partition descriptor. This lets `e2fsck` take its own exclusive device claim +while FDS retains the physical whole-disk reservation. The loop is removed +automatically when its last descriptor closes. This uses the existing kernel +loop driver, `libc` crate and base `e2fsprogs`; no new package or Rust dependency +is introduced. + +Failed or interrupted checks/repairs retain a quarantine record across cartridge +daemon restarts for the same insertion. They do not inherit an earlier SAFE +status. The checker holds the disk reservation and is killed if its supervising +daemon dies. The command runs synchronously: bay operations and orderly shutdown +wait for it to finish; the local shell remains usable. A client timeout is not +success and does not authorize removing media. Inspect the log and wait for the +service to finish before retrying. Logs are in `/run/fds/recovery/` and are +limited to 16 MiB per invocation. + +This narrow recovery command requires a readable GPT and primary ext filesystem +signature. It deliberately does not guess partition boundaries, recreate a +broken GPT, or force answers to destructive `e2fsck` questions. Such cases require +an offline backup and expert use of the included filesystem tools. It cannot +recover data that was never written to storage. + +For structured results, prefix the command with `fds --json`. After a successful +check or repair, remove/reinsert DATA to inspect it again. Recovery still keeps +it read-only until `fds data use N` is explicitly requested. + +## Prepare replacement SYSTEM media + +A known-good SYSTEM image must be supplied on separate source media. Do not use +a damaged image as an update source. For example, source DATA in bay 2 and an +unmounted replacement cartridge in bay 4: + +```sh +fds bay 2 +fds inspect image /run/fds/media/02/fds-system-cli.img +fds inspect BAY04 +fds burn system /run/fds/media/02/fds-system-cli.img BAY04 +``` + +Use the actual source mount shown by `fds bay 2`. If the destination already +contains a mounted cartridge, run `fds eject 4` first. The burn preview identifies +the source checksum, destination capacity/model/serial and confirmation command. +Follow [Media creation and writing](media-tools.md) for confirmation and status commands. +Completion requires device flush, readback and GPT verification. Recovery uses +the same protected writer as the main system: it refuses mounted destinations, +active root storage and disks containing internal FDS partition names. + +Keep the previous known-good SYSTEM cartridge. After a successful replacement +write, use `fds poweroff`, swap SYSTEM while powered off, and boot normally. +Rollback means restoring the previous SYSTEM cartridge; it does not undo DATA +file changes, application migrations, internal boot updates or EEPROM changes. + +## Capture diagnostics and shut down + +```sh +bash /usr/share/fds/capture-hardware /tmp/recovery-capture +cat /tmp/recovery-capture/status.tsv +fds power status +fds poweroff +``` + +The collector records identities, USB topology, mounts, kernel messages, packages +and boot events with checksums. Its output is temporary unless copied to a +healthy DATA cartridge after explicitly activating that DATA with `fds data use N`. +Do not activate the damaged cartridge merely to save a report. Eject writable +DATA after copying, or let `fds poweroff` perform the normal verified shutdown. +See [Shutdown](power.md) when shutdown reports a blocking DATA error. + +Recovery is independent of SYSTEM, but it still depends on working internal +boot storage, the kernel and firmware. An external rescue medium is required +when that layer fails. EEPROM preparation and rollback are described separately +in [EEPROM configuration](eeprom.md); no EEPROM is changed by these recovery tools. diff --git a/docs/developer/releases.md b/docs/developer/releases.md new file mode 100644 index 0000000..ed17e57 --- /dev/null +++ b/docs/developer/releases.md @@ -0,0 +1,170 @@ +# Release signatures and verification + +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + +[Documentation index](README.md) · [M12 evidence](m12-validation.md) + +`fds-release` signs an artifact manifest and verifies its signature and every +listed file. The static ARM executable is included in the base CLI package, so +CLI, development and recovery images can all verify downloads. The workstation +build supplies an x86_64 signer with the same format. + +The signing tool has passed host, static ARM and independent OpenSSL acceptance. +The complete frozen-input, versioned local 0.1.0 release passed acceptance; +see [M12 validation](m12-validation.md#final-local-release-acceptance) for its identity +and [Offline rebuilds](reproducible-builds.md) for the workflow. +The examples below describe the working signing interface; they do not identify +a published or hardware-qualified FDS release. + +## Build and test the tools + +After [bootstrap](getting-started.md), run: + +```sh +make signing +make signing-test +``` + +The workstation executable is +`target/x86_64-unknown-linux-gnu/release/fds-release`. The ARM executable is +`target/aarch64-unknown-linux-musl/release/fds-release`; `make tooling` also +exports it to `out/fds-release`. An ARM file cannot execute directly on the +x86_64 workstation. `make signing-test` runs that file through the project-local +QEMU interpreter as well as testing the host executable. + +OpenSSL is a **host test dependency** used as an independent Ed25519 +implementation. It is not linked into the FDS verifier. The test creates private +disposable keys under its `out/m12-signing.*` directory, checks exact signatures, +then exercises wrong keys, altered manifests, damaged or missing artifacts, +symlinks, FIFOs, malformed fields and unsafe paths. Test keys are not release +keys. + +## Create and retain a signing key + +For a local signing identity, choose an existing private directory outside the +checkout and its build output. This example creates a new directory; choose a +different name if it already exists: + +```sh +mkdir -m 700 "$HOME/fds-signing" +target/x86_64-unknown-linux-gnu/release/fds-release keygen \ + "$HOME/fds-signing/release-01" +``` + +The command creates `release-01.key` (32 secret bytes, mode 0600) and +`release-01.pub` (the hexadecimal public key). Existing files are never +overwritten. It prints SHA-256 of the decoded 32-byte public key, never the +secret. This fingerprint differs from `sha256sum release-01.pub`, which hashes +the hexadecimal text and its newline. Retain the private key separately from release downloads and source +archives. Anyone who has that key can sign a release under that identity. + +If public-key export failed after private-key creation, retain the private file +and export it to a new path: + +```sh +target/x86_64-unknown-linux-gnu/release/fds-release public-key \ + "$HOME/fds-signing/release-01.key" "$HOME/fds-signing/recovered.pub" +``` + +Private keys must be regular files owned by the invoking user with no group or +other permissions. The tool rejects symlinks and invalid key lengths. Key +generation uses Linux `getrandom`; temporary seed storage and the signing key +are zeroized when dropped. + +## Sign an assembled directory + +An assembled release directory must already contain its artifacts and a +`manifest.json` matching the format below. Signing verifies every artifact's +length and hash before creating `manifest.sig`: + +```sh +target/x86_64-unknown-linux-gnu/release/fds-release sign \ + /path/to/release-directory --key "$HOME/fds-signing/release-01.key" +``` + +If a signature already exists, use a new release directory. The tool deliberately +does not replace signatures or silently rewrite the manifest. A failed operation +may leave its new output file for inspection; it never declares it verified. + +## Verify before using images + +Obtain the signer's public key through a channel you already trust, or compare +its fingerprint against a separately authenticated value. A public key included +beside an untrusted download does not establish its identity by itself. + +On FDS, including recovery: + +```sh +fds-release verify /data/downloads/fds-release --key /data/keys/fds-release.pub +``` + +On the workstation, use the host executable instead: + +```sh +target/x86_64-unknown-linux-gnu/release/fds-release verify \ + /path/to/release-directory --key /path/to/trusted-fds-release.pub +``` + +A successful command reports `VERIFIED FDS/OS 0.1.0`, the number of checked +artifacts, and `Hardware validation: deferred`. Failure returns exit status 2 +and an explanation. Keep downloaded images unchanged between verification and +use. Verification covers only files listed in the manifest; unrelated extra +files are not endorsed. The [media writer](media-tools.md) performs its own +image/layout validation and readback checks when writing a cartridge. + +These are distribution signatures. The Pi firmware and stage0 currently do not +enforce them at boot, so this is not a secure-boot implementation or a claim +that the hardware has been tested. + +## Manifest and signature format + +The JSON document has exactly these fields: + +```json +{ + "format": 1, + "version": "0.1.0", + "source_epoch": 1789909701, + "source_sha256": "<64 lowercase hexadecimal characters>", + "void_commit": "02a3cbc132c3c4a3a9d59e9b98f517af5dd11cd1", + "hardware_validation": "deferred", + "files": [ + { + "name": "fds-system-cli-0.1.0.img", + "bytes": 123456, + "sha256": "<64 lowercase hexadecimal characters>" + } + ] +} +``` + +This is a schema illustration, not a usable manifest: the size and hash +placeholders must be replaced with values from the actual artifact. Format 1 +accepts the current tool version, a positive source epoch, a 40-character Void +commit, and 1–64 nonempty artifacts. Filenames must be flat safe ASCII names; +paths, duplicates, hidden names, reserved manifest filenames and unknown fields +are rejected. The manifest is bounded to 1 MiB. Files must be regular files, +opened without following symlinks; hashing streams their bytes and checks for +changes during the read. + +The signature is Ed25519 over the exact bytes +`FDS/OS release manifest v1` followed by a NUL byte and the raw `manifest.json` +bytes. JSON whitespace is therefore authenticated too. `manifest.sig` contains +128 lowercase hexadecimal characters and a newline. The public-key file contains +64 lowercase hexadecimal characters and a newline. Verification rejects weak +public keys and uses strict signature verification. + +The implementation uses locked `ed25519-dalek` 3.0.0 for Ed25519, +`zeroize` 1.9.0 for secret buffers, and the existing SHA-256/JSON crates. +Its locked transitive dependencies provide curve arithmetic, digest and +signature types. Default dalek features are disabled; no asynchronous runtime, +OpenSSL binding or shared target library is added. The dependencies and their +license metadata are recorded in `Cargo.lock` and the cached crate sources. +The build collects their license texts, including build dependencies and +vendored libusb notices, into `/usr/share/licenses/fds-cli/RUST-NOTICES.txt`. +The usage guide is installed at `/usr/share/doc/fds/releases.md` in every profile. +Algorithm interoperability is checked against the +[RFC 8032 test vector](https://www.rfc-editor.org/rfc/rfc8032.html#section-7.1) +and OpenSSL; strict verification follows the +[dalek API](https://docs.rs/ed25519-dalek/3.0.0/ed25519_dalek/struct.VerifyingKey.html). diff --git a/docs/reproducible-builds.md b/docs/developer/reproducible-builds.md similarity index 98% rename from docs/reproducible-builds.md rename to docs/developer/reproducible-builds.md index eb1f333..f23a17b 100644 --- a/docs/reproducible-builds.md +++ b/docs/developer/reproducible-builds.md @@ -1,5 +1,8 @@ # Frozen inputs and offline rebuilds +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [Build host](build-host.md) · [Release signatures](releases.md) M12 adds a local input snapshot, fresh restore, network-isolated build, and diff --git a/docs/roadmap.md b/docs/developer/roadmap.md similarity index 97% rename from docs/roadmap.md rename to docs/developer/roadmap.md index 2daf3e0..7b0d5b5 100644 --- a/docs/roadmap.md +++ b/docs/developer/roadmap.md @@ -1,5 +1,8 @@ # Roadmap and current status +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [Master plan](master-plan.md) · [M0 evidence](m0-validation.md) **M0–M12 software acceptance is complete. Physical Pi testing is deferred.** diff --git a/docs/rootfs.md b/docs/developer/rootfs.md similarity index 98% rename from docs/rootfs.md rename to docs/developer/rootfs.md index 723ef5b..0f9dd36 100644 --- a/docs/rootfs.md +++ b/docs/developer/rootfs.md @@ -1,5 +1,8 @@ # Building and using the ARM root filesystem +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [First build](getting-started.md) · [Validation](m1-validation.md) The assembler produces a configured **aarch64 glibc Linux userspace** at diff --git a/docs/services.md b/docs/developer/services.md similarity index 97% rename from docs/services.md rename to docs/developer/services.md index 68362c4..3f7b5dd 100644 --- a/docs/services.md +++ b/docs/developer/services.md @@ -1,5 +1,8 @@ # Services, readiness, and shutdown +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [Native init and ARM VM](init.md) · [Cartridges](cartridges.md) **Implemented in M2:** native s6 PID 1, a compiled base service graph, console diff --git a/docs/developer/software-format.md b/docs/developer/software-format.md new file mode 100644 index 0000000..6402feb --- /dev/null +++ b/docs/developer/software-format.md @@ -0,0 +1,120 @@ +# Software cartridge format 1 + +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + +[Build and use a cartridge](workstation.md) · [Original cartridge classes](cartridges.md) + +A software cartridge is a complete GPT disk image with **1 + m** partitions, +where `m` is 1–32. Partition 1 describes every software package in all payload +partitions. All partitions use read-only EROFS. This format is the current +PROGRAM creation format; existing single-partition PROGRAM media remain readable. +SYSTEM, DATA and ENVIRONMENT layouts retain their existing meanings. + +## Disk and filesystem layout + +```text +GPT protective MBR + primary GPT + 1 FDS_METADATA + FDS/CARTRIDGE.TOML + FDS/SOFTWARE.TOML + 2 FDS_PAYLOAD02 + bundles/demo.hello.tar.xz + bundles/demo.editor.tar.xz + 3 FDS_PAYLOAD03 + bundles/demo.report.tar.xz + ... +backup GPT +``` + +Partition entries are consecutive, use Linux filesystem type GUIDs, unique +nonzero partition UUIDs, no GPT attributes, and nonoverlapping MiB-aligned +extents. Partition 1 starts at LBA 2048. Both GPT copies, their headers and CRCs +must agree. A full-drive write to a larger USB device relocates the backup table +and header. The existing on-target image parser and the workstation writer +share these checks. + +`FDS/CARTRIDGE.TOML` uses the existing format-1 identity with `class = "program"` +and `media.writable = false`. It contains no startup command. A sample catalogue: + +```toml +format = 1 + +[[software]] +id = "demo.hello" +name = "AArch64 hello" +version = "1.0" +architecture = "aarch64" +partition = 2 +archive_bytes = 2308 +unpacked_bytes = 70504 +entries = 2 +sha256 = "REPLACE_WITH_THE_64_CHARACTER_LOWERCASE_SHA256" +[software.commands] +hello = "bin/hello" +``` + +Those lengths and the digest are illustrative. The builder generates actual +values. IDs and command names use the existing restricted FDS identifier syntax. +IDs are unique per cartridge. Every declared payload partition must contain +exactly the catalogue's `bundles/.tar.xz` archive names, and every payload +partition must be represented. Multiple software entries may share a partition. +The catalogue contains 1–128 software entries and at most 64 KiB of TOML. + +## Archive contract + +Bundles are deterministic USTAR archives compressed with single-threaded xz. +Entries are sorted, timestamps/UID/GID are zero, and permissions normalize to +0755 for directories/executables or 0644 for ordinary data. Host source symlinks +to regular files inside the root are flattened into regular files. Directory or +escaping symlinks are rejected; the archive itself never contains links. + +The reader rejects: + +- Absolute, parent-traversing, repeated-separator, non-UTF-8 or control-character paths. +- Duplicate entries, file-as-parent conflicts, links, devices, sockets, FIFOs, + privileged permissions, and GNU/PAX extension entries. +- Incorrect SHA-256, compressed length, unpacked length, entry count, or command + paths that do not name executable regular files. +- ELF files that are not little-endian 64-bit AArch64, or any ELF in a bundle + declared `architecture = "any"`. +- Invalid xz data, nonzero material after the tar end marker, and resource-limit + violations. Each archive is at most 512 MiB compressed, 1 GiB unpacked and + 65,536 entries. Xz decompression has a 256 MiB memory limit. + +A source path must fit USTAR's path fields. Packaging reports paths that cannot +be represented rather than emitting an unsupported extension header. + +## Guest lifecycle + +On insertion the daemon validates the complete GPT against kernel partition +geometry and the current USB disk identity. It reads metadata without following +symlinks, mounts every payload read-only/noexec/nosuid/nodev, and checks the +archive inventory and lengths. `fds bay BAY` reports the full catalogue; +`fds bays` stays compact. No archive command runs automatically on insertion. + +`fds run BAY -- SOFTWARE-ID:COMMAND [ARGUMENT...]` verifies and extracts that +package on first use. It creates a private temporary filesystem, applies the +archive contract, then makes the finished tree read-only and accessible for +execution. The runtime cache has a stricter **256 MiB per software tree** limit, +including reserved inode overhead; a host-valid larger bundle may therefore +need splitting before running. Caches live under `/run/fds/software//` and +vanish after eject, unplug, restart cleanup, or shutdown. + +Managed consumers run as UID/GID 1000 with no new privileges in the bay's cgroup. +`FDS_APP`, `PATH`, `LD_LIBRARY_PATH` and `XDG_DATA_DIRS` point to that software +root. DISPLAY/XAUTHORITY retain the established optional desktop integration. +The xz utility is already an explicit dependency of the mandatory `fds-base` +package; runtime extraction adds no target package. + +Build recipes, compilers, installation hooks and privileged archive scripts are +never part of this runtime path. + +Safe eject stops consumers, rejects unexpected mount aliases, unmounts caches, +payloads and metadata, then reports SAFE. Confirmed surprise removal permits +lazy detachment after stopping consumers. Service restart removes stale mounts +before rescanning. SYSTEM stays immutable throughout. + +SHA-256 detects corruption and binds an archive to its catalogue; it does not +establish a publisher's identity. Only run software you trust. Release signature +verification remains the separate [release workflow](releases.md). diff --git a/docs/stress-testing.md b/docs/developer/stress-testing.md similarity index 96% rename from docs/stress-testing.md rename to docs/developer/stress-testing.md index bace601..ea7bc83 100644 --- a/docs/stress-testing.md +++ b/docs/developer/stress-testing.md @@ -1,5 +1,8 @@ # Twelve-bay stress tests +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [Cartridge configuration](cartridges.md) · [Shutdown](power.md) M11 exercises twelve devices together and failures during media writing. The @@ -86,7 +89,7 @@ stress, use test media whose contents can be replaced. Any `fds burn` or ## Capture evidence on the target -Copy [capture-hardware](../tools/capture-hardware) to the DATA cartridge. At the +Copy [capture-hardware](../../tools/capture-hardware) to the DATA cartridge. At the FDS console, run it explicitly through Bash; DATA intentionally does not permit direct execution: @@ -109,7 +112,7 @@ identifiers before sharing the files. ## Physical acceptance sequence -Copy the [physical session template](../tests/hardware/session-template.json) +Copy the [physical session template](../../tests/hardware/session-template.json) and fill it with observed values. Null means unknown, not zero. Record every run, including failures. The following is an initial repeatable matrix, not evidence that it has already passed: diff --git a/docs/tooling.md b/docs/developer/tooling.md similarity index 97% rename from docs/tooling.md rename to docs/developer/tooling.md index fc81d46..af48b21 100644 --- a/docs/tooling.md +++ b/docs/developer/tooling.md @@ -1,5 +1,8 @@ # FDS Rust tools +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + [Documentation index](README.md) · [Implementation ledger](implementation-status.md) M3 introduces `fds-common`, the `fds` command, and the `fds-stage0` diagnostic diff --git a/docs/developer/troubleshooting.md b/docs/developer/troubleshooting.md new file mode 100644 index 0000000..9aaf9c4 --- /dev/null +++ b/docs/developer/troubleshooting.md @@ -0,0 +1,246 @@ +# Troubleshooting the build + +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + +[Documentation index](README.md) · [First build](getting-started.md) · [Build reference](build-host.md) + +Find the first failing command or `ERROR:` in the log, fix that problem, then +repeat the same step. Do not continue past a failed bootstrap and interpret later +missing files as independent problems. Run all commands below from the repository +root, as your regular user, unless explicitly shown otherwise. + +## Find the right log + +| Failing action | Where to look | +| --- | --- | +| `make bootstrap` | `out/logs/bootstrap.log` | +| Void build-container creation | `out/logs/xbps-bootstrap.log` | +| `make smoke-test` | `out/logs/smoke-test.log` | +| GNU hello build | `out/logs/xbps-hello.log` | +| `make check` | Terminal output; this target does not save a log automatically | + +For example, after a failed smoke-test: + +```sh +tail -n 80 out/logs/smoke-test.log +rg -n 'ERROR:|FAILED|error:|No space left|Permission denied' out/logs/ +df -h . +``` + +A search with no matches exits 1. Read the surrounding log too: upstream tools +may report failures using other wording. Save logs before re-running if you need +the previous output; see [Development](development.md#preserve-logs-and-identify-inputs). + +## Unsupported host, root execution, or missing commands + +- `M0 requires an x86_64 Linux build host` or `supports Arch Linux`: use the + supported host described in [Your first build](getting-started.md). Changing + the check alone does not port the build system to another platform. +- `Run as a normal user, not root`: leave the root shell and run the build from + your normal account. Only host package installation uses sudo. +- `Missing host command`: install the prerequisites from the first-build guide, + then repeat bootstrap. `readelf` comes from binutils, `bwrap` from bubblewrap, + and `flock` from util-linux. +- `checkout path without whitespace`: move or obtain the checkout at a path + with no spaces, tabs, or other whitespace, then use that directory consistently. + +If a previous root-run left files that your account cannot write, inspect their +ownership before correcting it. Do not respond by running all later builds as root. + +## Bubblewrap or user namespace failure + +Reproduce the same capability check bootstrap uses: + +```sh +bwrap --ro-bind / / --unshare-user --uid 0 --gid 0 true +``` + +If it reports `Operation not permitted` or a namespace error, the host kernel, +container policy, or security configuration prevents the unprivileged container. +Use an Arch host that permits this operation, or have the host administrator +resolve the restriction. Repeated downloads, sudo builds, and a different Pi +will not resolve this preflight failure. Success is a zero exit status with no +output; run `make bootstrap` again once that works. + +## Void submodule missing, wrong, or modified + +Inspect before changing anything: + +```sh +cat VOID_PACKAGES_COMMIT +git submodule status vendor/void-packages +git -C vendor/void-packages status --short +git -C vendor/void-packages diff HEAD -- +``` + +If the submodule is missing, `make bootstrap` initializes it. The directory must +be part of a Git checkout; an unpacked source archive is insufficient. If a clone +was interrupted by a network error, restore connectivity and repeat bootstrap. + +`Void checkout mismatch` means the checked-out commit differs from the explicit +pin. A `+` in `git submodule status` means it differs from the parent repository's +gitlink. These are two separate checks. Do not update the pin merely to silence +the error; first determine why the commits differ. + +`Tracked Void files were changed` means upstream source has local edits. Preserve +and review the diff. Move intentional FDS changes into the overlay design or a +separate patch for review; restore upstream only after saving work you need. +Generated ignored files such as `etc/conf`, `hostdir`, and the masterdir are +normal. Never use a broad reset or clean command without checking what it removes. + +## Configuration or overlay conflict + +For `etc/conf differs from config/xbps-src.conf`, compare the two: + +```sh +diff -u vendor/void-packages/etc/conf config/xbps-src.conf +``` + +Put the desired settings in `config/xbps-src.conf`, then follow +[the explicit synchronization procedure](development.md#change-build-configuration-deliberately). +A `diff` exit status of 1 means differences were found, not that the comparison failed. + +For `Stale overlay`, compare the named `vendor/void-packages/srcpkgs/fds-*` +directory with its source under `packages/`. Preserve any unique edits in the +source first. Move that specific generated copy into a backup directory under +`out/`, outside `srcpkgs/`, then run `./tools/prepare-void` again. Do not move or +remove unrelated upstream packages. The Dasung package is now an active overlay. After changing it, reconcile that +generated copy by this same procedure before rebuilding. + +`Overlay would replace an upstream package` is an intentional protection. Choose +an FDS-owned package name and design an overlay; do not disable the check or edit +the tracked upstream template in place. + +## Download failure, checksum mismatch, or apparent stall + +Bootstrap and package builds need network access. The first run can spend much +of its time downloading compiler packages. Read the latest log and check disk +space before assuming that a quiet download is a deadlock: + +```sh +tail -n 40 out/logs/bootstrap.log +tail -n 40 out/logs/xbps-hello.log +du -sh out/downloads vendor/void-packages/hostdir +df -h . +``` + +A log or directory may not exist if its step has not started. Bootstrap's static +XBPS download has retries and a timeout; an incomplete transfer uses a `.part` +filename. A retry re-downloads that partial file. Completed cached archives are +checksum-checked on every bootstrap. + +For a checksum failure, retain the error and compare the named file against +`config/host-tools.conf`. If the cached completed archive is corrupt, move that +specific archive to a backup location and repeat bootstrap so it downloads again. +Do not change the expected hash to match an unexplained download. If a clean +re-download still disagrees, stop and investigate the source. + +A Void mirror timeout is separate from the source pin: the templates are pinned, +but bootstrap and missing build dependencies still use rolling binary repositories. +An unavailable historical dependency may need a reviewed input/mirror solution; +blindly changing the source commit does not establish an equivalent build. + +## XBPS command not found or package index seems empty + +The XBPS tools are intentionally not installed in the host's `/usr/bin`. +`make bootstrap` creates `.host/xbps/usr/bin/`. FDS build helpers set PATH for you. +For manual queries, use the explicit paths in the [package guide](packages.md). + +When querying or indexing ARM output, set `XBPS_ARCH=aarch64`. The host's static +musl executable can otherwise default to its own architecture and ignore the ARM +package. `make smoke-test` uses the correct override. `tools/build-package` exports +packages but does not update `out/packages/aarch64-repodata`; use the indexing +command in the package guide after a standalone package build. + +## Rust target, linker, or architecture error + +Check the selected toolchain and local configuration: + +```sh +rustup show active-toolchain +rustup target list --installed +cat rust-toolchain.toml +cat .cargo/config.toml +printenv RUSTFLAGS CARGO_BUILD_TARGET CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER +``` + +`printenv` exits nonzero when a listed variable is unset; that is normal here. +Bootstrap installs the required target and rustfmt for the pinned toolchain. +Repeat it if those components are missing. + +Build from the repository root with the exact Cargo command in +[Development](development.md#edit-and-rebuild-the-rust-program). Environment +variables, command-line `+toolchain` overrides, or outside Cargo configuration +can change the target, flags, or linker. Reconcile such overrides with the +checked-in `rust-lld` and static CRT configuration. Adding an arbitrary system +linker or glibc library is not a repair for a static-musl build. + +## Exec format error, ldd output, and QEMU skips + +`./out/fds-smoketest` normally gives `Exec format error` on x86_64: it is an ARM +program. Run it explicitly with optional `qemu-aarch64`, as described in +[the first-build guide](getting-started.md#7-optionally-execute-the-arm-rust-program). +Do not change the target to x86_64 merely to make direct execution work. + +`ldd: not a dynamic executable` with exit 1 is expected for the static program. +Use `./tools/verify-elf out/fds-smoketest aarch64 static` for the actual check. +Host `ldd` cannot establish that a foreign ARM program will run. + +`SKIP: ARM execution` means QEMU is absent. Compilation and ELF checks still run. +If QEMU is present but fails, the smoke-test must fail; inspect its diagnostic. +The dynamic GNU hello program also requires an ARM glibc loader and libraries, +so the simple static-program QEMU command is not a complete hello runtime setup. + +## Missing artifacts or check fails before any build + +`make check` expects the Cargo release artifact in `target/`. Run the Rust build +or `make smoke-test` first. A bare Cargo build updates `target/`, while +`make smoke-test` also refreshes `out/fds-smoketest` and the output manifests. +If checksums fail after editing or replacing outputs, inspect what changed and +rebuild through smoke-test; do not edit the manifest to conceal a mismatch. + +For a Rust formatting failure, run `cargo fmt --all`, inspect the source diff, +then repeat `make check`. + +## Busy /tmp message during Void bootstrap + +The initial verified bootstrap emitted +`rm: cannot remove '//tmp': Device or resource busy` during upstream cleanup. +The bubblewrap bind mount explained that instance; upstream exited successfully, +and container GCC, the cross build, and the public M0 commands all passed. + +Do not treat every later `/tmp` error as harmless. Confirm a zero command exit +status and the final bootstrap PASS, then run smoke-test. If the command exits +nonzero, report it as a failure with the surrounding log. + +## Rootfs, images, and runtime commands + +M1 implements `make rootfs PROFILE=cli` and `make rootfs-test`. If the target is +missing, check that you are in the current checkout. Read [M1 rootfs](rootfs.md) +for prerequisites, shell access, logs, and retained failed-build directories. +Python 3.14+ is required for XBPS archive inspection. Supported profiles are +`cli`, `development`, and `recovery`; unknown names fail explicitly. Package +configuration must pass before an archive is published. + +`tools/in-rootfs` supports ARM child execution through a private user-namespace +binary-format handler. It requires Linux 6.7+ and namespaced `binfmt_misc` support. +The workstation's global registry is unchanged. For a single ELF executable, +`--direct` invokes QEMU without installing even a private child-execution handler. +For example: `./tools/in-rootfs out/rootfs-aarch64 --direct /usr/bin/xbps-pkgdb -a`. + +The tar is configured userspace, not a flashable image. M2 adds `make init-test` +for a full ARM boot and `make vm` for a temporary development shell. Follow +[Native init and ARM VM](init.md) for its dependencies, expected output, and +failure logs. For the full kernel/stage0/SYSTEM path, use `make all`, +`make boot-test`, then `make console-vm`. Run `fds bays` and `fds poweroff` +inside the booted FDS console, not on the Arch host. An exported ARM binary will +not execute directly on x86_64. Physical Pi acceptance remains deferred. + +## What to include in a bug report + +Include the exact failing command, its exit status if available, the relevant log +excerpt, host distribution and architecture, current Void pin, and whether this +is the first build or a previously working checkout. Mention local configuration +changes and toolchain overrides. Preserve the initial error; a later missing-file +message often only describes its consequence. diff --git a/docs/developer/workstation-tooling-plan.md b/docs/developer/workstation-tooling-plan.md new file mode 100644 index 0000000..cd0102d --- /dev/null +++ b/docs/developer/workstation-tooling-plan.md @@ -0,0 +1,24 @@ +# Workstation tooling plan + +The current source of requirements is [the active revision](current-revision.md). +The public tools remain native Linux Clap applications: `fds-cartridge` and +`fds-emulator`. Full OS cross builds keep their separate pinned Void environment. + +New software creation uses Void source templates and `xbps-src`. The image builder +installs packages and dependencies on the workstation, verifies installed trees, +and writes them directly into EROFS program partitions with format-2 metadata. +Creation no longer generates xz software bundles. Legacy readers stay available. + +The emulator still uses actual FDS ARM kernel/userspace, twelve virtual USB bays, +read-only SYSTEM/PROGRAM disks, private writable DATA overlays, the public serial +console, safe eject, forced removal and native shutdown. Hardware timing and +physical behavior require the procedures in the hardware notes. + +Acceptance must cover real source-package builds, image integrity and write/readback, +foreground PATH commands and terminal behavior, background `fds run`, eject/unplug, +service restart and the rendered X11 control panel. Keep the frozen 0.1.0 release +unchanged. Build new matching artifacts and record their hashes and source inputs. + +See [user instructions](../workstation.md) and [format reference](../software-format.md) +for the supported interface. Historical xz acceptance remains in +[the earlier workstation record](workstation-validation.md). diff --git a/docs/workstation-validation.md b/docs/developer/workstation-validation.md similarity index 97% rename from docs/workstation-validation.md rename to docs/developer/workstation-validation.md index d65c0ff..acd700c 100644 --- a/docs/workstation-validation.md +++ b/docs/developer/workstation-validation.md @@ -1,5 +1,8 @@ # Workstation extension acceptance +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + **Software acceptance passed on 2026-09-21.** This record covers the new native Linux software/cartridge builder, whole-image writer, public QEMU emulator and guest software runtime. It is separate from the immutable local 0.1.0 release. diff --git a/docs/developer/workstation.md b/docs/developer/workstation.md new file mode 100644 index 0000000..7b0bb23 --- /dev/null +++ b/docs/developer/workstation.md @@ -0,0 +1,349 @@ +# Build software cartridges and run FDS in QEMU + +Development reference and historical context. For current operating instructions, use the [user manual](../README.md). Acceptance applies only to the source and artifacts identified in each record. + + +This guide uses a **Linux workstation**, not the Raspberry Pi. It takes you from +source software to a cartridge disk image, runs that image in FDS, and explains +how to write the same complete image to USB. All commands run from the checkout +root unless marked as guest commands. + +The two native host tools are `fds-cartridge` and `fds-emulator`. They use typed +Clap interfaces; append `--help` to any command or subcommand. The target's `fds` +command remains a static AArch64 executable. The new host tools are never part +of the Pi base image. Dasung support remains in the base boot bundle. + +The accepted local `out/fds-os-0.1.0/` release predates this extension. Leave that +release intact and build current images for the emulator. See +[workstation acceptance](workstation-validation.md) for current evidence. + +For accumulated build and test images, use `make clean-preview` and `make clean`. +Current images, native tools, latest test fixtures and personal emulator sessions +are kept; obsolete automated test sessions are disposable. See +[Cleanup](cleanup.md) for the exact scope and how to preserve an older test run. + +## Install and build the workstation tools + +Use a normal Linux account with Rust/Cargo, a C linker, Bash and Make. The +repository's `rust-toolchain.toml` selects the Rust version. Runtime prerequisites +are: + +| Tool | Purpose | +| --- | --- | +| `qemu-system-aarch64` | Runs the FDS ARM kernel and userspace using portable TCG | +| `qemu-img` | Creates a separate writable overlay for DATA; source images stay unchanged | +| `mkfs.erofs`, `fsck.erofs` from erofs-utils | Build and verify cartridge partitions; tested with 1.9.4 | +| `xz` | Compress/decompress software tarballs with a decompression memory limit | +| `bwrap` from bubblewrap | Restricts filesystem inspection to a private writable staging directory | +| A software-specific cross compiler | Builds AArch64 applications; unnecessary for portable scripts | +| `sfdisk`, `tar`, `mke2fs`, Python 3 | Independent integration tests; not needed to boot an existing image | + +Install these with your distribution's package manager. Unprivileged user +namespaces must work for bubblewrap. Building **these host tools** does not need +Void, XBPS, an Arch host, a Pi, root, or a running FDS guest: + +```sh +make workstation +export PATH="$PWD/out/workstation:$PATH" +fds-cartridge doctor +fds-emulator doctor +``` + +Current acceptance ran on Arch Linux x86_64. The tools use native Linux +interfaces and distribution-provided utilities; other distributions and ARM +workstations have not yet been exercised by this acceptance run. + +The outputs are native binaries in `out/workstation/`. The build explicitly +selects the workstation architecture instead of the workspace's default ARM +target. Rust dependencies are pinned in `Cargo.lock`. `tar` 0.4.46 and its +`filetime` dependency provide archive decoding/creation; FDS additionally checks +entry types, paths, counts, sizes, executable architecture and hashes. + +For this repository's **existing Arch/Void setup**, reuse its local tools: + +```sh +make workstation +./tools/prepare-image-tools +./tools/in-void xbps-install -y qemu-img +out/workstation/fds-cartridge --image-tool-runner tools/in-image-tools doctor +out/workstation/fds-emulator doctor --qemu-runner tools/in-void +``` + +The install command above changes only the project-local build container. The +optional runners accept a program name followed by its arguments; omitting them +uses programs in the workstation's `PATH`. Set the image runner on every +cartridge create, inspect or preview invocation that needs it. A QEMU runner is +saved in the session and reused for later DATA insertions. + +## Build two software bundles + +Each software recipe declares an id, display name, version, target architecture, +root directory, and named executable commands. A trusted `[build]` section may +run a workstation compiler. It is executed only by `software build`, never by +`software pack`, inspection, or a cartridge insertion. + +The [hello recipe](../../examples/software/hello/software.toml) builds an AArch64 +C executable. Install an AArch64 glibc cross compiler as `aarch64-linux-gnu-gcc`, +then run: + +```sh +fds-cartridge software build examples/software/hello/software.toml out/demo-hello +fds-cartridge software pack examples/software/report/software.toml out/demo-report +fds-cartridge software inspect out/demo-hello +``` + +If your cross compiler has a different name, edit the recipe's `build.command`. +With this checkout's existing Void cross toolchain, the equivalent is: + +```sh +mkdir -p examples/software/hello/root/bin +./tools/in-void aarch64-linux-gnu-gcc -O2 examples/software/hello/hello.c \ + -o examples/software/hello/root/bin/hello +out/workstation/fds-cartridge software pack \ + examples/software/hello/software.toml out/demo-hello +out/workstation/fds-cartridge software pack \ + examples/software/report/software.toml out/demo-report +``` + +Use **one** of those hello workflows. Outputs must not already exist: choose +new output names when rebuilding, then update your cartridge recipe. The example +bundles each contain `software.toml` and `.tar.xz`. The archive contains the +contents of the software root, such as `bin/hello`; it has no leading `root/`. + +For your own application, copy a recipe and populate its root with `bin/`, +`lib/` and `share/` as needed. C/C++ glibc builds target AArch64; static Rust +programs should use `aarch64-unknown-linux-musl`. Set `architecture = "any"` only +for scripts/data without ELF files. An x86_64 executable is rejected. Shared +libraries must be compatible with the target SYSTEM's glibc, or bundled as +appropriate. There is no cross-cartridge dependency resolver. + +## Assemble the cartridge image + +The example cartridge recipe groups the two bundles into two payload partitions: + +```sh +fds-cartridge create examples/software/cartridge.toml out/demo-tools.img +fds-cartridge inspect out/demo-tools.img +``` + +For the project-local EROFS tools, use: + +```sh +out/workstation/fds-cartridge --image-tool-runner tools/in-image-tools \ + create examples/software/cartridge.toml out/demo-tools.img +``` + +The resulting complete disk image has **three GPT partitions**: + +| Partition | GPT name | Contents | +| --- | --- | --- | +| 1 | `FDS_METADATA` | Cartridge identity and the software catalogue | +| 2 | `FDS_PAYLOAD02` | `bundles/demo.hello.tar.xz` | +| 3 | `FDS_PAYLOAD03` | `bundles/demo.report.tar.xz` | + +To put both programs in one payload partition, use one `[[payload]]` entry with +`bundles = ["../../out/demo-hello", "../../out/demo-report"]`. This makes a +two-partition image. Each payload group creates exactly one partition. The first +partition identifies every software bundle and its payload partition. + +Creation finishes only after verifying both GPT tables, all EROFS partitions, +every archive and the catalogue. Inspection prints JSON, including offsets, +lengths, SHA-256 and available commands. Repeating an unchanged recipe produces +the same image bytes. See [the format](software-format.md) for limits and rules. + +## Boot current FDS in the emulator + +You need three matching current artifacts: the FDS Pi kernel, its initramfs, and +a SYSTEM image containing the guest software runtime. Build these on the OS +build workstation using [the build guide](getting-started.md). For an already +bootstrapped checkout: + +```sh +make rootfs PROFILE=cli +make system-card PROFILE=cli +make initramfs +make workstation +out/workstation/fds-emulator --session out/my-emulator start \ + --qemu-runner tools/in-void +``` + +If QEMU is installed directly on your Linux workstation, omit `--qemu-runner`. +To use images copied from another build machine, supply their paths: + +```sh +fds-emulator --session out/my-emulator start \ + --kernel /path/to/kernel_2712.img \ + --initramfs /path/to/fds-initramfs.img \ + --system /path/to/fds-system-cli.img +``` + +Create `out/` first if using a fresh directory. The session directory must be +**new**, private, and short enough for Unix sockets (under 90 bytes including its +absolute parent path). Start waits for the FDS prompt and verifies the emulator +bay configuration. It uses two emulated CPUs and 1024 MiB RAM by default; +`--memory-mib 2048` increases RAM. `--timeout` adjusts the boot deadline. + +The session records inputs, virtual devices, serial output and QEMU diagnostics. +QEMU stays running after the command exits. SYSTEM and software images are +read-only. No physical disks, host network interface, or Pi monitor are attached. + +## Insert, use and remove cartridges + +From the workstation: + +```sh +fds-emulator --session out/my-emulator insert 01 out/demo-tools.img +fds-emulator --session out/my-emulator guest -- fds bay 01 +fds-emulator --session out/my-emulator guest -- fds run 01 -- demo.hello:hello +fds-emulator --session out/my-emulator guest -- fds run 01 -- demo.report:report +fds-emulator --session out/my-emulator guest -- tail -20 /run/log/cartridged/current +fds-emulator --session out/my-emulator eject 01 +``` + +Insertion is asynchronous: `fds bay 01` may briefly show `EMPTY` before USB +storage discovery completes. Retry the status command until it reports +`MOUNTED READ ONLY`, or inspect its error. The catalogue lists exact commands. +`fds run` starts a managed process and prints its PID; its output goes to the +cartridge service log. Packages are verified and extracted into temporary, +read-only RAM filesystems on first run. They are not compiled on the guest. + +All twelve bays, numbered `01`–`12`, are available. Simultaneously mounted +PROGRAM cartridges need distinct cartridge IDs; inserting duplicate IDs is +reported as a guest error. `eject` asks FDS to stop +managed consumers, release mounts and declare `SAFE`, then removes the virtual +USB device. If FDS refuses, QEMU leaves the cartridge attached. To deliberately +simulate an accidental pull, use: + +```sh +fds-emulator --session out/my-emulator unplug 01 +``` + +A forced unplug can lose DATA writes. The guest must then detect the removal +and clean up its processes and mounts. `status` reports the actual QEMU devices +and block nodes as well as the recorded image paths. + +Use the interactive shell if preferred: + +```sh +fds-emulator --session out/my-emulator console +``` + +At the `FDS>` prompt, type `fds bays`, `fds bay 01`, or `fds run 01 -- +demo.hello:hello`. **Ctrl-] detaches**; it does not stop the VM. Use a second +workstation terminal for insertion/unplug. Detach the console before `guest`, +`eject`, or ordinary `stop`, because they also need exclusive serial access. + +Finish with: + +```sh +fds-emulator --session out/my-emulator stop +``` + +This invokes native FDS shutdown. `stop --force` cuts virtual power without that +sequence. Logs and DATA overlays remain in the session directory. Choose a new +session directory to boot again; stopped sessions are retained for inspection, +not resumed from RAM snapshots. + +## Writable DATA images + +Insert an existing FDS DATA disk image with the same `insert` command. The +emulator creates a unique `data--.qcow2` overlay in the session directory; +all guest writes go there. The original image remains unchanged. Safe eject +retains the overlay, while inserting the original image again starts a fresh +one. Software cartridges remain read-only. + +After eject or shutdown, export an overlay if you need the changed DATA contents: + +```sh +qemu-img convert -f qcow2 -O raw /path/to/session/data-02-IDENTIFIER.qcow2 out/saved-data.img +``` + +Keep the original backing image at its recorded path until conversion completes. +Never convert or edit an image while it is attached to a running VM. + +## Write the completed image to USB + +The burn flow always starts with the complete image created above. It does not +construct partitions directly on a drive. Identify the intended **whole USB +drive**, unmount it, then make a preview as your normal user: + +```sh +fds-cartridge preview out/demo-tools.img /dev/sdX out/usb-preview.json +``` + +Replace `/dev/sdX` with the actual whole USB drive. The JSON records its model, +size, insertion identity, exact image hash and an exact `confirmation` string. +Review these before copying the full phrase into the write command: + +```sh +sudo /absolute/path/to/fds-cartridge write out/usb-preview.json \ + --confirm 'COPY THE EXACT confirmation VALUE FROM THE PREVIEW' +``` + +The writer rechecks source and target identity, requires a USB whole disk, +rejects mounted/protected storage, writes the **full image**, flushes and verifies +readback. On larger drives it relocates the backup GPT to the end. A replaced +USB drive or changed image requires a fresh preview. This destroys the selected +drive's existing contents. No physical drive has been written by the automated +acceptance tests. + +To rehearse on a disposable file without root or USB hardware: + +```sh +truncate -s 64M out/disposable-usb.img +fds-cartridge preview out/demo-tools.img out/disposable-usb.img \ + out/file-preview.json --file-target +fds-cartridge write out/file-preview.json --confirm 'COPY THE EXACT confirmation VALUE' +fds-cartridge inspect out/disposable-usb.img +``` + +The file must be at least as large as the source image. Use `--image-tool-runner` +for preview/inspection if EROFS tools are provided by the local wrapper. + +## Troubleshooting + +| Symptom | What to do | +| --- | --- | +| Required executable not found | Run the corresponding `doctor`; install the named host utility or select an explicit runner | +| Bubblewrap namespace failure | Enable unprivileged user namespaces according to your workstation policy; do not run creation/inspection as root | +| Output already exists | Use a new bundle/image/preview/session name; creation does not overwrite outputs | +| Unknown `fds.emulator` option or missing emulator settings | Rebuild both the current initramfs and SYSTEM; the frozen 0.1.0 images predate this feature | +| Console in use | Detach with Ctrl-] before guest commands, safe eject, or shutdown | +| Guest command timed out | Inspect `console.log`; the command may still be running, so do not blindly repeat a write | +| Incomplete insertion | Run `unplug BAY` to reconcile the recorded intent with actual QEMU devices, then reinsert | +| Software digest/path/architecture error | Rebuild the bundle and cartridge on the workstation; the guest will not run an invalid archive | +| Cache limit exceeded | Reduce the software bundle; one runtime tree including inode overhead is limited to 256 MiB | +| Safe eject blocked | Close unmanaged processes or extra mounts using the media, then retry; `unplug` is only for deliberate failure simulation | +| VM will not boot | Read session `console.log` and `qemu.log`; verify all three supplied boot artifacts belong to the current build | + +QEMU exercises the actual Linux/FDS software path. It does not emulate Pi +firmware, RP1, USB power sequencing, the physical twelve-bay wiring, or Dasung +power recovery. Those acceptance checks remain hardware procedures. + +## Physical Pi acceptance procedure (deferred) + +After assembling and calibrating the physical bay map, use a disposable USB +cartridge and the current matching SYSTEM/initramfs. Perform these checks on the +Pi; the VM results do not replace them: + +1. Build the hello/report cartridge on the workstation, record its image hash, + and use preview/confirmation to write that entire image to the chosen USB drive. +2. Insert it in a calibrated bay. Record `fds --json bay BAY`; confirm both + software entries and their declared payload partitions appear. +3. Run `demo.hello:hello` and `demo.report:report`. Save their output from the + cartridge log and confirm UID 1000. Check `/proc/self/mountinfo` for read-only + payload and software-cache mounts. +4. Run `fds eject BAY`, verify SAFE, then remove and reinsert the cartridge in + another calibrated bay. Repeat both commands. Record any USB enumeration or + I/O errors. +5. With a disposable cartridge and a deliberately long-running test program, + test surprise removal separately. Verify its managed processes terminate and + all of that bay's software mounts disappear. Do not use valuable DATA for + this failure test. +6. Exercise native shutdown with active software, cold boot, and a real USB + power cycle. Save actual observations and kernel/service logs. Follow the + separate [Dasung hardware procedure](dasung.md) for monitor recovery. + +Record the Pi, hub, USB drive and kernel versions with the result. Hardware +latency, electrical behavior and data durability remain unverified until these +measurements are performed on the assembled machine. diff --git a/docs/eeprom.md b/docs/eeprom.md index 8f8ae34..04d2117 100644 --- a/docs/eeprom.md +++ b/docs/eeprom.md @@ -1,12 +1,10 @@ # Pi 5 EEPROM configuration -[Boot images](boot.md) · [Recovery](recovery.md) · [Implementation ledger](implementation-status.md) +[Boot images](developer/boot.md) · [Recovery](recovery.md) · [User manual](README.md) -M12 now provides an offline configuration workflow using the pinned official -Raspberry Pi tool and a real Pi 5 firmware image. The host checks passed; -application to a physical Pi, boot order, PMIC behavior and timing remain deferred. -`tools/configure-pi-eeprom` only creates files. It never reads or writes a hardware -EEPROM, invokes a firmware updater, or reboots a machine. +The profile tool creates reviewable firmware/configuration files on the +workstation. Applying them to a Pi is a separate maintenance operation. Preserve +the machine's original firmware and configuration before changing them. ## Preview a profile on the build workstation @@ -15,7 +13,6 @@ From the repository root: ```sh ./tools/configure-pi-eeprom --profile production ./tools/configure-pi-eeprom --profile development -make eeprom-test ``` The first invocation downloads three checksum-pinned inputs into @@ -56,17 +53,14 @@ before applying anything to that machine. Boot order is read from the right. Both profiles avoid scanning the twelve USB cartridges for firmware boot. Development retains an SD rescue path. Both disable network-install keyboard detection; Raspberry Pi documents that this detection -adds USB initialization and enumeration work. These settings do not establish a -measured FDS boot improvement. [Official bootloader configuration](https://www.raspberrypi.com/documentation/computers/raspberry-pi.html#bootloader-configuration). +adds USB initialization and enumeration work. [Official bootloader configuration](https://www.raspberrypi.com/documentation/computers/raspberry-pi.html#bootloader-configuration). On Pi 5, the power-off setting requests PMIC standby on halt; the dedicated power button remains the wake mechanism. The wait setting leaves cold power-on boot -enabled. Whether the assembled computer and attached hardware behave as intended -still needs a physical test. [Official power settings](https://www.raspberrypi.com/documentation/computers/raspberry-pi.html#POWER_OFF_ON_HALT). +enabled. After shutdown, check that attached power hardware follows the selected policy. [Official power settings](https://www.raspberrypi.com/documentation/computers/raspberry-pi.html#POWER_OFF_ON_HALT). The source profiles are [production.conf](../config/eeprom/production.conf) and -[development.conf](../config/eeprom/development.conf). No GPIO wake option or -unmeasured HDMI tuning is added. +[development.conf](../config/eeprom/development.conf). Review these files before choosing a profile. ## Preserve the machine's settings @@ -102,12 +96,10 @@ This checks file construction and configuration roundtrip, not hardware compatibility. No secure-boot key, fuse, customer signature or OTP setting is modified by this helper. -## Apply and roll back during physical testing +## Apply and roll back -This step is deliberately outside the host/VM acceptance run. Use the Pi's -maintenance environment and the official installed EEPROM utilities. Once the -reviewed files and saved original inputs are available there, the upstream -configuration interface is: +Apply the reviewed configuration from a Raspberry Pi maintenance environment +with the official EEPROM utilities, then follow the updater's restart instructions. ```sh sudo rpi-eeprom-config --apply ./configured.conf ./base.bin @@ -126,24 +118,4 @@ If the firmware version itself changed, use the saved matching original firmware for the rollback operation. Keep the maintenance SD and an external EEPROM rescue route available; internal recovery cannot repair an EEPROM that prevents internal boot. The official [EEPROM update and recovery guide](https://www.raspberrypi.com/documentation/computers/raspberry-pi.html#raspberry-pi-boot-eeprom) -describes that physical workflow. FDS has not yet tested it on your Pi. - -## Inputs, dependencies and software evidence - -The input lock is [inputs.json](../config/eeprom/inputs.json), pinned to -`raspberrypi/rpi-eeprom` commit `2fee426f27b6c54d3f5b6f36efd9a2fe1286a45d` and -Pi 5 preview firmware `pieeprom-2026-09-12.bin`. Every download is SHA-256 checked; -a changed cached input is rejected. The -[official source tool](https://github.com/raspberrypi/rpi-eeprom/blob/2fee426f27b6c54d3f5b6f36efd9a2fe1286a45d/rpi-eeprom-config) -is stored unchanged in the cache. Void's tracked source remains unchanged. - -The host workflow uses existing Python and curl. It does not add a target daemon, -package or Rust dependency. The upstream parser's optional signing dependencies -are not needed for configuration-only operations. - -`make eeprom-test` passed production/development roundtrips, unchanged firmware -payloads, repeatable binary output, refusal to overwrite an existing directory, -preservation of custom settings, exact rollback of conditional settings, and -rejection of malformed images, device nodes and oversized configuration files. -Initial evidence is `out/m12-eeprom.zi144zib/`, with log -`out/logs/m12-eeprom-check.log`. The complete local release acceptance is recorded in [M12 validation](m12-validation.md). +describes that physical workflow. diff --git a/docs/getting-started.md b/docs/getting-started.md index 2476797..0101954 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,232 +1,92 @@ -# Your first build +# Build and start FDS -[Documentation index](README.md) · [Troubleshooting](troubleshooting.md) +This guide creates a bootable FDS SYSTEM image and starts it in QEMU. Use an +x86_64 Arch Linux workstation for the complete OS build. To build application +cartridges on another Linux distribution, use the [workstation guide](workstation.md). +A prepared set of kernel, initramfs and SYSTEM images can be run on a Linux +workstation without building the OS locally. -This walkthrough takes you from an FDS Git checkout to two verified ARM Linux -artifacts. It explains the output as you go. You do not need a Raspberry Pi. +## Prepare the workstation -At the end you will have a small Rust program built with static musl and GNU hello -packaged for aarch64 glibc. This proves that the workstation can build the two -kinds of software FDS/OS needs. It does not produce a bootable OS image. +Use a normal user account, a Git checkout with its submodule metadata, and a +checkout path without spaces. Initial preparation needs Internet access. The +host must support unprivileged user namespaces and namespaced binfmt_misc +(Linux 6.7 or newer). Full builds and VM tests use tens of GiB per build tree; +check free space with `df -h .` and use [cleanup](cleanup.md) between build batches. -## 1. Check the machine and checkout - -Use an x86_64 Arch Linux workstation as a normal user. Check it with: - -```sh -uname -m -cat /etc/os-release -id -u -``` - -Expect `x86_64`, `ID=arch`, and a user ID other than `0`. The bootstrap script -currently rejects other host platforms and root execution. - -Enter your FDS checkout. For the existing development workspace: - -```sh -cd /home/felis/source/fds -pwd -git status --short -``` - -On another workstation, substitute its checkout directory. It must contain -`Makefile`, `Cargo.toml`, `tools/`, `.gitmodules`, and Git metadata. Avoid spaces -in the path. There is no configured public remote in this workspace; obtain a -Git checkout from the project owner rather than using an invented clone URL. - -You need HTTPS access to GitHub, Void repositories, Rust distribution servers, -and package source hosts. Initial downloads total hundreds of MB. Plan for -several GB of free disk space; the verified workstation used roughly 1.6 GB for -the Void masterdir and 455 MB for its hostdir cache, before Rustup and other -outputs. These are observations, not fixed requirements or a build-time promise. - -```sh -df -h . -``` - -## 2. Install host prerequisites - -On an otherwise maintained Arch installation, install the required packages: +From the checkout directory, install host prerequisites: ```sh sudo pacman -S --needed bash coreutils binutils git curl make file tar xz gzip zstd \ - bubblewrap rustup ca-certificates findutils diffutils grep sed gawk util-linux -``` - -This is the host package installation step. Subsequent build commands run as your -normal user. The alternative `./tools/bootstrap-host --install-deps` runs the same -package installation and then continues bootstrap; choose one route. - -The [dependency reference](build-host.md#host-dependencies) explains why each -package is needed. This project uses Rustup to select its pinned Rust compiler, -not an arbitrary system Rust version. If pacman reports an existing Rust package -conflict, resolve the host package choice before continuing; do not force file -replacement. - -Check that bubblewrap can create a user namespace: - -```sh + bubblewrap rustup ca-certificates findutils diffutils grep sed gawk util-linux \ + python e2fsprogs libarchive lz4 bwrap --ro-bind / / --unshare-user --uid 0 --gid 0 true ``` -Success produces no output and exits normally. If it fails, use -[Troubleshooting](troubleshooting.md#bubblewrap-or-user-namespace-failure). -This is a host capability requirement, not something a Pi can fix. - -## 3. Prepare the build environment - -From the repository root: +Python must be 3.14 or newer. The namespace check exits successfully without +output. Build commands below run as your normal user, sequentially; they share +one project-local Void build container. ```sh make bootstrap -``` - -This performs these steps: - -1. Initializes `vendor/void-packages` at the recorded Git commit. -2. Downloads the pinned static XBPS archive, verifies its SHA-256, and extracts - the host tools into `.host/xbps/`. -3. Copies the checked-in Void build configuration into the local upstream checkout. -4. Installs Rust 1.98.0, rustfmt, and `aarch64-unknown-linux-musl` through Rustup. -5. Fetches the locked workspace crates needed for Cargo to resolve the Dasung member, - including when building the dependency-free smoketest offline. -6. Creates the x86_64 glibc Void build container and verifies that GCC runs inside it. - -The Void build container supplies compilers and package tools; it is not the -future FDS root filesystem. Rustup changes your user toolchain installation. -XBPS host tools stay inside this repository, and no Pi storage is written. - -The final success marker is: - -```text -PASS: M0 host bootstrap; run make smoke-test to build both aarch64 artifacts -``` - -The command saves its output in `out/logs/bootstrap.log`. Re-running bootstrap -reuses the checkout and downloads, but still checks their pins and integrity. -If it exits nonzero, stop here and resolve that failure before running smoke-test. - -## 4. Cross-build both artifacts - -```sh make smoke-test -``` - -First Cargo builds `rust/fds-smoketest` for ARM, using the static musl libraries -and linker bundled with the Rust toolchain. Then xbps-src cross-compiles GNU -hello and packages it for aarch64 glibc. The first package build downloads the -cross compiler, target libraries, and build dependencies such as texinfo. - -The script inspects actual ELF files and local package metadata. It checks: - -| Artifact | Required result | -| --- | --- | -| Rust smoketest | AArch64, static linkage, no dynamic interpreter or shared library requirement | -| GNU hello in the XBPS package | AArch64, glibc loader and `libc.so.6` dependency | - -A successful run contains these messages; other compiler output appears between them: - -```text -PASS: aarch64 static ELF -PASS: aarch64 glibc ELF -PASS: XBPS package hello-2.12.3_1 architecture=aarch64 (glibc) -PASS: M0 smoke test complete -``` - -Without QEMU, it also reports: - -```text -SKIP: ARM execution (optional qemu-aarch64 not installed); ELF verification passed -``` - -That skip means the ARM program was compiled and inspected, but not executed. -It does not hide a failed compiler or ELF check. If QEMU is installed, execution -must pass; an emulator failure is a real smoke-test failure. - -## 5. Run the protection and formatting checks - -```sh make check ``` -Run this after smoke-test: it expects the built Rust executable to exist. It -checks that bad inputs are rejected, that the Void pin and upstream files are -protected, that overlays cannot replace upstream packages, and that Rust source -is formatted. Several `PASS: rejects ...` lines are expected; they mean the -negative tests worked. The final commands include: +Bootstrap prepares pinned Rust and XBPS tools and the Void build container. +The smoke test checks the AArch64 build toolchains; `make check` validates the +build guardrails. Stop and resolve any failed command before continuing. Logs +are in `out/logs/`. Repeated preparation reuses verified downloads and caches. -```text -PASS: M0 guardrail checks complete -cargo fmt --all -- --check -``` - -Rustfmt is normally silent on success. `make check` does not rebuild the ARM -program. After a source change, rebuild first, as described in -[Development](development.md#edit-and-rebuild-the-rust-program). - -## 6. Inspect what you built +## Build the images ```sh -ls -lh out/fds-smoketest out/packages/ -file out/fds-smoketest -./tools/verify-elf out/fds-smoketest aarch64 static -sha256sum -c out/manifests/artifacts.sha256 +make rootfs PROFILE=cli +make system-card PROFILE=cli +make initramfs +make workstation ``` -`file` should include `ARM aarch64` and `statically linked` (or `static-pie linked`). -The checksum command should report `OK` for both artifacts. Hashes verify the -files against this run's manifest; they are not release signatures. +The rootfs build includes the kernel and base packages. These are the three +inputs needed by the emulator: -For ELF details, use: +| File | Purpose | +| --- | --- | +| `out/kernel/boot/kernel_2712.img` | ARM kernel | +| `out/fds-initramfs.img` | Early startup and SYSTEM discovery | +| `out/fds-system-cli.img` | Read-only operating system cartridge | + +`make rootfs PROFILE=development` followed by `make system-card PROFILE=development` +builds a SYSTEM with compilers, Git, Vim, debuggers and display diagnostics. +Both profiles include WindowMaker and the Dasung daemon. The desktop starts on +request. `make all` also assembles recovery, boot and internal-storage images; +see [internal installation](internal-storage.md) when preparing a physical machine. + +## Start the emulator + +With the project-local QEMU tools: ```sh -readelf -hW out/fds-smoketest -readelf -lW out/fds-smoketest -readelf -dW out/fds-smoketest +./tools/in-void xbps-install -y qemu-img +out/workstation/fds-emulator --session out/my-emulator start --qemu-runner tools/in-void +out/workstation/fds-emulator --session out/my-emulator console ``` -The header names AArch64. There should be no `INTERP` segment or `NEEDED` library. -An x86_64 host's `ldd` may say `not a dynamic executable` and return 1; that message -is expected here, but is insufficient by itself to prove a foreign ELF is static. -The [package guide](packages.md#inspect-the-built-package) walks through the -corresponding inspection of GNU hello. - -## 7. Optionally execute the ARM Rust program - -The [Arch qemu-user package](https://archlinux.org/packages/extra/x86_64/qemu-user/) -provides user-mode emulation. Install it if you want to execute the ARM program -on the x86_64 workstation: +If QEMU is installed directly on the workstation, omit `--qemu-runner`. +The session directory must be new. At the `FDS>` prompt: ```sh -sudo pacman -S --needed qemu-user -qemu-aarch64 out/fds-smoketest +fds info +fds bays +fds --help ``` -Expected output: +The shell runs as the ordinary `fds` user. Press **Ctrl-]** to detach from the +console; the VM remains running. Stop it from the workstation: -```text -FDS/OS M0: aarch64 static-musl OK +```sh +out/workstation/fds-emulator --session out/my-emulator stop ``` -Because this executable is static, this command needs no ARM rootfs or glibc -sysroot. Explicitly invoking `qemu-aarch64` also avoids needing automatic binfmt -registration. Re-running `make smoke-test` records its own QEMU check when the -emulator is on PATH. - -This executes one Linux userspace program. It does not emulate the Pi's firmware, -USB bays, display, battery, or operating-system boot. QEMU execution was not -performed during the initial M0 validation; see the [recorded results](m0-validation.md). - -## 8. Decide what to do next - -- To build the configured OS filesystem and try its ARM programs, follow [Rootfs](rootfs.md): `make rootfs PROFILE=cli`, then `make rootfs-test`. -- To boot native s6 and open a development console, follow [Native init and ARM VM](init.md): `make init-test`, then `make vm`. -- To build the included Dasung monitor package, follow [Dasung](dasung.md). -- To edit the existing Rust program, follow [Development](development.md). -- To understand the `.xbps` artifact, follow [Packages](packages.md). -- To understand the intended removable operating system, read [Architecture](architecture.md) and [Cartridges](cartridges.md). -- To see when bootable images become possible, read [Roadmap](roadmap.md). - -Bootstrap, smoke-test, and check complete the M0 foundation. Continue with the -M1 guide for a configured rootfs archive. There is still no flashing step. +Continue with [building and inserting software cartridges](workstation.md). +That guide also covers using copied boot images, DATA overlays and USB writing. diff --git a/docs/images/fds-control.png b/docs/images/fds-control.png new file mode 100644 index 0000000..e53bd26 Binary files /dev/null and b/docs/images/fds-control.png differ diff --git a/docs/internal-storage.md b/docs/internal-storage.md index 92c2cd4..fe9edcc 100644 --- a/docs/internal-storage.md +++ b/docs/internal-storage.md @@ -1,11 +1,11 @@ # Internal storage and machine settings -[Documentation index](README.md) · [Boot images](boot.md) · [Recovery](recovery.md) +[Documentation index](README.md) · [Boot images](developer/boot.md) · [Recovery](recovery.md) -The internal NVMe supplies the Pi's firmware boot files, independent recovery, -and machine settings. SYSTEM remains a separate removable cartridge. Ordinary -user files belong on DATA, not internal storage. No physical disk is written by -any build command below; Pi/NVMe boot and power-loss tests remain deferred. +Internal storage contains boot files, an independent recovery system and +persistent machine settings. The operating system itself lives on a removable +SYSTEM cartridge. This guide covers preparing the internal disk and updating +its settings from recovery. ## Build the complete disk image @@ -42,17 +42,7 @@ For a 2 GiB recovery allocation or larger settings partition: ./image/build-internal --recovery-mib 2048 --internal-mib 512 ``` -Output directories supplied with `--output-directory` must already exist and be -empty. The builder accepts ordinary image files, never a host block-device -output. Final hardware provisioning and identity checks belong to the physical -acceptance procedure; do not confuse a partition payload with the complete disk. - -## Install the internal disk when hardware is available - -This is a **deferred physical procedure**. The image and virtual NVMe workflow -are software-tested; writing and booting the user's actual NVMe still require -the hardware. Installation erases the selected disk, including any existing -machine settings. Retain backups before replacing an existing installation. +## Install the internal disk Use an NVMe enclosure or another Linux machine that can access the target drive while the Pi is off. First verify a downloaded release using a separately trusted @@ -132,12 +122,7 @@ alternative complete GPT images, each containing one `FDS_SYSTEM` partition. Readback and backup-GPT relocation apply to this disk too. Check its partition label, safely disconnect it, and insert exactly one SYSTEM cartridge in the Pi before normal boot. Subsequent cartridge creation and updates can use FDS's -confirmed [media workflow](media-tools.md) after bay calibration. - -On the first physical boot, check `fds info`, `fds machine status`, and -`fds bays`. The supplied bay map is empty until calibration. Follow -[physical acceptance](stress-testing.md) to measure ports, test the Dasung -display, and record actual boot/shutdown behavior before relying on the machine. +confirmed [media workflow](developer/media-tools.md) after bay calibration. ## Configure the machine before building @@ -149,9 +134,9 @@ Copy `config/machine/` to your own directory. It contains three files: - `hardware-catalog.toml`: optional USB identification names using the same schema as the base catalog. Entries are data and cannot run commands. -The supplied bay map is deliberately empty because the physical wiring has not -been measured. Do not invent Pi USB paths. USB 2 and USB 3 companion ports need -explicit aliases for the same bay. +The supplied bay map is empty. Record actual controller/port identities for +each physical slot before installing settings. USB 2 and USB 3 companion ports +need explicit aliases for the same bay. ```sh cp -a config/machine out/my-machine @@ -253,21 +238,3 @@ diagnostic snapshot, including metadata already inspected during this boot. Retrieve it with `fds machine fetch cartridges-first.json /tmp/saved-inventory.json`. The daemon's live metadata cache remains volatile and is rebuilt from currently attached devices; saved snapshots are never used to authorize media actions. -Together, explicit boot reports, inventory snapshots and hardware captures provide -the boot history, cached metadata and diagnostics assigned to FDS_INTERNAL in -master-plan section 12, without adding internal writes to startup or shutdown. - -## Dependencies and validation - -No new target package or Rust crate is required. The host image-tool prefix adds -`e2fsprogs` for ext4 creation, inspection and validation; it already supplies FAT -and EROFS tools. A private unprivileged user namespace gives created files root -ownership without requiring a root build session. - -`make internal-test` exercises the actual packaged runtime in ARM VMs with -virtual NVMe. Acceptance evidence belongs in [M12 validation](m12-validation.md); -a passing VM does not verify the Pi EEPROM, PCIe path or physical flash durability. - -The Linux [ext4 mount documentation](https://www.kernel.org/doc/html/latest/admin-guide/ext4.html) -explains why read-only loading also disables journal replay. Filesystem creation -options follow the upstream [mke2fs manual](https://man7.org/linux/man-pages/man8/mke2fs.8.html). diff --git a/docs/recovery.md b/docs/recovery.md index ddee7d3..050d71f 100644 --- a/docs/recovery.md +++ b/docs/recovery.md @@ -1,15 +1,9 @@ # Recovery and rollback -[Documentation index](README.md) · [Boot images](boot.md) · [Media tools](media-tools.md) · [EEPROM](eeprom.md) +[Documentation index](README.md) · [Boot images](developer/boot.md) · [Write cartridges](workstation.md#write-a-cartridge-to-usb) · [EEPROM](eeprom.md) -Recovery is a separate, read-only FDS image. It contains native s6, Bash, GNU -utilities, XBPS, filesystem tools, the static FDS tools, and the base Dasung -controller. It does not need files or programs from a SYSTEM cartridge. -The software workflow is covered by `make recovery-test`; physical Pi recovery remains deferred. -The [complete internal disk](internal-storage.md) includes this recovery image; -its guide separates verified image construction from deferred physical -installation. The build commands below create ordinary files and do not write a -host disk. +FDS recovery is an independent maintenance system on internal storage. It can +inspect a failed SYSTEM, check DATA and help prepare replacement cartridges. ## Build the recovery image @@ -18,16 +12,8 @@ From the repository root, run these commands sequentially: ```sh make recovery -make rootfs-test -make recovery-test ``` -The full VM test also needs the Pi kernel/initramfs and a known-good CLI SYSTEM -image. On a fresh checkout, build those first with `make rootfs PROFILE=cli`, -`make rootfs-test`, `make initramfs`, and `make system-card PROFILE=cli`, then -run the recovery sequence above. Run builds and VM tests sequentially because -they share the same project-local Void environment. - `make recovery` builds the `recovery` rootfs profile independently, configures all packages and caches at image construction time, then creates EROFS. Its outputs are: @@ -54,9 +40,7 @@ Stage0 can enter recovery in two ways: Restore normal mode when maintenance is complete. Stage0 requires exactly one readable `FDS_RECOVERY` partition and mounts it -read-only. It does not silently choose between duplicate partitions. The ARM VM -suite supplies disposable virtual partitions; physical firmware/NVMe/display -behavior must still be checked on the Pi. +read-only. It does not silently choose between duplicate partitions. The local prompt is: @@ -94,10 +78,6 @@ inactive SYSTEM mounts read-only under `/run/fds/media/NN`; `fds bay N` reports the actual path. A bad filesystem or manifest produces an error instead of running anything from the cartridge. -An empty bay configuration produces `UNCONFIGURED`, not guessed bay numbers. -See [Cartridges](cartridges.md) for calibration. Permanent machine configuration -on `FDS_INTERNAL` is part of the remaining M12 work. - ## Check and repair DATA Start with a read-only check. For example, for DATA in bay 2: @@ -112,10 +92,6 @@ disk exclusively, verifies GPT and the kernel partition identity, and invokes `e2fsck -f -n`. It does not repair the filesystem. A clean result leaves DATA unmounted and reports `SAFE TO REMOVE`. -If DATA was explicitly activated writable, eject that session first. Close any -shell whose current directory is on DATA and any other reader before checking; -an ordinary busy-unmount failure is reported, never bypassed with lazy unmount. - If the check reports problems, review its log and preserve a backup where possible. Preview the repair with: @@ -133,13 +109,6 @@ repairs and stops when manual judgement is required. It then flushes the device, invalidates its block cache, and runs a second `e2fsck -f -n`. Only a successful verification produces `DATA REPAIRED AND VERIFIED` and `SAFE TO REMOVE`. -The checker uses a temporary kernel loop device backed by the already verified -partition descriptor. This lets `e2fsck` take its own exclusive device claim -while FDS retains the physical whole-disk reservation. The loop is removed -automatically when its last descriptor closes. This uses the existing kernel -loop driver, `libc` crate and base `e2fsprogs`; no new package or Rust dependency -is introduced. - Failed or interrupted checks/repairs retain a quarantine record across cartridge daemon restarts for the same insertion. They do not inherit an earlier SAFE status. The checker holds the disk reservation and is killed if its supervising @@ -175,7 +144,7 @@ fds burn system /run/fds/media/02/fds-system-cli.img BAY04 Use the actual source mount shown by `fds bay 2`. If the destination already contains a mounted cartridge, run `fds eject 4` first. The burn preview identifies the source checksum, destination capacity/model/serial and confirmation command. -Follow [Media creation and writing](media-tools.md) for confirmation and status commands. +Follow [Media creation and writing](developer/media-tools.md) for confirmation and status commands. Completion requires device flush, readback and GPT verification. Recovery uses the same protected writer as the main system: it refuses mounted destinations, active root storage and disks containing internal FDS partition names. @@ -199,7 +168,7 @@ and boot events with checksums. Its output is temporary unless copied to a healthy DATA cartridge after explicitly activating that DATA with `fds data use N`. Do not activate the damaged cartridge merely to save a report. Eject writable DATA after copying, or let `fds poweroff` perform the normal verified shutdown. -See [Shutdown](power.md) when shutdown reports a blocking DATA error. +See [Shutdown](developer/power.md) when shutdown reports a blocking DATA error. Recovery is independent of SYSTEM, but it still depends on working internal boot storage, the kernel and firmware. An external rescue medium is required diff --git a/docs/releases.md b/docs/releases.md index 0e35ce8..545bc6c 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,88 +1,12 @@ -# Release signatures and verification +# Verify a release -[Documentation index](README.md) · [M12 evidence](m12-validation.md) +Verify a downloaded release before writing its images to a cartridge or internal +disk. Verification checks the release manifest signature and the length and +SHA-256 of every listed artifact. Obtain the signer's public key through a +trusted channel independent of the download. -`fds-release` signs an artifact manifest and verifies its signature and every -listed file. The static ARM executable is included in the base CLI package, so -CLI, development and recovery images can all verify downloads. The workstation -build supplies an x86_64 signer with the same format. - -The signing tool has passed host, static ARM and independent OpenSSL acceptance. -The complete frozen-input, versioned local 0.1.0 release passed acceptance; -see [M12 validation](m12-validation.md#final-local-release-acceptance) for its identity -and [Offline rebuilds](reproducible-builds.md) for the workflow. -The examples below describe the working signing interface; they do not identify -a published or hardware-qualified FDS release. - -## Build and test the tools - -After [bootstrap](getting-started.md), run: - -```sh -make signing -make signing-test -``` - -The workstation executable is -`target/x86_64-unknown-linux-gnu/release/fds-release`. The ARM executable is -`target/aarch64-unknown-linux-musl/release/fds-release`; `make tooling` also -exports it to `out/fds-release`. An ARM file cannot execute directly on the -x86_64 workstation. `make signing-test` runs that file through the project-local -QEMU interpreter as well as testing the host executable. - -OpenSSL is a **host test dependency** used as an independent Ed25519 -implementation. It is not linked into the FDS verifier. The test creates private -disposable keys under its `out/m12-signing.*` directory, checks exact signatures, -then exercises wrong keys, altered manifests, damaged or missing artifacts, -symlinks, FIFOs, malformed fields and unsafe paths. Test keys are not release -keys. - -## Create and retain a signing key - -For a local signing identity, choose an existing private directory outside the -checkout and its build output. This example creates a new directory; choose a -different name if it already exists: - -```sh -mkdir -m 700 "$HOME/fds-signing" -target/x86_64-unknown-linux-gnu/release/fds-release keygen \ - "$HOME/fds-signing/release-01" -``` - -The command creates `release-01.key` (32 secret bytes, mode 0600) and -`release-01.pub` (the hexadecimal public key). Existing files are never -overwritten. It prints SHA-256 of the decoded 32-byte public key, never the -secret. This fingerprint differs from `sha256sum release-01.pub`, which hashes -the hexadecimal text and its newline. Retain the private key separately from release downloads and source -archives. Anyone who has that key can sign a release under that identity. - -If public-key export failed after private-key creation, retain the private file -and export it to a new path: - -```sh -target/x86_64-unknown-linux-gnu/release/fds-release public-key \ - "$HOME/fds-signing/release-01.key" "$HOME/fds-signing/recovered.pub" -``` - -Private keys must be regular files owned by the invoking user with no group or -other permissions. The tool rejects symlinks and invalid key lengths. Key -generation uses Linux `getrandom`; temporary seed storage and the signing key -are zeroized when dropped. - -## Sign an assembled directory - -An assembled release directory must already contain its artifacts and a -`manifest.json` matching the format below. Signing verifies every artifact's -length and hash before creating `manifest.sig`: - -```sh -target/x86_64-unknown-linux-gnu/release/fds-release sign \ - /path/to/release-directory --key "$HOME/fds-signing/release-01.key" -``` - -If a signature already exists, use a new release directory. The tool deliberately -does not replace signatures or silently rewrite the manifest. A failed operation -may leave its new output file for inspection; it never declares it verified. +On a workstation, build the verifier with `make signing` after bootstrap. On FDS, +`fds-release` is included in the base system and recovery. ## Verify before using images @@ -103,65 +27,17 @@ target/x86_64-unknown-linux-gnu/release/fds-release verify \ /path/to/release-directory --key /path/to/trusted-fds-release.pub ``` -A successful command reports `VERIFIED FDS/OS 0.1.0`, the number of checked -artifacts, and `Hardware validation: deferred`. Failure returns exit status 2 -and an explanation. Keep downloaded images unchanged between verification and -use. Verification covers only files listed in the manifest; unrelated extra -files are not endorsed. The [media writer](media-tools.md) performs its own -image/layout validation and readback checks when writing a cartridge. +Signature verification is an explicit pre-installation step. It is not enforced +by Pi firmware or stage0 during boot. -These are distribution signatures. The Pi firmware and stage0 currently do not -enforce them at boot, so this is not a secure-boot implementation or a claim -that the hardware has been tested. +Success reports `VERIFIED` and identifies the release. Failure returns a nonzero +exit code with an explanation. Keep files unchanged between verification and +writing. Files absent from the signed manifest are outside its scope. -## Manifest and signature format +The cartridge writer separately checks image layout, destination identity and +readback. Release verification does not install images; follow the +[USB-writing workflow](workstation.md#write-a-cartridge-to-usb) or +[internal installation](internal-storage.md). -The JSON document has exactly these fields: - -```json -{ - "format": 1, - "version": "0.1.0", - "source_epoch": 1789909701, - "source_sha256": "<64 lowercase hexadecimal characters>", - "void_commit": "02a3cbc132c3c4a3a9d59e9b98f517af5dd11cd1", - "hardware_validation": "deferred", - "files": [ - { - "name": "fds-system-cli-0.1.0.img", - "bytes": 123456, - "sha256": "<64 lowercase hexadecimal characters>" - } - ] -} -``` - -This is a schema illustration, not a usable manifest: the size and hash -placeholders must be replaced with values from the actual artifact. Format 1 -accepts the current tool version, a positive source epoch, a 40-character Void -commit, and 1–64 nonempty artifacts. Filenames must be flat safe ASCII names; -paths, duplicates, hidden names, reserved manifest filenames and unknown fields -are rejected. The manifest is bounded to 1 MiB. Files must be regular files, -opened without following symlinks; hashing streams their bytes and checks for -changes during the read. - -The signature is Ed25519 over the exact bytes -`FDS/OS release manifest v1` followed by a NUL byte and the raw `manifest.json` -bytes. JSON whitespace is therefore authenticated too. `manifest.sig` contains -128 lowercase hexadecimal characters and a newline. The public-key file contains -64 lowercase hexadecimal characters and a newline. Verification rejects weak -public keys and uses strict signature verification. - -The implementation uses locked `ed25519-dalek` 3.0.0 for Ed25519, -`zeroize` 1.9.0 for secret buffers, and the existing SHA-256/JSON crates. -Its locked transitive dependencies provide curve arithmetic, digest and -signature types. Default dalek features are disabled; no asynchronous runtime, -OpenSSL binding or shared target library is added. The dependencies and their -license metadata are recorded in `Cargo.lock` and the cached crate sources. -The build collects their license texts, including build dependencies and -vendored libusb notices, into `/usr/share/licenses/fds-cli/RUST-NOTICES.txt`. -The usage guide is installed at `/usr/share/doc/fds/releases.md` in every profile. -Algorithm interoperability is checked against the -[RFC 8032 test vector](https://www.rfc-editor.org/rfc/rfc8032.html#section-7.1) -and OpenSSL; strict verification follows the -[dalek API](https://docs.rs/ed25519-dalek/3.0.0/ed25519_dalek/struct.VerifyingKey.html). +Release maintainers can find key creation, signing and format details in the +[developer reference](developer/releases.md). diff --git a/docs/software-format.md b/docs/software-format.md index b2f5dfe..8dc864a 100644 --- a/docs/software-format.md +++ b/docs/software-format.md @@ -1,117 +1,107 @@ -# Software cartridge format 1 +# Software cartridge format -[Build and use a cartridge](workstation.md) · [Original cartridge classes](cartridges.md) +A PROGRAM cartridge is a complete GPT disk image with one EROFS metadata partition +and one or more EROFS payload partitions. New media uses catalogue format 2 and +stores installed Void package trees. Source recipes and build tools remain on +the workstation; FDS executes the installed files directly. -A software cartridge is a complete GPT disk image with **1 + m** partitions, -where `m` is 1–32. Partition 1 describes every software package in all payload -partitions. All partitions use read-only EROFS. This format is the current -PROGRAM creation format; existing single-partition PROGRAM media remain readable. -SYSTEM, DATA and ENVIRONMENT layouts retain their existing meanings. - -## Disk and filesystem layout - -```text -GPT protective MBR + primary GPT - 1 FDS_METADATA - FDS/CARTRIDGE.TOML - FDS/SOFTWARE.TOML - 2 FDS_PAYLOAD02 - bundles/demo.hello.tar.xz - bundles/demo.editor.tar.xz - 3 FDS_PAYLOAD03 - bundles/demo.report.tar.xz - ... -backup GPT -``` - -Partition entries are consecutive, use Linux filesystem type GUIDs, unique -nonzero partition UUIDs, no GPT attributes, and nonoverlapping MiB-aligned -extents. Partition 1 starts at LBA 2048. Both GPT copies, their headers and CRCs -must agree. A full-drive write to a larger USB device relocates the backup table -and header. The existing on-target image parser and the workstation writer -share these checks. - -`FDS/CARTRIDGE.TOML` uses the existing format-1 identity with `class = "program"` -and `media.writable = false`. It contains no startup command. A sample catalogue: +## Software source recipe ```toml -format = 1 - -[[software]] -id = "demo.hello" -name = "AArch64 hello" +format = 2 +id = "example.hello" +name = "Hello" version = "1.0" -architecture = "aarch64" -partition = 2 -archive_bytes = 2308 -unpacked_bytes = 70504 -entries = 2 -sha256 = "REPLACE_WITH_THE_64_CHARACTER_LOWERCASE_SHA256" -[software.commands] -hello = "bin/hello" + +[commands] +hello = "usr/bin/hello" + +[source] +package = "hello" ``` -Those lengths and the digest are illustrative. The builder generates actual -values. IDs and command names use the existing restricted FDS identifier syntax. -IDs are unique per cartridge. Every declared payload partition must contain -exactly the catalogue's `bundles/.tar.xz` archive names, and every payload -partition must be represented. Multiple software entries may share a partition. -The catalogue contains 1–128 software entries and at most 64 KiB of TOML. +`package` names a source package in the selected `void-packages` checkout. For a +custom package, also set `template = "void"`: that directory must contain a +normal Void `template` and any `files/` or `patches/` directories it uses. Paths +are relative to this recipe. The [included hello package](../examples/software/hello/void/template) +is a complete example. -## Archive contract +Custom sources are copied into an untracked `srcpkgs/PACKAGE` directory. Existing +tracked Void sources are never overwritten. If a generated source copy differs, +inspect it, preserve any independent edits, then remove that generated copy and +retry. An unchanged copy can be reused. -Bundles are deterministic USTAR archives compressed with single-threaded xz. -Entries are sorted, timestamps/UID/GID are zero, and permissions normalize to -0755 for directories/executables or 0644 for ordinary data. Host source symlinks -to regular files inside the root are flattened into regular files. Directory or -escaping symlinks are rejected; the archive itself never contains links. +`fds-cartridge` builds with `xbps-src -a aarch64`, installs the resulting package +and runtime dependencies into a fresh tree, and records installed XBPS versions. +Use package dependency declarations to request runtime libraries and utilities. +Each software tree is self-contained with respect to its installed dependencies; +there is no cross-cartridge package resolver. -The reader rejects: +Software IDs and public command names use lowercase ASCII letters, digits, +periods, hyphens and underscores, with a maximum length of 64. Command paths are +relative to the installed root and must resolve to executable files within it. -- Absolute, parent-traversing, repeated-separator, non-UTF-8 or control-character paths. -- Duplicate entries, file-as-parent conflicts, links, devices, sockets, FIFOs, - privileged permissions, and GNU/PAX extension entries. -- Incorrect SHA-256, compressed length, unpacked length, entry count, or command - paths that do not name executable regular files. -- ELF files that are not little-endian 64-bit AArch64, or any ELF in a bundle - declared `architecture = "any"`. -- Invalid xz data, nonzero material after the tar end marker, and resource-limit - violations. Each archive is at most 512 MiB compressed, 1 GiB unpacked and - 65,536 entries. Xz decompression has a 256 MiB memory limit. +## Cartridge recipe -A source path must fit USTAR's path fields. Packaging reports paths that cannot -be represented rather than emitting an unsupported extension header. +```toml +format = 2 +id = "example.tools" +name = "Example tools" +version = "1.0" -## Guest lifecycle +[[payload]] +sources = ["hello/software.toml", "report/software.toml"] +``` -On insertion the daemon validates the complete GPT against kernel partition -geometry and the current USB disk identity. It reads metadata without following -symlinks, mounts every payload read-only/noexec/nosuid/nodev, and checks the -archive inventory and lengths. `fds bay BAY` reports the full catalogue; -`fds bays` stays compact. No archive command runs automatically on insertion. +Each payload section creates one partition. Sources may be recipe paths or +previously built directories containing `software.toml` and `root/`. The metadata +partition records which software belongs to which partition. Software IDs and +cartridge identity must satisfy the metadata validation rules. -`fds run BAY -- SOFTWARE-ID:COMMAND [ARGUMENT...]` verifies and extracts that -package on first use. It creates a private temporary filesystem, applies the -archive contract, then makes the finished tree read-only and accessible for -execution. The runtime cache has a stricter **256 MiB per software tree** limit, -including reserved inode overhead; a host-valid larger bundle may therefore -need splitting before running. Caches live under `/run/fds/software//` and -vanish after eject, unplug, restart cleanup, or shutdown. +## On-disk contents -Managed consumers run as UID/GID 1000 with no new privileges in the bay's cgroup. -`FDS_APP`, `PATH`, `LD_LIBRARY_PATH` and `XDG_DATA_DIRS` point to that software -root. DISPLAY/XAUTHORITY retain the established optional desktop integration. -The xz utility is already an explicit dependency of the mandatory `fds-base` -package; runtime extraction adds no target package. +```text +Partition 1: FDS_METADATA + FDS/CARTRIDGE.TOML + FDS/SOFTWARE.TOML +Partition 2: FDS_PAYLOAD02 + programs/example.hello/usr/bin/hello + programs/example.hello/usr/lib/... + programs/example.hello/var/db/xbps/... + programs/example.report/... +``` -Build recipes, compilers, installation hooks and privileged archive scripts are -never part of this runtime path. +The software catalogue records identity, architecture, commands, partition, +installed package versions, entry count, byte count and SHA-256. `installed=true` +selects the direct-tree contract. The retained `unpacked_bytes` field records +installed file bytes; it does not imply an archive or guest extraction. -Safe eject stops consumers, rejects unexpected mount aliases, unmounts caches, -payloads and metadata, then reports SAFE. Confirmed surprise removal permits -lazy detachment after stopping consumers. Service restart removes stale mounts -before rescanning. SYSTEM stays immutable throughout. +The tree digest covers sorted paths, entry types, modes, contents and symlink +targets. Root-relative package symlinks are relocated to relative in-tree links +when building. Escaping links, special files, privileged/writeable executable +trees, wrong-architecture ELF files and inconsistent metadata are rejected. +Current per-tree bounds are 1 GiB of file data and 65,536 entries. -SHA-256 detects corruption and binds an archive to its catalogue; it does not -establish a publisher's identity. Only run software you trust. Release signature -verification remains the separate [release workflow](releases.md). +## Runtime behavior + +FDS verifies the cartridge's geometry, identity, inventory and tree integrity +on insertion. Payloads are mounted read-only with `nosuid,nodev`. No software +command runs automatically. Published command aliases appear in `/run/fds/bin`. + +Programs run as UID/GID 1000 in the bay's managed process group. `FDS_APP` names +the installed tree; PATH, library paths and data search paths include its +`usr/bin`, `usr/lib` and `usr/share`. Dynamically linked ELF entry points use the +package tree's own loader. Programs that hard-code unrelated system paths still +need to be compatible with FDS; the tree is not a container or a writable root. + +Safe eject stops consumers and releases every payload before reporting SAFE. +A digest detects corruption, not publisher identity: choose software sources +you trust. Distribution signatures are covered by [release verification](releases.md). + +## Older media + +The reader retains support for catalogue format 1 with xz payloads and legacy +single-partition PROGRAM cartridges containing `app/`. Old xz payloads retain +their temporary extraction behavior and limits. Creation commands produce only +the installed-tree format. Rebuild old software from its Void source recipe to +use direct execution. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 98f27a8..3bdca8a 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1,243 +1,37 @@ -# Troubleshooting the build +# Troubleshooting -[Documentation index](README.md) · [First build](getting-started.md) · [Build reference](build-host.md) +Read the complete command error and its log before retrying. Build logs are in +`out/logs/`; emulator `console.log` and `qemu.log` are in the selected session. +On FDS, `fds bay N`, `fds profiles` and `fds power status` report the relevant +operation state. Append `--help` to a command for accepted options. -Find the first failing command or `ERROR:` in the log, fix that problem, then -repeat the same step. Do not continue past a failed bootstrap and interpret later -missing files as independent problems. Run all commands below from the repository -root, as your regular user, unless explicitly shown otherwise. - -## Find the right log - -| Failing action | Where to look | +| Symptom | Resolution | | --- | --- | -| `make bootstrap` | `out/logs/bootstrap.log` | -| Void build-container creation | `out/logs/xbps-bootstrap.log` | -| `make smoke-test` | `out/logs/smoke-test.log` | -| GNU hello build | `out/logs/xbps-hello.log` | -| `make check` | Terminal output; this target does not save a log automatically | +| Build disk is full | Stop builds and VMs; run `make clean-preview`, then `make clean`. See [retention rules](cleanup.md). | +| Missing host utility | Install the named utility or select the documented image/XBPS/QEMU runner. `doctor` checks image and emulator prerequisites. | +| Bubblewrap/user namespace failure | Check that the host permits unprivileged namespaces. Run the check in [setup](getting-started.md); do not switch software builds to root. | +| Void checkout is unprepared | Bootstrap the checkout using upstream xbps-src instructions, then select it with `--void-packages`. | +| Stale generated overlay/source package | Compare the source and generated copy. Preserve independent edits, remove only that generated copy, and retry. Never overwrite tracked upstream templates. | +| Output already exists | Use a new output directory, cartridge image, preview or emulator session. Creation does not overwrite existing outputs. | +| Executable architecture or tree integrity error | Rebuild the source package and cartridge. FDS cannot run an invalid payload. | +| A command is unavailable | Wait for `MOUNTED READ ONLY`, inspect `fds --json bay N`, and use its qualified command alias. Run `hash -r` after removal if Bash retained an old path. | +| Empty or unconfigured physical bays | Run `fds topology` and calibrate the machine's bay map. Do not infer slots from disk names. | +| Duplicate cartridge ID | Give the second cartridge a distinct identity and rebuild it. | +| Console already in use | Detach the emulator console with Ctrl-] before `guest`, safe eject or shutdown. | +| Guest command timed out | Inspect the console log. It may still be running; do not repeat a write until its state is known. | +| Desktop activation fails | Inspect `fds profiles` and `/run/log/xserver/current`, `/run/log/desktop/current`. Check the display configuration. | +| Control panel cannot connect | Check `/run/log/cartridged/current` and that the cartridge service is running. The panel reconnects automatically. | +| Eject is blocked | Close programs, shells and extra mounts using the cartridge, then retry. Wait for SAFE before removal. | +| Shutdown is blocked | Read `fds power status`; resolve the reported storage problem and retry poweroff. `fds power resume` cancels a recoverable pending shutdown. | +| DATA needs checking | Use the [recovery workflow](recovery.md); preserve a backup before repair where possible. | -For example, after a failed smoke-test: +To inspect cartridge service output on FDS: ```sh -tail -n 80 out/logs/smoke-test.log -rg -n 'ERROR:|FAILED|error:|No space left|Permission denied' out/logs/ -df -h . +tail -80 /run/log/cartridged/current +fds --json bay 01 ``` -A search with no matches exits 1. Read the surrounding log too: upstream tools -may report failures using other wording. Save logs before re-running if you need -the previous output; see [Development](development.md#preserve-logs-and-identify-inputs). - -## Unsupported host, root execution, or missing commands - -- `M0 requires an x86_64 Linux build host` or `supports Arch Linux`: use the - supported host described in [Your first build](getting-started.md). Changing - the check alone does not port the build system to another platform. -- `Run as a normal user, not root`: leave the root shell and run the build from - your normal account. Only host package installation uses sudo. -- `Missing host command`: install the prerequisites from the first-build guide, - then repeat bootstrap. `readelf` comes from binutils, `bwrap` from bubblewrap, - and `flock` from util-linux. -- `checkout path without whitespace`: move or obtain the checkout at a path - with no spaces, tabs, or other whitespace, then use that directory consistently. - -If a previous root-run left files that your account cannot write, inspect their -ownership before correcting it. Do not respond by running all later builds as root. - -## Bubblewrap or user namespace failure - -Reproduce the same capability check bootstrap uses: - -```sh -bwrap --ro-bind / / --unshare-user --uid 0 --gid 0 true -``` - -If it reports `Operation not permitted` or a namespace error, the host kernel, -container policy, or security configuration prevents the unprivileged container. -Use an Arch host that permits this operation, or have the host administrator -resolve the restriction. Repeated downloads, sudo builds, and a different Pi -will not resolve this preflight failure. Success is a zero exit status with no -output; run `make bootstrap` again once that works. - -## Void submodule missing, wrong, or modified - -Inspect before changing anything: - -```sh -cat VOID_PACKAGES_COMMIT -git submodule status vendor/void-packages -git -C vendor/void-packages status --short -git -C vendor/void-packages diff HEAD -- -``` - -If the submodule is missing, `make bootstrap` initializes it. The directory must -be part of a Git checkout; an unpacked source archive is insufficient. If a clone -was interrupted by a network error, restore connectivity and repeat bootstrap. - -`Void checkout mismatch` means the checked-out commit differs from the explicit -pin. A `+` in `git submodule status` means it differs from the parent repository's -gitlink. These are two separate checks. Do not update the pin merely to silence -the error; first determine why the commits differ. - -`Tracked Void files were changed` means upstream source has local edits. Preserve -and review the diff. Move intentional FDS changes into the overlay design or a -separate patch for review; restore upstream only after saving work you need. -Generated ignored files such as `etc/conf`, `hostdir`, and the masterdir are -normal. Never use a broad reset or clean command without checking what it removes. - -## Configuration or overlay conflict - -For `etc/conf differs from config/xbps-src.conf`, compare the two: - -```sh -diff -u vendor/void-packages/etc/conf config/xbps-src.conf -``` - -Put the desired settings in `config/xbps-src.conf`, then follow -[the explicit synchronization procedure](development.md#change-build-configuration-deliberately). -A `diff` exit status of 1 means differences were found, not that the comparison failed. - -For `Stale overlay`, compare the named `vendor/void-packages/srcpkgs/fds-*` -directory with its source under `packages/`. Preserve any unique edits in the -source first. Move that specific generated copy into a backup directory under -`out/`, outside `srcpkgs/`, then run `./tools/prepare-void` again. Do not move or -remove unrelated upstream packages. The Dasung package is now an active overlay. After changing it, reconcile that -generated copy by this same procedure before rebuilding. - -`Overlay would replace an upstream package` is an intentional protection. Choose -an FDS-owned package name and design an overlay; do not disable the check or edit -the tracked upstream template in place. - -## Download failure, checksum mismatch, or apparent stall - -Bootstrap and package builds need network access. The first run can spend much -of its time downloading compiler packages. Read the latest log and check disk -space before assuming that a quiet download is a deadlock: - -```sh -tail -n 40 out/logs/bootstrap.log -tail -n 40 out/logs/xbps-hello.log -du -sh out/downloads vendor/void-packages/hostdir -df -h . -``` - -A log or directory may not exist if its step has not started. Bootstrap's static -XBPS download has retries and a timeout; an incomplete transfer uses a `.part` -filename. A retry re-downloads that partial file. Completed cached archives are -checksum-checked on every bootstrap. - -For a checksum failure, retain the error and compare the named file against -`config/host-tools.conf`. If the cached completed archive is corrupt, move that -specific archive to a backup location and repeat bootstrap so it downloads again. -Do not change the expected hash to match an unexplained download. If a clean -re-download still disagrees, stop and investigate the source. - -A Void mirror timeout is separate from the source pin: the templates are pinned, -but bootstrap and missing build dependencies still use rolling binary repositories. -An unavailable historical dependency may need a reviewed input/mirror solution; -blindly changing the source commit does not establish an equivalent build. - -## XBPS command not found or package index seems empty - -The XBPS tools are intentionally not installed in the host's `/usr/bin`. -`make bootstrap` creates `.host/xbps/usr/bin/`. FDS build helpers set PATH for you. -For manual queries, use the explicit paths in the [package guide](packages.md). - -When querying or indexing ARM output, set `XBPS_ARCH=aarch64`. The host's static -musl executable can otherwise default to its own architecture and ignore the ARM -package. `make smoke-test` uses the correct override. `tools/build-package` exports -packages but does not update `out/packages/aarch64-repodata`; use the indexing -command in the package guide after a standalone package build. - -## Rust target, linker, or architecture error - -Check the selected toolchain and local configuration: - -```sh -rustup show active-toolchain -rustup target list --installed -cat rust-toolchain.toml -cat .cargo/config.toml -printenv RUSTFLAGS CARGO_BUILD_TARGET CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER -``` - -`printenv` exits nonzero when a listed variable is unset; that is normal here. -Bootstrap installs the required target and rustfmt for the pinned toolchain. -Repeat it if those components are missing. - -Build from the repository root with the exact Cargo command in -[Development](development.md#edit-and-rebuild-the-rust-program). Environment -variables, command-line `+toolchain` overrides, or outside Cargo configuration -can change the target, flags, or linker. Reconcile such overrides with the -checked-in `rust-lld` and static CRT configuration. Adding an arbitrary system -linker or glibc library is not a repair for a static-musl build. - -## Exec format error, ldd output, and QEMU skips - -`./out/fds-smoketest` normally gives `Exec format error` on x86_64: it is an ARM -program. Run it explicitly with optional `qemu-aarch64`, as described in -[the first-build guide](getting-started.md#7-optionally-execute-the-arm-rust-program). -Do not change the target to x86_64 merely to make direct execution work. - -`ldd: not a dynamic executable` with exit 1 is expected for the static program. -Use `./tools/verify-elf out/fds-smoketest aarch64 static` for the actual check. -Host `ldd` cannot establish that a foreign ARM program will run. - -`SKIP: ARM execution` means QEMU is absent. Compilation and ELF checks still run. -If QEMU is present but fails, the smoke-test must fail; inspect its diagnostic. -The dynamic GNU hello program also requires an ARM glibc loader and libraries, -so the simple static-program QEMU command is not a complete hello runtime setup. - -## Missing artifacts or check fails before any build - -`make check` expects the Cargo release artifact in `target/`. Run the Rust build -or `make smoke-test` first. A bare Cargo build updates `target/`, while -`make smoke-test` also refreshes `out/fds-smoketest` and the output manifests. -If checksums fail after editing or replacing outputs, inspect what changed and -rebuild through smoke-test; do not edit the manifest to conceal a mismatch. - -For a Rust formatting failure, run `cargo fmt --all`, inspect the source diff, -then repeat `make check`. - -## Busy /tmp message during Void bootstrap - -The initial verified bootstrap emitted -`rm: cannot remove '//tmp': Device or resource busy` during upstream cleanup. -The bubblewrap bind mount explained that instance; upstream exited successfully, -and container GCC, the cross build, and the public M0 commands all passed. - -Do not treat every later `/tmp` error as harmless. Confirm a zero command exit -status and the final bootstrap PASS, then run smoke-test. If the command exits -nonzero, report it as a failure with the surrounding log. - -## Rootfs, images, and runtime commands - -M1 implements `make rootfs PROFILE=cli` and `make rootfs-test`. If the target is -missing, check that you are in the current checkout. Read [M1 rootfs](rootfs.md) -for prerequisites, shell access, logs, and retained failed-build directories. -Python 3.14+ is required for XBPS archive inspection. Supported profiles are -`cli`, `development`, and `recovery`; unknown names fail explicitly. Package -configuration must pass before an archive is published. - -`tools/in-rootfs` supports ARM child execution through a private user-namespace -binary-format handler. It requires Linux 6.7+ and namespaced `binfmt_misc` support. -The workstation's global registry is unchanged. For a single ELF executable, -`--direct` invokes QEMU without installing even a private child-execution handler. -For example: `./tools/in-rootfs out/rootfs-aarch64 --direct /usr/bin/xbps-pkgdb -a`. - -The tar is configured userspace, not a flashable image. M2 adds `make init-test` -for a full ARM boot and `make vm` for a temporary development shell. Follow -[Native init and ARM VM](init.md) for its dependencies, expected output, and -failure logs. For the full kernel/stage0/SYSTEM path, use `make all`, -`make boot-test`, then `make console-vm`. Run `fds bays` and `fds poweroff` -inside the booted FDS console, not on the Arch host. An exported ARM binary will -not execute directly on x86_64. Physical Pi acceptance remains deferred. - -## What to include in a bug report - -Include the exact failing command, its exit status if available, the relevant log -excerpt, host distribution and architecture, current Void pin, and whether this -is the first build or a previously working checkout. Mention local configuration -changes and toolchain overrides. Preserve the initial error; a later missing-file -message often only describes its consequence. +To inspect a stopped emulator, read its saved logs directly on the workstation. +Rebooting uses a new session directory; existing DATA overlays remain in the old +session and can be [exported](workstation.md#save-emulator-data). diff --git a/docs/workstation-tooling-plan.md b/docs/workstation-tooling-plan.md deleted file mode 100644 index e982b26..0000000 --- a/docs/workstation-tooling-plan.md +++ /dev/null @@ -1,60 +0,0 @@ -# Workstation emulator and software cartridges - -Software implementation and acceptance are complete; see the -[acceptance record](workstation-validation.md) and [user guide](workstation.md). -This follow-up extends the accepted 0.1.0 checkpoint. That signed release remains -unchanged and does not contain the new format or emulator interface. Physical -hardware acceptance remains deferred. - -## Required result - -- A Linux workstation CLI boots the real FDS kernel, initramfs and SYSTEM image - with QEMU, exposes its console, and controls twelve virtual USB cartridge bays. -- Cartridge image insertion and removal use QMP USB hotplug. Safe eject asks the - guest to release the cartridge; forced removal explicitly simulates pulling it. -- Software cartridges use GPT with exactly `1 + m` partitions. Partition 1 is - `FDS_METADATA`; partitions 2 through `m + 1` are `FDS_PAYLOAD02`, and so on. - All are read-only EROFS containers. This permits multiple software archives - in one partition without inventing a raw-partition archive container format. -- Metadata includes the cartridge identity and a catalogue locating every - software bundle, its version, architecture, commands, lengths and SHA-256. - Each software payload is an xz-compressed tarball, stored as - `bundles/.tar.xz` in its declared payload partition. -- Workstation tools build software from explicit trusted recipes, package the - resulting trees, construct and verify a complete disk image, then separately - preview/confirm a whole-USB write and verify its readback. Building must not - depend on running on the Pi or on an Arch-specific container. -- The guest recognizes this layout, reports its software catalogue, and runs a - selected software command as the ordinary user. Archives never run build hooks - on the guest. Runtime extraction is bounded, verified and temporary; eject - and surprise removal stop consumers and release every associated resource. -- SYSTEM and writable DATA retain their boot/data layouts. Legacy PROGRAM - reading may remain for existing images, but newly built software cartridges - use the metadata-first multi-partition format. - -## Acceptance gates - -1. Typed Clap host CLIs, actionable help and Linux prerequisite diagnostics. -2. Real workstation builds of at least two software programs, including an - AArch64 executable; archive inspection and independent xz/tar checks. -3. `1 + m` images with multiple software packages sharing a partition as well - as packages in separate partitions; independent GPT/filesystem inspection. -4. Negative checks for overlapping/corrupt GPT, wrong catalogue mappings, - archive corruption, unsupported architecture, unsafe paths/types and size - limits. Existing image/write rejection tests remain passing. -5. Complete-image write/readback using disposable regular files, including a - larger target and confirmation/source/target-change rejection. Actual USB - writes are deferred until the user selects physical hardware. -6. Real FDS boot under the public emulator, insert, catalogue, run, safe eject, - reinsert, forced removal during execution, and cleanup/restart behavior. - Every software partition must be exercised, not just the first payload. -7. Required rootfs/native-init regression and relevant DATA, PROGRAM, media and - shutdown checks; English installation, usage, troubleshooting and format docs. - -The QMP implementation follows the [QEMU reference](https://www.qemu.org/docs/master/interop/qemu-qmp-ref.html), -including waiting for completed device deletion before releasing block nodes. -The Rust `tar` library supplies archive entry decoding; FDS imposes stricter -path, entry-type and size rules instead of trusting an archive's ownership or -permissions. The existing `xz` utility supplies streaming compression and -decompression with a memory limit. QEMU is host-only; no emulator or software -build daemon is added to the Pi boot graph. diff --git a/docs/workstation.md b/docs/workstation.md index e88f3df..017d4b3 100644 --- a/docs/workstation.md +++ b/docs/workstation.md @@ -1,166 +1,133 @@ -# Build software cartridges and run FDS in QEMU +# Build software cartridges and use the emulator -This guide uses a **Linux workstation**, not the Raspberry Pi. It takes you from -source software to a cartridge disk image, runs that image in FDS, and explains -how to write the same complete image to USB. All commands run from the checkout -root unless marked as guest commands. +This walkthrough runs on a Linux workstation. It builds applications from Void +source packages, creates a complete cartridge image, runs it in FDS, and explains +how to write it to USB. Commands run from the FDS checkout root unless marked +as guest commands. -The two native host tools are `fds-cartridge` and `fds-emulator`. They use typed -Clap interfaces; append `--help` to any command or subcommand. The target's `fds` -command remains a static AArch64 executable. The new host tools are never part -of the Pi base image. Dasung support remains in the base boot bundle. +## Prepare the tools -The accepted local `out/fds-os-0.1.0/` release predates this extension. Leave that -release intact and build current images for the emulator. See -[workstation acceptance](workstation-validation.md) for current evidence. - -## Install and build the workstation tools - -Use a normal Linux account with Rust/Cargo, a C linker, Bash and Make. The -repository's `rust-toolchain.toml` selects the Rust version. Runtime prerequisites -are: - -| Tool | Purpose | -| --- | --- | -| `qemu-system-aarch64` | Runs the FDS ARM kernel and userspace using portable TCG | -| `qemu-img` | Creates a separate writable overlay for DATA; source images stay unchanged | -| `mkfs.erofs`, `fsck.erofs` from erofs-utils | Build and verify cartridge partitions; tested with 1.9.4 | -| `xz` | Compress/decompress software tarballs with a decompression memory limit | -| `bwrap` from bubblewrap | Restricts filesystem inspection to a private writable staging directory | -| A software-specific cross compiler | Builds AArch64 applications; unnecessary for portable scripts | -| `sfdisk`, `tar`, `mke2fs`, Python 3 | Independent integration tests; not needed to boot an existing image | - -Install these with your distribution's package manager. Unprivileged user -namespaces must work for bubblewrap. Building **these host tools** does not need -Void, XBPS, an Arch host, a Pi, root, or a running FDS guest: +The native tools are `fds-cartridge` and `fds-emulator`. Build them with Rust, +a C linker and Make; the repository selects its pinned Rust toolchain: ```sh make workstation +mkdir -p out export PATH="$PWD/out/workstation:$PATH" -fds-cartridge doctor -fds-emulator doctor ``` -Current acceptance ran on Arch Linux x86_64. The tools use native Linux -interfaces and distribution-provided utilities; other distributions and ARM -workstations have not yet been exercised by this acceptance run. +| Prerequisite | Used for | +| --- | --- | +| Prepared Void `void-packages` checkout and native XBPS utilities | Build source packages and install their AArch64 runtime dependencies | +| `mkfs.erofs`, `fsck.erofs` | Create and inspect read-only payload filesystems | +| `bwrap` | Inspect filesystems and install package trees in a user namespace | +| `xz` | Read older software cartridges and package/source archives | +| `qemu-system-aarch64`, `qemu-img` | Run the emulator and create writable DATA overlays | -The outputs are native binaries in `out/workstation/`. The build explicitly -selects the workstation architecture instead of the workspace's default ARM -target. Rust dependencies are pinned in `Cargo.lock`. `tar` 0.4.46 and its -`filetime` dependency provide archive decoding/creation; FDS additionally checks -entry types, paths, counts, sizes, executable architecture and hashes. +Install tools through your Linux distribution's package manager. Enable +unprivileged user namespaces for bubblewrap. Software builds run as an ordinary +user and execute trusted Void build templates on the workstation. Run source builds sequentially when they +share the same Void checkout. -For this repository's **existing Arch/Void setup**, reuse its local tools: +Prepare a Void source checkout using its [upstream instructions](https://github.com/void-linux/void-packages/blob/02a3cbc132c3c4a3a9d59e9b98f517af5dd11cd1/README.md). +Complete `xbps-src binary-bootstrap` for the workstation architecture before +using it. The workstation itself need not run Void. Select that checkout with +`--void-packages /path/to/void-packages`; native `xbps-*` tools must be in PATH, +or selected with `--xbps-bin /path/to/bin`. Builds target AArch64/glibc. + +For the complete FDS build workstation, [bootstrap](getting-started.md) already +prepares `vendor/void-packages` and `.host/xbps/usr/bin`. Reuse its local tools: ```sh -make workstation ./tools/prepare-image-tools ./tools/in-void xbps-install -y qemu-img -out/workstation/fds-cartridge --image-tool-runner tools/in-image-tools doctor -out/workstation/fds-emulator doctor --qemu-runner tools/in-void +fds-cartridge --image-tool-runner tools/in-image-tools doctor +fds-emulator doctor --qemu-runner tools/in-void ``` -The install command above changes only the project-local build container. The -optional runners accept a program name followed by its arguments; omitting them -uses programs in the workstation's `PATH`. Set the image runner on every -cartridge create, inspect or preview invocation that needs it. A QEMU runner is -saved in the session and reused for later DATA insertions. +An optional runner receives a utility name followed by its arguments. Image +runners select EROFS utilities; XBPS runners provide the package-installation +namespace. Without runners, the tools use native utilities and bubblewrap. -## Build two software bundles +## Build the example cartridge -Each software recipe declares an id, display name, version, target architecture, -root directory, and named executable commands. A trusted `[build]` section may -run a workstation compiler. It is executed only by `software build`, never by -`software pack`, inspection, or a cartridge insertion. +The repository includes hello and system-report Void source packages. Each +software recipe names a package, its source-template directory and its public +commands. A cartridge recipe groups those software recipes into payloads. -The [hello recipe](../examples/software/hello/software.toml) builds an AArch64 -C executable. Install an AArch64 glibc cross compiler as `aarch64-linux-gnu-gcc`, -then run: +With native tools and a prepared checkout: ```sh -fds-cartridge software build examples/software/hello/software.toml out/demo-hello -fds-cartridge software pack examples/software/report/software.toml out/demo-report -fds-cartridge software inspect out/demo-hello -``` - -If your cross compiler has a different name, edit the recipe's `build.command`. -With this checkout's existing Void cross toolchain, the equivalent is: - -```sh -mkdir -p examples/software/hello/root/bin -./tools/in-void aarch64-linux-gnu-gcc -O2 examples/software/hello/hello.c \ - -o examples/software/hello/root/bin/hello -out/workstation/fds-cartridge software pack \ - examples/software/hello/software.toml out/demo-hello -out/workstation/fds-cartridge software pack \ - examples/software/report/software.toml out/demo-report -``` - -Use **one** of those hello workflows. Outputs must not already exist: choose -new output names when rebuilding, then update your cartridge recipe. The example -bundles each contain `software.toml` and `.tar.xz`. The archive contains the -contents of the software root, such as `bin/hello`; it has no leading `root/`. - -For your own application, copy a recipe and populate its root with `bin/`, -`lib/` and `share/` as needed. C/C++ glibc builds target AArch64; static Rust -programs should use `aarch64-unknown-linux-musl`. Set `architecture = "any"` only -for scripts/data without ELF files. An x86_64 executable is rejected. Shared -libraries must be compatible with the target SYSTEM's glibc, or bundled as -appropriate. There is no cross-cartridge dependency resolver. - -## Assemble the cartridge image - -The example cartridge recipe groups the two bundles into two payload partitions: - -```sh -fds-cartridge create examples/software/cartridge.toml out/demo-tools.img +fds-cartridge --void-packages /path/to/void-packages \ + create examples/software/cartridge.toml out/demo-tools.img fds-cartridge inspect out/demo-tools.img ``` -For the project-local EROFS tools, use: +With the project-local Void and image tools, use this equivalent command: ```sh -out/workstation/fds-cartridge --image-tool-runner tools/in-image-tools \ +fds-cartridge --image-tool-runner tools/in-image-tools \ + --xbps-tool-runner tools/in-void --xbps-bin "$PWD/.host/xbps/usr/bin" \ create examples/software/cartridge.toml out/demo-tools.img +fds-cartridge --image-tool-runner tools/in-image-tools inspect out/demo-tools.img ``` -The resulting complete disk image has **three GPT partitions**: +Choose one route. Output paths must be new. The command builds each package +with `xbps-src`, installs the package and runtime dependencies into private +staging trees, verifies their architecture and integrity, and writes the complete +image. Source build logs go to standard error; the final result is JSON. -| Partition | GPT name | Contents | -| --- | --- | --- | -| 1 | `FDS_METADATA` | Cartridge identity and the software catalogue | -| 2 | `FDS_PAYLOAD02` | `bundles/demo.hello.tar.xz` | -| 3 | `FDS_PAYLOAD03` | `bundles/demo.report.tar.xz` | +The example image contains: -To put both programs in one payload partition, use one `[[payload]]` entry with -`bundles = ["../../out/demo-hello", "../../out/demo-report"]`. This makes a -two-partition image. Each payload group creates exactly one partition. The first -partition identifies every software bundle and its payload partition. +| Partition | Contents | +| --- | --- | +| 1: `FDS_METADATA` | Cartridge identity and software catalogue | +| 2: `FDS_PAYLOAD02` | `programs/demo.hello/`, including `usr/bin/hello` and libraries | +| 3: `FDS_PAYLOAD03` | `programs/demo.report/`, including `usr/bin/report` and dependencies | -Creation finishes only after verifying both GPT tables, all EROFS partitions, -every archive and the catalogue. Inspection prints JSON, including offsets, -lengths, SHA-256 and available commands. Repeating an unchanged recipe produces -the same image bytes. See [the format](software-format.md) for limits and rules. +There are no new xz software bundles. Programs are ready to run directly from +EROFS. Inspection validates both GPT tables, filesystem contents and software +hashes, then prints partition and command details as JSON. -## Boot current FDS in the emulator +## Build a reusable installed-software directory -You need three matching current artifacts: the FDS Pi kernel, its initramfs, and -a SYSTEM image containing the guest software runtime. Build these on the OS -build workstation using [the build guide](getting-started.md). For an already -bootstrapped checkout: +To separate package compilation from cartridge assembly: ```sh -make rootfs PROFILE=cli -make system-card PROFILE=cli -make initramfs -make workstation -out/workstation/fds-emulator --session out/my-emulator start \ - --qemu-runner tools/in-void +fds-cartridge --xbps-tool-runner tools/in-void \ + --xbps-bin "$PWD/.host/xbps/usr/bin" \ + software build examples/software/hello/software.toml out/demo-hello +fds-cartridge software inspect out/demo-hello ``` -If QEMU is installed directly on your Linux workstation, omit `--qemu-runner`. -To use images copied from another build machine, supply their paths: +The output contains `software.toml` and `root/`. A cartridge's `sources` list can +name this directory instead of a source recipe. For example: + +```toml +format = 2 +id = "my.tools" +name = "My tools" +version = "1.0" + +[[payload]] +sources = ["../out/demo-hello"] +``` + +Paths are relative to the cartridge recipe. Put several sources in one list to +share a partition, or add `[[payload]]` sections for separate partitions. See +[the format reference](software-format.md) for complete source metadata and limits. + +## Start FDS + +Build matching boot inputs using [Build and start FDS](getting-started.md), or +copy a kernel, initramfs and SYSTEM image from an FDS build machine. + +```sh +fds-emulator --session out/my-emulator start --qemu-runner tools/in-void +``` + +Omit `--qemu-runner` when QEMU is installed directly on the workstation. To use +copied boot images: ```sh fds-emulator --session out/my-emulator start \ @@ -169,173 +136,97 @@ fds-emulator --session out/my-emulator start \ --system /path/to/fds-system-cli.img ``` -Create `out/` first if using a fresh directory. The session directory must be -**new**, private, and short enough for Unix sockets (under 90 bytes including its -absolute parent path). Start waits for the FDS prompt and verifies the emulator -bay configuration. It uses two emulated CPUs and 1024 MiB RAM by default; -`--memory-mib 2048` increases RAM. `--timeout` adjusts the boot deadline. +The session directory must be new and private. Keep its absolute path short +(under 90 bytes) for Unix sockets. Start waits for the FDS prompt and uses two +emulated CPUs and 1024 MiB RAM; `--memory-mib 2048` increases memory. SYSTEM and +PROGRAM images are read-only. The session holds logs and any DATA overlays. -The session records inputs, virtual devices, serial output and QEMU diagnostics. -QEMU stays running after the command exits. SYSTEM and software images are -read-only. No physical disks, host network interface, or Pi monitor are attached. - -## Insert, use and remove cartridges +## Insert and run software From the workstation: ```sh fds-emulator --session out/my-emulator insert 01 out/demo-tools.img fds-emulator --session out/my-emulator guest -- fds bay 01 -fds-emulator --session out/my-emulator guest -- fds run 01 -- demo.hello:hello +fds-emulator --session out/my-emulator guest -- hello 'Hello from the emulator' fds-emulator --session out/my-emulator guest -- fds run 01 -- demo.report:report +``` + +Discovery is asynchronous. Wait until the bay reports `MOUNTED READ ONLY` before +running commands. Direct commands return their output and exit status to the +caller. `fds run` starts a background job; read its output with: + +```sh fds-emulator --session out/my-emulator guest -- tail -20 /run/log/cartridged/current +``` + +Open an interactive shell with `fds-emulator --session out/my-emulator console`. +At `FDS>`, use `hello`, `report`, `fds bays` or other guest commands. **Ctrl-]** +detaches without stopping the VM. Detach before using `guest`, `eject` or ordinary +`stop`, since they also need the serial console. + +All twelve virtual bays accept images. Simultaneously inserted PROGRAM cartridges +must have distinct cartridge IDs. Duplicate identities produce a guest error. +Command-name collisions are resolved as described in [Using cartridges](cartridges.md). + +## Eject and stop + +```sh fds-emulator --session out/my-emulator eject 01 -``` - -Insertion is asynchronous: `fds bay 01` may briefly show `EMPTY` before USB -storage discovery completes. Retry the status command until it reports -`MOUNTED READ ONLY`, or inspect its error. The catalogue lists exact commands. -`fds run` starts a managed process and prints its PID; its output goes to the -cartridge service log. Packages are verified and extracted into temporary, -read-only RAM filesystems on first run. They are not compiled on the guest. - -All twelve bays, numbered `01`–`12`, are available. Simultaneously mounted -PROGRAM cartridges need distinct cartridge IDs; inserting duplicate IDs is -reported as a guest error. `eject` asks FDS to stop -managed consumers, release mounts and declare `SAFE`, then removes the virtual -USB device. If FDS refuses, QEMU leaves the cartridge attached. To deliberately -simulate an accidental pull, use: - -```sh -fds-emulator --session out/my-emulator unplug 01 -``` - -A forced unplug can lose DATA writes. The guest must then detect the removal -and clean up its processes and mounts. `status` reports the actual QEMU devices -and block nodes as well as the recorded image paths. - -Use the interactive shell if preferred: - -```sh -fds-emulator --session out/my-emulator console -``` - -At the `FDS>` prompt, type `fds bays`, `fds bay 01`, or `fds run 01 -- -demo.hello:hello`. **Ctrl-] detaches**; it does not stop the VM. Use a second -workstation terminal for insertion/unplug. Detach the console before `guest`, -`eject`, or ordinary `stop`, because they also need exclusive serial access. - -Finish with: - -```sh +fds-emulator --session out/my-emulator status fds-emulator --session out/my-emulator stop ``` -This invokes native FDS shutdown. `stop --force` cuts virtual power without that -sequence. Logs and DATA overlays remain in the session directory. Choose a new -session directory to boot again; stopped sessions are retained for inspection, -not resumed from RAM snapshots. +Eject asks FDS to stop programs and release storage before removing the virtual +USB device. If FDS refuses, the cartridge stays attached. `stop` performs native +FDS shutdown. Session logs remain available; create a new session to boot again. -## Writable DATA images +`unplug BAY` simulates a physical pull, and `stop --force` cuts virtual power. +Use these only for deliberate failure simulation; DATA writes can be lost. -Insert an existing FDS DATA disk image with the same `insert` command. The -emulator creates a unique `data--.qcow2` overlay in the session directory; -all guest writes go there. The original image remains unchanged. Safe eject -retains the overlay, while inserting the original image again starts a fresh -one. Software cartridges remain read-only. +## Save emulator DATA -After eject or shutdown, export an overlay if you need the changed DATA contents: +Inserting a DATA image creates a writable `data--.qcow2` overlay in the +session directory. Eject retains it. Reinserting the original image creates a +fresh overlay. To export the changed contents after eject or shutdown: ```sh -qemu-img convert -f qcow2 -O raw /path/to/session/data-02-IDENTIFIER.qcow2 out/saved-data.img +qemu-img convert -f qcow2 -O raw /path/to/session/data-02-ID.qcow2 out/saved-data.img ``` -Keep the original backing image at its recorded path until conversion completes. -Never convert or edit an image while it is attached to a running VM. +Keep its backing image at the recorded path until export completes. Never convert +an overlay while a running VM is using it. Personal session directories are +preserved by `make clean`; keep them under your own names, such as `my-emulator`. -## Write the completed image to USB +## Write a cartridge to USB -The burn flow always starts with the complete image created above. It does not -construct partitions directly on a drive. Identify the intended **whole USB -drive**, unmount it, then make a preview as your normal user: +Identify the intended whole USB drive, unmount its partitions, then preview: ```sh fds-cartridge preview out/demo-tools.img /dev/sdX out/usb-preview.json ``` -Replace `/dev/sdX` with the actual whole USB drive. The JSON records its model, -size, insertion identity, exact image hash and an exact `confirmation` string. -Review these before copying the full phrase into the write command: +Replace `/dev/sdX` with the actual destination. The JSON identifies the drive, +capacity, image hash and exact confirmation phrase. Check those values before +writing: this replaces the selected drive's contents. ```sh sudo /absolute/path/to/fds-cartridge write out/usb-preview.json \ --confirm 'COPY THE EXACT confirmation VALUE FROM THE PREVIEW' ``` -The writer rechecks source and target identity, requires a USB whole disk, -rejects mounted/protected storage, writes the **full image**, flushes and verifies -readback. On larger drives it relocates the backup GPT to the end. A replaced -USB drive or changed image requires a fresh preview. This destroys the selected -drive's existing contents. No physical drive has been written by the automated -acceptance tests. +The writer rechecks identity, writes the entire disk image, flushes it and verifies +readback. On a larger drive it moves the backup GPT to the end. A changed image +or reinserted drive requires a fresh preview. Use the image-tool runner for +preview and inspection if your EROFS utilities are in the local container. -To rehearse on a disposable file without root or USB hardware: +To rehearse with a disposable regular file: ```sh -truncate -s 64M out/disposable-usb.img -fds-cartridge preview out/demo-tools.img out/disposable-usb.img \ - out/file-preview.json --file-target +truncate -s "$(stat -c %s out/demo-tools.img)" out/disposable-usb.img +fds-cartridge preview out/demo-tools.img out/disposable-usb.img out/file-preview.json --file-target fds-cartridge write out/file-preview.json --confirm 'COPY THE EXACT confirmation VALUE' -fds-cartridge inspect out/disposable-usb.img ``` -The file must be at least as large as the source image. Use `--image-tool-runner` -for preview/inspection if EROFS tools are provided by the local wrapper. - -## Troubleshooting - -| Symptom | What to do | -| --- | --- | -| Required executable not found | Run the corresponding `doctor`; install the named host utility or select an explicit runner | -| Bubblewrap namespace failure | Enable unprivileged user namespaces according to your workstation policy; do not run creation/inspection as root | -| Output already exists | Use a new bundle/image/preview/session name; creation does not overwrite outputs | -| Unknown `fds.emulator` option or missing emulator settings | Rebuild both the current initramfs and SYSTEM; the frozen 0.1.0 images predate this feature | -| Console in use | Detach with Ctrl-] before guest commands, safe eject, or shutdown | -| Guest command timed out | Inspect `console.log`; the command may still be running, so do not blindly repeat a write | -| Incomplete insertion | Run `unplug BAY` to reconcile the recorded intent with actual QEMU devices, then reinsert | -| Software digest/path/architecture error | Rebuild the bundle and cartridge on the workstation; the guest will not run an invalid archive | -| Cache limit exceeded | Reduce the software bundle; one runtime tree including inode overhead is limited to 256 MiB | -| Safe eject blocked | Close unmanaged processes or extra mounts using the media, then retry; `unplug` is only for deliberate failure simulation | -| VM will not boot | Read session `console.log` and `qemu.log`; verify all three supplied boot artifacts belong to the current build | - -QEMU exercises the actual Linux/FDS software path. It does not emulate Pi -firmware, RP1, USB power sequencing, the physical twelve-bay wiring, or Dasung -power recovery. Those acceptance checks remain hardware procedures. - -## Physical Pi acceptance procedure (deferred) - -After assembling and calibrating the physical bay map, use a disposable USB -cartridge and the current matching SYSTEM/initramfs. Perform these checks on the -Pi; the VM results do not replace them: - -1. Build the hello/report cartridge on the workstation, record its image hash, - and use preview/confirmation to write that entire image to the chosen USB drive. -2. Insert it in a calibrated bay. Record `fds --json bay BAY`; confirm both - software entries and their declared payload partitions appear. -3. Run `demo.hello:hello` and `demo.report:report`. Save their output from the - cartridge log and confirm UID 1000. Check `/proc/self/mountinfo` for read-only - payload and software-cache mounts. -4. Run `fds eject BAY`, verify SAFE, then remove and reinsert the cartridge in - another calibrated bay. Repeat both commands. Record any USB enumeration or - I/O errors. -5. With a disposable cartridge and a deliberately long-running test program, - test surprise removal separately. Verify its managed processes terminate and - all of that bay's software mounts disappear. Do not use valuable DATA for - this failure test. -6. Exercise native shutdown with active software, cold boot, and a real USB - power cycle. Save actual observations and kernel/service logs. Follow the - separate [Dasung hardware procedure](dasung.md) for monitor recovery. - -Record the Pi, hub, USB drive and kernel versions with the result. Hardware -latency, electrical behavior and data durability remain unverified until these -measurements are performed on the assembled machine. +See [troubleshooting](troubleshooting.md) for failed builds, namespace errors, +busy consoles, invalid media and blocked ejects. diff --git a/examples/software/cartridge.toml b/examples/software/cartridge.toml index 77324b9..22da73e 100644 --- a/examples/software/cartridge.toml +++ b/examples/software/cartridge.toml @@ -1,11 +1,11 @@ -format = 1 +format = 2 id = "demo.tools" name = "Hello and system report" version = "1.0" # Paths are relative to this recipe. Each payload entry creates one partition. [[payload]] -bundles = ["../../out/demo-hello"] +sources = ["hello/software.toml"] [[payload]] -bundles = ["../../out/demo-report"] +sources = ["report/software.toml"] diff --git a/examples/software/hello/software.toml b/examples/software/hello/software.toml index 4f96049..d9254ac 100644 --- a/examples/software/hello/software.toml +++ b/examples/software/hello/software.toml @@ -1,14 +1,12 @@ -format = 1 +format = 2 id = "demo.hello" name = "AArch64 hello" version = "1.0" -architecture = "aarch64" -root = "root" [commands] -hello = "bin/hello" +hello = "usr/bin/hello" -# This trusted command runs only on the workstation. It is never put on media. -[build] -directory = "." -command = ["sh", "-eu", "-c", "mkdir -p root/bin; aarch64-linux-gnu-gcc -O2 hello.c -o root/bin/hello"] +# Void source templates run only on the workstation. +[source] +package = "fds-demo-hello" +template = "void" diff --git a/examples/software/hello/hello.c b/examples/software/hello/void/files/hello.c similarity index 100% rename from examples/software/hello/hello.c rename to examples/software/hello/void/files/hello.c diff --git a/examples/software/hello/void/template b/examples/software/hello/void/template new file mode 100644 index 0000000..5ca62c0 --- /dev/null +++ b/examples/software/hello/void/template @@ -0,0 +1,15 @@ +# Small example of a native Void source package, cross-built by xbps-src. +pkgname=fds-demo-hello +version=1.0 +revision=1 +archs="aarch64" +short_desc="FDS cartridge hello example" +maintainer="FDS/OS maintainers" +license="MIT" +homepage="https://docs.voidlinux.org/xbps/" +do_build() { + ${CC} ${CFLAGS} "${FILESDIR}/hello.c" ${LDFLAGS} -o hello +} +do_install() { + vbin hello +} diff --git a/examples/software/report/software.toml b/examples/software/report/software.toml index f3f811f..fbdf1da 100644 --- a/examples/software/report/software.toml +++ b/examples/software/report/software.toml @@ -1,9 +1,11 @@ -format = 1 +format = 2 id = "demo.report" name = "System report" version = "1.0" -architecture = "any" -root = "root" [commands] -report = "bin/report" +report = "usr/bin/report" + +[source] +package = "fds-demo-report" +template = "void" diff --git a/examples/software/report/root/bin/report b/examples/software/report/void/files/report similarity index 57% rename from examples/software/report/root/bin/report rename to examples/software/report/void/files/report index 020ee80..c5d5542 100755 --- a/examples/software/report/root/bin/report +++ b/examples/software/report/void/files/report @@ -2,3 +2,4 @@ printf 'FDS cartridge system report\n' uname -m id +[ "${1-}" != hold ] || exec tail -f /dev/null diff --git a/examples/software/report/void/template b/examples/software/report/void/template new file mode 100644 index 0000000..5e141fa --- /dev/null +++ b/examples/software/report/void/template @@ -0,0 +1,12 @@ +pkgname=fds-demo-report +version=1.0 +revision=1 +archs="aarch64" +depends="bash coreutils" +short_desc="FDS cartridge report example" +maintainer="FDS/OS maintainers" +license="MIT" +homepage="https://docs.voidlinux.org/xbps/" +do_install() { + vbin "${FILESDIR}/report" +} diff --git a/image/build-initramfs b/image/build-initramfs index a521ac4..ce47f02 100755 --- a/image/build-initramfs +++ b/image/build-initramfs @@ -21,7 +21,7 @@ args = parser.parse_args() payloads = {'fds-stage0': args.stage0.resolve(strict=True), 'dasungd': project/'out/dasungd'} for tool in ('lz4', 'zstd'): - if not shutil.which(tool): sys.exit(f'ERROR: missing host {tool}; see docs/m4-work.md') + if not shutil.which(tool): sys.exit(f'ERROR: missing host {tool}; see docs/developer/m4-work.md') for executable in ('fds-stage0', 'dasungd'): subprocess.run([str(project/'tools/verify-elf'), str(payloads[executable]), 'aarch64', 'static'], check=True) epoch = int(subprocess.check_output(['git', '-C', str(project/'vendor/void-packages'), 'show', '-s', '--format=%ct', 'HEAD'])) diff --git a/packages/fds-base-files/files/profile b/packages/fds-base-files/files/profile index a7d9d81..a48afc6 100644 --- a/packages/fds-base-files/files/profile +++ b/packages/fds-base-files/files/profile @@ -1,5 +1,5 @@ # English UTF-8 by default. Interactive shells are userspace, never PID 1. -export PATH=/usr/bin:/bin +export PATH=/usr/bin:/bin:/run/fds/bin export LANG=en_US.UTF-8 export TZ=UTC umask 022 diff --git a/packages/fds-cli/template b/packages/fds-cli/template index 6d39ab8..748c565 100644 --- a/packages/fds-cli/template +++ b/packages/fds-cli/template @@ -14,6 +14,8 @@ do_install() { local input="${XBPS_SRCDISTDIR}/${pkgname}-${version}" (cd "$input" && sha256sum -c SHA256SUMS) || return 1 vbin "${input}/fds" + vbin "${input}/fds-program" + vbin "${input}/fds-control" vbin "${input}/fds-boottrace" vbin "${input}/fds-burn" vbin "${input}/fds-inspect" diff --git a/packages/fds-eink/files/WMRootMenu b/packages/fds-eink/files/WMRootMenu index 366c4f0..2a29d27 100644 --- a/packages/fds-eink/files/WMRootMenu +++ b/packages/fds-eink/files/WMRootMenu @@ -1,4 +1,5 @@ ("FDS/OS", + ("FDS Control", EXEC, "fds-control"), ("Terminal", EXEC, "/usr/libexec/fds/terminal"), ("Enable Ethernet", EXEC, "fds network on"), ("Disable Ethernet", EXEC, "fds network off"), diff --git a/packages/fds-eink/files/terminal b/packages/fds-eink/files/terminal index 9259011..0aa4892 100644 --- a/packages/fds-eink/files/terminal +++ b/packages/fds-eink/files/terminal @@ -1,5 +1,6 @@ #!/bin/bash export PS1='FDS> ' +export PATH=/usr/bin:/bin:/run/fds/bin exec /usr/bin/xterm -class FDS -name fds-terminal -title 'FDS Terminal' \ -bg white -fg black -cr black -fn '-xos4-terminus-medium-r-normal--20-200-72-72-c-100-iso10646-1' \ -ut -bc -uc -sb -geometry 100x35+40+40 -e /bin/bash --noprofile --norc -i diff --git a/packages/fds-eink/files/windowmaker-session b/packages/fds-eink/files/windowmaker-session index e1e6c09..7333d6a 100644 --- a/packages/fds-eink/files/windowmaker-session +++ b/packages/fds-eink/files/windowmaker-session @@ -10,4 +10,5 @@ done xset s off xset -dpms xsetroot -solid white +fds-control & exec wmaker --no-dock --no-clip --no-autolaunch diff --git a/packages/fds-init/files/console-session b/packages/fds-init/files/console-session index 8730ba0..e2e7281 100755 --- a/packages/fds-init/files/console-session +++ b/packages/fds-init/files/console-session @@ -7,11 +7,11 @@ if [[ $(cat /usr/share/fds/image-profile) == recovery ]]; then cd /run/fds/recovery-home printf '\nFDS RECOVERY — LOCAL MAINTENANCE CONSOLE\nDATA stays read-only until explicitly selected.\nUse fds recovery help for inspection and repair commands.\n' exec /usr/bin/env -i HOME=/run/fds/recovery-home USER=root LOGNAME=root \ - SHELL=/bin/bash PATH=/usr/bin:/bin LANG=en_US.UTF-8 TERM="${TERM:-linux}" \ + SHELL=/bin/bash PATH=/usr/bin:/bin:/run/fds/bin LANG=en_US.UTF-8 TERM="${TERM:-linux}" \ /bin/bash --login fi cd /home/fds exec /usr/bin/s6-setuidgid fds /usr/bin/env -i \ HOME=/home/fds USER=fds LOGNAME=fds SHELL=/bin/bash \ - PATH=/usr/bin:/bin LANG=en_US.UTF-8 TERM="${TERM:-linux}" \ + PATH=/usr/bin:/bin:/run/fds/bin LANG=en_US.UTF-8 TERM="${TERM:-linux}" \ /bin/bash --login diff --git a/rust/fds-cartridged/src/consumers.rs b/rust/fds-cartridged/src/consumers.rs index c51ed66..360a5d0 100644 --- a/rust/fds-cartridged/src/consumers.rs +++ b/rust/fds-cartridged/src/consumers.rs @@ -69,11 +69,36 @@ pub fn start( ) -> Result { start_group(&format!("bay{bay}"), arguments, working, environment) } +pub fn foreground( + bay: Bay, + arguments: &[String], + working: &str, + environment: &[(String, String)], + descriptors: [OwnedFd; 3], + terminal: bool, +) -> Result { + spawn( + &format!("bay{bay}"), + arguments, + working, + environment, + Some((descriptors, terminal)), + ) +} pub fn start_group( name: &str, arguments: &[String], working: &str, environment: &[(String, String)], +) -> Result { + spawn(name, arguments, working, environment, None) +} +fn spawn( + name: &str, + arguments: &[String], + working: &str, + environment: &[(String, String)], + io: Option<([OwnedFd; 3], bool)>, ) -> Result { if !fds_common::manifest::identifier(name) { return Err(Error("Invalid process group".into())); @@ -95,11 +120,11 @@ pub fn start_group( .custom_flags(libc::O_CLOEXEC) .open(path.join("cgroup.procs"))?; let mut command = Command::new(&arguments[0]); + let working = c(working)?; command .args(&arguments[1..]) - .current_dir(working) .env_clear() - .env("PATH", "/usr/bin:/bin") + .env("PATH", "/usr/bin:/bin:/run/fds/bin") .env("HOME", "/home/fds") .env("USER", "fds") .env("LOGNAME", "fds") @@ -109,6 +134,14 @@ pub fn start_group( .stdout(Stdio::inherit()) .stderr(Stdio::inherit()); command.envs(environment.iter().cloned()); + let mut terminal = false; + if let Some(([input, output, errors], tty)) = io { + command + .stdin(Stdio::from(input)) + .stdout(Stdio::from(output)) + .stderr(Stdio::from(errors)); + terminal = tty; + } // Only async-signal-safe syscalls are used in the forked child. Writing 0 // moves the child itself, avoiding PID reuse and parent/child migration races. unsafe { @@ -119,6 +152,9 @@ pub fn start_group( if libc::setsid() < 0 { return Err(io::Error::last_os_error()); } + if terminal && libc::ioctl(0, libc::TIOCSCTTY, 0) < 0 { + return Err(io::Error::last_os_error()); + } if libc::setgroups(0, std::ptr::null()) < 0 || libc::setgid(1000) < 0 || libc::setuid(1000) < 0 @@ -128,6 +164,10 @@ pub fn start_group( if libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) < 0 { return Err(io::Error::last_os_error()); } + // Resolve client-selected working directories only as the user. + if libc::chdir(working.as_ptr()) < 0 { + return Err(io::Error::last_os_error()); + } let mut mask: libc::sigset_t = std::mem::zeroed(); libc::sigemptyset(&mut mask); if libc::sigprocmask(libc::SIG_SETMASK, &mask, std::ptr::null_mut()) < 0 { diff --git a/rust/fds-cartridged/src/main.rs b/rust/fds-cartridged/src/main.rs index 345d274..06b91c9 100644 --- a/rust/fds-cartridged/src/main.rs +++ b/rust/fds-cartridged/src/main.rs @@ -4,6 +4,7 @@ mod data_sessions; mod media; mod power; mod profiles; +mod programs; mod recovery; mod server; mod software; diff --git a/rust/fds-cartridged/src/media.rs b/rust/fds-cartridged/src/media.rs index ac0cbe6..d887415 100644 --- a/rust/fds-cartridged/src/media.rs +++ b/rust/fds-cartridged/src/media.rs @@ -228,7 +228,7 @@ impl Mounted { ("FDS_APP".into(), app.display().to_string()), ( "PATH".into(), - format!("{}/bin:/usr/bin:/bin", app.display()), + format!("{}/bin:/usr/bin:/bin:/run/fds/bin", app.display()), ), ( "LD_LIBRARY_PATH".into(), diff --git a/rust/fds-cartridged/src/programs.rs b/rust/fds-cartridged/src/programs.rs new file mode 100644 index 0000000..15d0dc4 --- /dev/null +++ b/rust/fds-cartridged/src/programs.rs @@ -0,0 +1,88 @@ +//! A stable PATH directory is updated as validated cartridges appear/disappear. +use crate::media::Mounted; +use fds_common::{ + Bay, Error, Result, + manifest::{Class, identifier}, +}; +use std::{ + collections::BTreeMap, + fs, + os::unix::fs::{PermissionsExt, symlink}, + path::Path, +}; +pub const BIN: &str = "/run/fds/bin"; +pub type Commands = BTreeMap; + +pub fn collect(mounts: &BTreeMap) -> Result { + let mut result = BTreeMap::new(); + for (&bay, mount) in mounts { + if mount.manifest.cartridge.class != Class::Program || mount.fault.is_some() { + continue; + } + let mut commands = Vec::new(); + if let Some(software) = &mount.software { + for entry in &software.catalogue.software { + for name in entry.commands.keys() { + commands.push(( + name.clone(), + format!("{}:{name}", entry.id), + format!("b{bay}:{}:{name}", entry.id), + )); + } + } + } else { + let root = Path::new(&mount.path).join("app/bin"); + if root.is_dir() { + for entry in fs::read_dir(root)? { + let entry = entry?; + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + if identifier(&name) + && entry.path().is_file() + && entry + .path() + .canonicalize()? + .starts_with(Path::new(&mount.path).join("app")) + && entry.path().metadata()?.permissions().mode() & 0o111 != 0 + { + commands.push((name.clone(), name.clone(), format!("b{bay}:{name}"))); + } + } + } + } + commands.sort(); + for (name, selector, qualified) in commands { + result.entry(name).or_insert((bay, selector.clone())); + // A fully qualified spelling always identifies this cartridge. + result.insert(qualified, (bay, selector)); + } + } + Ok(result) +} +pub fn publish(commands: &Commands) -> Result<()> { + fs::create_dir_all(BIN)?; + fs::set_permissions(BIN, fs::Permissions::from_mode(0o755))?; + for entry in fs::read_dir(BIN)? { + let entry = entry?; + if !commands.contains_key(&entry.file_name().to_string_lossy().into_owned()) { + if !entry.file_type()?.is_symlink() { + return Err(Error( + "Unexpected file in cartridge command directory".into(), + )); + } + fs::remove_file(entry.path())?; + } + } + for name in commands.keys() { + let path = Path::new(BIN).join(name); + if path.symlink_metadata().is_ok() { + if fs::read_link(&path)? != Path::new("/usr/bin/fds-program") { + return Err(Error("Unexpected cartridge command link".into())); + } + } else { + symlink("/usr/bin/fds-program", path)?; + } + } + Ok(()) +} diff --git a/rust/fds-cartridged/src/server.rs b/rust/fds-cartridged/src/server.rs index b2a7c0f..998cdcf 100644 --- a/rust/fds-cartridged/src/server.rs +++ b/rust/fds-cartridged/src/server.rs @@ -32,6 +32,7 @@ struct State { profiles: profiles::Manager, burning: burning::Manager, power: power::Manager, + commands: crate::programs::Commands, } impl State { fn scan(&mut self) -> Result<()> { @@ -63,6 +64,7 @@ impl State { software: None, mount: None, consumers: consumers::count(bay)?, + commands: Vec::new(), }; // A hub in a bay may contain several functions, but multiple actual // devices are ambiguous until an explicit composite policy exists. @@ -109,8 +111,113 @@ impl State { if !self.power.frozen() { self.profiles.reconcile(&self.mounts, &devices)?; } + self.commands = crate::programs::collect(&self.mounts)?; + crate::programs::publish(&self.commands)?; + for entry in &mut self.bays { + entry.commands = self + .commands + .iter() + .filter_map(|(alias, (bay, selector))| { + (*bay == entry.bay && alias.starts_with(&format!("b{bay}:"))).then(|| { + fds_common::control::PublishedCommand { + selector: selector.clone(), + alias: alias.clone(), + } + }) + }) + .collect(); + } Ok(()) } + fn program(&mut self, name: &str) -> Result<(Bay, fds_common::launch::Program)> { + if self.recovery || self.power.frozen() { + return Err(Error( + "Program launches are unavailable during recovery or shutdown".into(), + )); + } + let (bay, selector) = self + .commands + .get(name) + .cloned() + .ok_or_else(|| Error(format!("Cartridge command {name} is no longer available")))?; + let mount = self + .mounts + .get_mut(&bay) + .ok_or_else(|| Error("Cartridge was removed".into()))?; + let (arguments, environment) = mount.program(&[selector])?; + Ok(( + bay, + fds_common::launch::Program { + arguments, + environment, + }, + )) + } + fn foreground( + &mut self, + name: &str, + arguments: &[String], + terminal: bool, + working: &str, + term: &str, + client: &mut Client, + ) -> Result { + self.program(name)?; + if arguments.len() > 120 + || arguments.iter().any(|a| a.contains('\0')) + || !working.starts_with('/') + || working.len() > 4096 + || working.contains('\0') + || term.len() > 128 + || term.chars().any(char::is_control) + { + return Err(Error( + "Invalid foreground program arguments or environment".into(), + )); + } + client.socket.set_nonblocking(false)?; + client + .socket + .set_read_timeout(Some(Duration::from_secs(5)))?; + client + .socket + .set_write_timeout(Some(Duration::from_secs(5)))?; + let result = (|| { + client.socket.write_all(b"R")?; + let descriptors: [OwnedFd; 3] = fds_common::launch::receive_fds(&mut client.socket, 3)? + .try_into() + .map_err(|_| Error("Expected three program I/O descriptors".into()))?; + let (bay, mut program) = self.program(name)?; + program.arguments.extend_from_slice(arguments); + program.environment.push(("TERM".into(), term.into())); + let mut child = consumers::foreground( + bay, + &program.arguments, + working, + &program.environment, + descriptors, + terminal, + )?; + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, child.id(), 0) } as i32; + if fd < 0 { + let _ = child.kill(); + self.children.push(child); + return Err(std::io::Error::last_os_error().into()); + } + let pidfd = unsafe { OwnedFd::from_raw_fd(fd) }; + if let Err(error) = fds_common::launch::send_fds(&client.socket, &[pidfd.as_raw_fd()]) { + let _ = child.kill(); + self.children.push(child); + return Err(error); + } + client.foreground = Some(child); + let mut reply = Response::failure(""); + reply.error = None; + Ok(reply) + })(); + client.socket.set_nonblocking(true)?; + result + } fn mapped_devices(&self) -> Vec<(Bay, topology::UsbDevice)> { self.bays .iter() @@ -628,6 +735,7 @@ impl State { }, media_job: None, recovery: None, + exit_status: None, disk: None, power: None, }) @@ -796,6 +904,7 @@ struct Client { offset: usize, deadline: Instant, uid: u32, + foreground: Option, waiting: Option<(String, u64)>, } fn peer_uid(socket: &UnixStream) -> Result> { @@ -910,6 +1019,7 @@ pub fn run(notify: bool) -> Result<()> { profiles: profiles::Manager::new(!recovery), burning: burning::Manager::load()?, power: power::Manager::load()?, + commands: BTreeMap::new(), }; for n in 1..=12 { let bay = Bay::try_from(n)?; @@ -965,6 +1075,7 @@ pub fn run(notify: bool) -> Result<()> { state.profiles.shutdown()?; } cleanup_stale_mounts()?; + crate::programs::publish(&BTreeMap::new())?; state.scan()?; let mut clients: Vec = Vec::new(); loop { @@ -1000,7 +1111,7 @@ pub fn run(notify: bool) -> Result<()> { fd: client.socket.as_raw_fd(), events: if client.output.is_some() { libc::POLLOUT - } else if client.waiting.is_some() { + } else if client.waiting.is_some() || client.foreground.is_some() { 0 } else { libc::POLLIN @@ -1010,6 +1121,7 @@ pub fn run(notify: bool) -> Result<()> { } let timeout = clients .iter() + .filter(|c| c.foreground.is_none()) .map(|c| { c.deadline .saturating_duration_since(Instant::now()) @@ -1071,9 +1183,25 @@ pub fn run(notify: bool) -> Result<()> { if (fds[0].revents != 0 && consume_events(&events)?) || console_changed { state.scan()?; } + // Reserve connection capacity for eject/status even with many foreground jobs. + let foreground_count = clients.iter().filter(|c| c.foreground.is_some()).count(); // Process existing clients before accepting more; vectors stay aligned. for index in (0..clients.len()).rev() { let client = &mut clients[index]; + if let Some(child) = &mut client.foreground { + if let Some(status) = child.try_wait()? { + use std::os::unix::process::ExitStatusExt; + let mut reply = Response::failure(""); + reply.error = None; + reply.exit_status = Some(status.into_raw()); + let mut output = + serde_json::to_vec(&reply).map_err(|e| Error(e.to_string()))?; + output.push(b'\n'); + client.output = Some(output); + client.foreground = None; + client.deadline = Instant::now() + Duration::from_secs(5); + } + } let ready = fds[index + 5].revents; if let Some((id, sequence)) = &client.waiting { let job = state.burning.status(id); @@ -1097,8 +1225,8 @@ pub fn run(notify: bool) -> Result<()> { client.deadline = Instant::now() + Duration::from_secs(5); } } - let mut remove = - Instant::now() >= client.deadline || ready & (libc::POLLERR | libc::POLLNVAL) != 0; + let mut remove = (client.foreground.is_none() && Instant::now() >= client.deadline) + || ready & (libc::POLLERR | libc::POLLNVAL) != 0; if !remove && ready & libc::POLLIN != 0 && client.output.is_none() { let mut chunk = [0u8; 4096]; match client.socket.read(&mut chunk) { @@ -1124,9 +1252,31 @@ pub fn run(notify: bool) -> Result<()> { client.waiting = Some((id.clone(), *sequence)); } } - state - .reply(request, client.uid) - .unwrap_or_else(Response::failure) + if let Request::Program { + name, + arguments, + terminal, + working, + term, + } = request + { + if foreground_count >= 64 { + Response::failure( + "Too many foreground programs; close a program and retry", + ) + } else { + state + .foreground( + &name, &arguments, terminal, &working, + &term, client, + ) + .unwrap_or_else(Response::failure) + } + } else { + state + .reply(request, client.uid) + .unwrap_or_else(Response::failure) + } } Err(_) => Response::failure("Invalid control request"), } @@ -1140,7 +1290,9 @@ pub fn run(notify: bool) -> Result<()> { .unwrap(); } output.push(b'\n'); - if client.waiting.is_some() { + if client.foreground.is_some() { + // SIGCHLD wakes the loop when this foreground command exits. + } else if client.waiting.is_some() { client.deadline = Instant::now() + Duration::from_secs(90); } else { client.output = Some(output); @@ -1173,7 +1325,11 @@ pub fn run(notify: bool) -> Result<()> { remove = true; } if remove { - clients.swap_remove(index); + let mut removed = clients.swap_remove(index); + if let Some(mut child) = removed.foreground.take() { + let _ = child.kill(); + state.children.push(child); + } } } if fds[1].revents & libc::POLLIN != 0 { @@ -1181,7 +1337,7 @@ pub fn run(notify: bool) -> Result<()> { match listener.accept() { Ok((socket, _)) => { let uid = peer_uid(&socket)?; - if clients.len() >= 16 || uid.is_none() { + if clients.len() >= 128 || uid.is_none() { continue; } socket.set_nonblocking(true)?; @@ -1192,6 +1348,7 @@ pub fn run(notify: bool) -> Result<()> { offset: 0, deadline: Instant::now() + Duration::from_secs(5), uid: uid.unwrap(), + foreground: None, waiting: None, }); } diff --git a/rust/fds-cartridged/src/software.rs b/rust/fds-cartridged/src/software.rs index 717cc46..008583a 100644 --- a/rust/fds-cartridged/src/software.rs +++ b/rust/fds-cartridged/src/software.rs @@ -2,7 +2,7 @@ use crate::media::{self, c, checked}; use fds_burn::{device::Disk, image}; use fds_common::{Bay, Error, Result, read_text, sysfs::BlockPartition}; -use fds_software::{Catalogue, archive}; +use fds_software::{Catalogue, archive, tree}; use std::{ collections::{BTreeMap, BTreeSet}, fs::{self, File, OpenOptions}, @@ -136,7 +136,14 @@ impl Mounted { &format!("/proc/self/fd/{}", source.as_raw_fd()), &path, "erofs", - libc::MS_RDONLY | libc::MS_NOEXEC | libc::MS_NOSUID | libc::MS_NODEV, + libc::MS_RDONLY + | libc::MS_NOSUID + | libc::MS_NODEV + | if result.catalogue.format == 1 { + libc::MS_NOEXEC + } else { + 0 + }, "", )?; result.payloads.insert( @@ -148,7 +155,11 @@ impl Mounted { key, }, ); - let bundles = Path::new(&path).join("bundles"); + let bundles = Path::new(&path).join(if result.catalogue.format == 2 { + "programs" + } else { + "bundles" + }); if !fs::symlink_metadata(&bundles)?.is_dir() { return Err(Error("Payload bundles must be a real directory".into())); } @@ -157,14 +168,20 @@ impl Mounted { .software .iter() .filter(|s| s.partition == spec.number) - .map(|s| format!("{}.tar.xz", s.id)) + .map(|s| { + if s.installed { + s.id.clone() + } else { + format!("{}.tar.xz", s.id) + } + }) .collect(); let actual: BTreeSet<_> = fs::read_dir(&bundles)? .map(|e| Ok(e?.file_name().to_string_lossy().into_owned())) .collect::>()?; - if actual != expected { + if actual != expected || fs::read_dir(&path)?.count() != 1 { return Err(Error( - "Payload archive inventory disagrees with catalogue".into(), + "Payload software inventory disagrees with catalogue".into(), )); } for software in result @@ -173,6 +190,10 @@ impl Mounted { .iter() .filter(|s| s.partition == spec.number) { + if software.installed { + tree::verify(&Path::new(&path).join(software.root_path()), software)?; + continue; + } if archive::open(&Path::new(&path).join(software.archive_path()))? .metadata()? .len() @@ -226,6 +247,32 @@ impl Mounted { if media::key(&payload.partition)? != payload.key { return Err(Error("Software payload was removed".into())); } + if software.installed { + let root = Path::new(&payload.path).join(software.root_path()); + let mut args = tree::executable(&root, executable)?; + args.extend_from_slice(&arguments[1..]); + let root = root.display().to_string(); + return Ok(( + args, + vec![ + ("FDS_APP".into(), root.clone()), + ( + "PATH".into(), + format!("{root}/usr/bin:{root}/bin:/usr/bin:/bin:/run/fds/bin"), + ), + ( + "LD_LIBRARY_PATH".into(), + format!("{root}/usr/lib:{root}/lib"), + ), + ( + "XDG_DATA_DIRS".into(), + format!("{root}/usr/share:/usr/share"), + ), + ("DISPLAY".into(), ":0".into()), + ("XAUTHORITY".into(), "/run/fds/x11/authority".into()), + ], + )); + } if !self.caches.contains_key(id) { // Each executable tree receives its own bounded, read-only tmpfs. // Root owns every path; the consumer only receives ordinary UID 1000. diff --git a/rust/fds-cli/Cargo.toml b/rust/fds-cli/Cargo.toml index 951449a..c66dd5b 100644 --- a/rust/fds-cli/Cargo.toml +++ b/rust/fds-cli/Cargo.toml @@ -9,6 +9,10 @@ description = "FDS system and cartridge command interface" name = "fds" path = "src/main.rs" +[[bin]] +name = "fds-program" +path = "src/program.rs" + [dependencies] clap.workspace = true fds-burn = { path = "../fds-burn" } diff --git a/rust/fds-cli/src/main.rs b/rust/fds-cli/src/main.rs index ab823ca..3e5df5c 100644 --- a/rust/fds-cli/src/main.rs +++ b/rust/fds-cli/src/main.rs @@ -159,6 +159,7 @@ fn inspect_manifest(path: &Path, json: bool) -> Result<()> { } fn cartridge(request: Request, json: bool) -> Result<()> { let debug = matches!(request, Request::Topology); + let details = matches!(request, Request::Bay { .. }); let response = control::request(&request)?; if json { println!( @@ -237,6 +238,11 @@ fn cartridge(request: Request, json: bool) -> Result<()> { if let Some(mount) = bay.mount { println!(" MOUNT {mount}"); } + if details { + for command in bay.commands { + println!(" FOREGROUND {}", command.alias); + } + } if bay.consumers > 0 { println!(" MANAGED PROCESSES {}", bay.consumers); } diff --git a/rust/fds-cli/src/program.rs b/rust/fds-cli/src/program.rs new file mode 100644 index 0000000..eddbc19 --- /dev/null +++ b/rust/fds-cli/src/program.rs @@ -0,0 +1,142 @@ +//! Foreground cartridge commands preserve terminal I/O and managed process tracking. +mod program_io; +use clap::{Parser, Subcommand}; +use fds_common::{Error, Result, launch}; +use std::{ffi::OsString, path::PathBuf, process::ExitCode}; + +// Parse argv[0] as a typed positional: Clap multicall uses file_stem(), which +// removes dotted software IDs. All options/arguments still go through Clap. +#[derive(Parser)] +#[command(no_binary_name = true, disable_help_flag = true)] +struct Invocation { + executable: PathBuf, + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + arguments: Vec, +} +#[derive(Parser)] +#[command( + name = "fds-program", + version, + disable_help_subcommand = true, + about = "Run a published cartridge command in the foreground" +)] +struct Explicit { + #[command(subcommand)] + command: Published, +} +#[derive(Subcommand)] +enum Published { + #[command(external_subcommand)] + Command(Vec), +} +impl Invocation { + fn command(self) -> std::result::Result<(String, Vec), clap::Error> { + let name = self + .executable + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| { + clap::Error::raw( + clap::error::ErrorKind::InvalidUtf8, + "Invalid cartridge command name", + ) + })? + .to_owned(); + if name == "fds-program" { + let parsed = Explicit::try_parse_from( + std::iter::once(self.executable.into_os_string()).chain(self.arguments), + )?; + let Published::Command(mut arguments) = parsed.command; + let command = arguments.remove(0).into_string().map_err(|_| { + clap::Error::raw( + clap::error::ErrorKind::InvalidUtf8, + "Invalid cartridge command name", + ) + })?; + Ok((command, arguments)) + } else { + Ok((name, self.arguments)) + } + } +} +fn run() -> Result { + let (name, arguments) = Invocation::parse() + .command() + .unwrap_or_else(|error| error.exit()); + if unsafe { libc::geteuid() } == 0 { + if unsafe { libc::setgroups(0, std::ptr::null()) } < 0 + || unsafe { libc::setgid(1000) } < 0 + || unsafe { libc::setuid(1000) } < 0 + { + return Err(std::io::Error::last_os_error().into()); + } + } + let io = program_io::Io::new()?; + let arguments = arguments + .into_iter() + .map(|value| { + value + .into_string() + .map_err(|_| Error("Program arguments must be UTF-8".into())) + }) + .collect::>>()?; + let (socket, child) = launch::start( + &name, + arguments, + io.terminal(), + std::env::current_dir()?.display().to_string(), + std::env::var("TERM").unwrap_or_else(|_| "linux".into()), + &io.descriptors(), + )?; + io.wait(socket, child) +} +fn main() -> ExitCode { + match run() { + Ok(status) => ExitCode::from(if libc::WIFEXITED(status) { + libc::WEXITSTATUS(status) as u8 + } else { + (128 + libc::WTERMSIG(status)) as u8 + }), + Err(error) => { + eprintln!("fds-program: {error}"); + ExitCode::from(126) + } + } +} +#[cfg(test)] +mod tests { + use super::*; + use clap::CommandFactory; + #[test] + fn clap_preserves_dotted_aliases_and_application_options() { + ::command().debug_assert(); + Explicit::command().debug_assert(); + for name in ["hello", "hello.world", "b01:demo.report:shell"] { + let (command, arguments) = Invocation::try_parse_from([ + format!("/run/fds/bin/{name}"), + "--help".into(), + "two words".into(), + ]) + .unwrap() + .command() + .unwrap(); + assert_eq!(command, name); + assert_eq!(arguments, ["--help", "two words"]); + } + let (command, arguments) = + Invocation::try_parse_from(["fds-program", "hello", "--version"]) + .unwrap() + .command() + .unwrap(); + assert_eq!(command, "hello"); + assert_eq!(arguments, ["--version"]); + assert_eq!( + Invocation::try_parse_from(["fds-program", "--help"]) + .unwrap() + .command() + .unwrap_err() + .kind(), + clap::error::ErrorKind::DisplayHelp + ); + } +} diff --git a/rust/fds-cli/src/program_io.rs b/rust/fds-cli/src/program_io.rs new file mode 100644 index 0000000..70c9e3a --- /dev/null +++ b/rust/fds-cli/src/program_io.rs @@ -0,0 +1,345 @@ +//! Terminal proxy for foreground programs whose privileged parent is the service. +use fds_common::{ + Error, Result, + control::{LIMIT, Response}, +}; +use std::{ + io::{Read, Write}, + os::{ + fd::{AsRawFd, FromRawFd, OwnedFd}, + unix::net::UnixStream, + }, +}; + +pub struct Io { + master: Option, + slave: Option, + terminal: Option, + signals: OwnedFd, + old_mask: libc::sigset_t, + pending: std::collections::VecDeque, +} +fn checked(result: i32) -> Result { + if result < 0 { + Err(std::io::Error::last_os_error().into()) + } else { + Ok(result) + } +} +impl Io { + pub fn new() -> Result { + let mut mask = unsafe { std::mem::zeroed::() }; + let mut old_mask = unsafe { std::mem::zeroed::() }; + unsafe { + libc::sigemptyset(&mut mask); + for signal in [ + libc::SIGINT, + libc::SIGTERM, + libc::SIGHUP, + libc::SIGQUIT, + libc::SIGWINCH, + libc::SIGTSTP, + libc::SIGCONT, + ] { + libc::sigaddset(&mut mask, signal); + } + checked(libc::sigprocmask(libc::SIG_BLOCK, &mask, &mut old_mask))?; + } + let fd = unsafe { libc::signalfd(-1, &mask, libc::SFD_CLOEXEC | libc::SFD_NONBLOCK) }; + if fd < 0 { + let error = std::io::Error::last_os_error(); + unsafe { + libc::sigprocmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut()); + } + return Err(error.into()); + } + let signals = unsafe { OwnedFd::from_raw_fd(fd) }; + let mut io = Self { + master: None, + slave: None, + terminal: None, + signals, + old_mask, + pending: Default::default(), + }; + if unsafe { libc::isatty(0) == 1 && libc::isatty(1) == 1 } { + let mut term = unsafe { std::mem::zeroed::() }; + let mut size = unsafe { std::mem::zeroed::() }; + checked(unsafe { libc::tcgetattr(0, &mut term) })?; + checked(unsafe { libc::ioctl(0, libc::TIOCGWINSZ, &mut size) })?; + let (mut master, mut slave) = (-1, -1); + checked(unsafe { + libc::openpty(&mut master, &mut slave, std::ptr::null_mut(), &term, &size) + })?; + io.master = Some(unsafe { OwnedFd::from_raw_fd(master) }); + io.slave = Some(unsafe { OwnedFd::from_raw_fd(slave) }); + for fd in [master, slave] { + checked(unsafe { libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC) })?; + } + checked(unsafe { libc::fcntl(master, libc::F_SETFL, libc::O_NONBLOCK) })?; + io.terminal = Some(term); + io.raw()?; + } + Ok(io) + } + pub fn terminal(&self) -> bool { + self.terminal.is_some() + } + pub fn descriptors(&self) -> [i32; 3] { + if let Some(slave) = &self.slave { + [ + slave.as_raw_fd(), + slave.as_raw_fd(), + if unsafe { libc::isatty(2) } == 1 { + slave.as_raw_fd() + } else { + 2 + }, + ] + } else { + [0, 1, 2] + } + } + fn raw(&self) -> Result<()> { + if let Some(term) = self.terminal { + let mut raw = term; + unsafe { + libc::cfmakeraw(&mut raw); + } + raw.c_lflag |= libc::ISIG; + checked(unsafe { libc::tcsetattr(0, libc::TCSANOW, &raw) })?; + } + Ok(()) + } + fn restore(&self) { + if let Some(term) = self.terminal { + unsafe { + libc::tcsetattr(0, libc::TCSANOW, &term); + } + } + } + fn write_terminal(&mut self, bytes: &[u8]) -> Result<()> { + if self.pending.len() + bytes.len() > 65536 { + return Err(Error("Terminal input queue exceeded its limit".into())); + } + self.pending.extend(bytes); + Ok(()) + } + fn flush_input(&mut self) -> Result<()> { + let Some(master) = &self.master else { + return Ok(()); + }; + let bytes = self.pending.as_slices().0; + if bytes.is_empty() { + return Ok(()); + } + let count = unsafe { libc::write(master.as_raw_fd(), bytes.as_ptr().cast(), bytes.len()) }; + if count > 0 { + self.pending.drain(..count as usize); + } else if count < 0 { + let error = std::io::Error::last_os_error(); + if !matches!( + error.kind(), + std::io::ErrorKind::Interrupted | std::io::ErrorKind::WouldBlock + ) { + return Err(error.into()); + } + } + Ok(()) + } + fn output(&self) -> Result { + let Some(master) = &self.master else { + return Ok(false); + }; + loop { + let mut buffer = [0u8; 8192]; + let count = + unsafe { libc::read(master.as_raw_fd(), buffer.as_mut_ptr().cast(), buffer.len()) }; + if count > 0 { + std::io::stdout().write_all(&buffer[..count as usize])?; + continue; + } + if count == 0 { + return Ok(false); + } + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::Interrupted { + continue; + } + if error.raw_os_error() == Some(libc::EIO) { + return Ok(false); + } + if error.kind() == std::io::ErrorKind::WouldBlock { + return Ok(true); + } + return Err(error.into()); + } + } + pub fn wait(mut self, mut socket: UnixStream, child: OwnedFd) -> Result { + self.slave.take(); + let mut bytes = Vec::new(); + let mut input = self.master.is_some(); + let mut output = self.master.is_some(); + loop { + let mut fds = [ + libc::pollfd { + fd: socket.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }, + libc::pollfd { + fd: self.signals.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }, + libc::pollfd { + fd: if input && self.pending.len() < 32768 { + 0 + } else { + -1 + }, + events: libc::POLLIN, + revents: 0, + }, + libc::pollfd { + fd: if output { + self.master.as_ref().unwrap().as_raw_fd() + } else { + -1 + }, + events: libc::POLLIN + | if self.pending.is_empty() { + 0 + } else { + libc::POLLOUT + }, + revents: 0, + }, + ]; + let result = unsafe { libc::poll(fds.as_mut_ptr(), fds.len() as _, -1) }; + if result < 0 + && std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted + { + continue; + } + checked(result)?; + if fds[3].revents & (libc::POLLIN | libc::POLLHUP | libc::POLLERR) != 0 { + output = self.output()?; + } + if output && fds[3].revents & libc::POLLOUT != 0 { + self.flush_input()?; + } + if fds[2].revents != 0 { + let mut buffer = [0u8; 4096]; + let count = unsafe { libc::read(0, buffer.as_mut_ptr().cast(), buffer.len()) }; + if count > 0 { + self.write_terminal(&buffer[..count as usize])?; + } else if count == 0 { + input = false; + } else if std::io::Error::last_os_error().kind() != std::io::ErrorKind::Interrupted + { + return Err(std::io::Error::last_os_error().into()); + } + } + if fds[1].revents != 0 { + let mut info = unsafe { std::mem::zeroed::() }; + if unsafe { + libc::read( + self.signals.as_raw_fd(), + (&mut info as *mut libc::signalfd_siginfo).cast(), + std::mem::size_of_val(&info), + ) + } > 0 + { + let signal = info.ssi_signo as i32; + if signal == libc::SIGWINCH { + if let Some(master) = &self.master { + let mut size = unsafe { std::mem::zeroed::() }; + if unsafe { libc::ioctl(0, libc::TIOCGWINSZ, &mut size) } == 0 { + unsafe { + libc::ioctl(master.as_raw_fd(), libc::TIOCSWINSZ, &size); + } + } + } + } else if signal == libc::SIGTSTP { + self.restore(); + if let Some(master) = &self.master { + let group = unsafe { libc::tcgetpgrp(master.as_raw_fd()) }; + if group > 0 { + unsafe { + libc::kill(-group, libc::SIGSTOP); + } + } + } + unsafe { + libc::kill(libc::getpid(), libc::SIGSTOP); + } + self.raw()?; + if let Some(master) = &self.master { + let group = unsafe { libc::tcgetpgrp(master.as_raw_fd()) }; + if group > 0 { + unsafe { + libc::kill(-group, libc::SIGCONT); + } + } + } + } else if self.master.is_some() + && matches!(signal, libc::SIGINT | libc::SIGQUIT) + { + let index = if signal == libc::SIGINT { + libc::VINTR + } else { + libc::VQUIT + }; + self.write_terminal(&[self.terminal.as_ref().unwrap().c_cc[index]])?; + } else if signal != libc::SIGCONT { + unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + child.as_raw_fd(), + signal, + std::ptr::null::(), + 0, + ); + } + } + } + } + if fds[0].revents != 0 { + let mut chunk = [0u8; 4096]; + loop { + match socket.read(&mut chunk) { + Ok(0) => { + let _ = self.output()?; + let reply: Response = serde_json::from_slice(&bytes) + .map_err(|e| Error(format!("Invalid program completion: {e}")))?; + if let Some(error) = reply.error { + return Err(Error(error)); + } + return reply + .exit_status + .ok_or_else(|| Error("Missing program exit status".into())); + } + Ok(n) => { + bytes.extend_from_slice(&chunk[..n]); + if bytes.len() > LIMIT { + return Err(Error("Program response is too large".into())); + } + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => break, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e.into()), + } + } + } + } + } +} +impl Drop for Io { + fn drop(&mut self) { + self.restore(); + unsafe { + libc::sigprocmask(libc::SIG_SETMASK, &self.old_mask, std::ptr::null_mut()); + } + } +} diff --git a/rust/fds-common/src/control.rs b/rust/fds-common/src/control.rs index 9ede8e0..798bc67 100644 --- a/rust/fds-common/src/control.rs +++ b/rust/fds-common/src/control.rs @@ -47,6 +47,14 @@ pub enum Request { bay: Bay, arguments: Vec, }, + /// Foreground PATH launcher; stdio is passed to an ordinary managed child. + Program { + name: String, + arguments: Vec, + terminal: bool, + working: String, + term: String, + }, MediaPrepare { bay: Bay, image: String, @@ -105,6 +113,11 @@ impl MediaJob { } } #[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PublishedCommand { + pub selector: String, + pub alias: String, +} +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct BayState { pub bay: Bay, pub state: String, @@ -114,6 +127,8 @@ pub struct BayState { pub manifest: Option, pub mount: Option, pub consumers: usize, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub commands: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub software: Option, } @@ -168,6 +183,8 @@ pub struct Response { pub power: Option, #[serde(default)] pub recovery: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exit_status: Option, } impl Response { pub fn failure(error: impl ToString) -> Self { @@ -182,6 +199,7 @@ impl Response { disk: None, power: None, recovery: None, + exit_status: None, } } } diff --git a/rust/fds-common/src/launch.rs b/rust/fds-common/src/launch.rs new file mode 100644 index 0000000..a3dc4a2 --- /dev/null +++ b/rust/fds-common/src/launch.rs @@ -0,0 +1,171 @@ +//! Foreground launch transfers ordinary stdio to a managed child and receives its pidfd. +use crate::{ + Error, Result, + control::{LIMIT, Request, Response, SOCKET}, +}; +use serde::{Deserialize, Serialize}; +use std::{ + io::{Read, Write}, + os::{ + fd::{AsRawFd, FromRawFd, OwnedFd}, + unix::net::UnixStream, + }, + time::Duration, +}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Program { + pub arguments: Vec, + pub environment: Vec<(String, String)>, +} + +pub fn send_fds(socket: &UnixStream, descriptors: &[i32]) -> Result<()> { + if descriptors.is_empty() || descriptors.len() > 3 { + return Err(Error("Invalid descriptor count".into())); + } + let mut marker = b'F'; + let mut vector = libc::iovec { + iov_base: (&mut marker as *mut u8).cast(), + iov_len: 1, + }; + let mut control = [0usize; 8]; + let mut message: libc::msghdr = unsafe { std::mem::zeroed() }; + message.msg_iov = &mut vector; + message.msg_iovlen = 1; + message.msg_control = control.as_mut_ptr().cast(); + message.msg_controllen = + unsafe { libc::CMSG_SPACE(std::mem::size_of_val(descriptors) as _) } as _; + unsafe { + let header = libc::CMSG_FIRSTHDR(&message); + (*header).cmsg_level = libc::SOL_SOCKET; + (*header).cmsg_type = libc::SCM_RIGHTS; + (*header).cmsg_len = libc::CMSG_LEN(std::mem::size_of_val(descriptors) as _) as _; + for (index, descriptor) in descriptors.iter().enumerate() { + std::ptr::write_unaligned( + libc::CMSG_DATA(header).cast::().add(index), + *descriptor, + ); + } + if libc::sendmsg(socket.as_raw_fd(), &message, libc::MSG_NOSIGNAL) != 1 { + return Err(std::io::Error::last_os_error().into()); + } + } + Ok(()) +} + +pub fn receive_fds(socket: &mut UnixStream, expected: usize) -> Result> { + let mut marker = 0u8; + let mut vector = libc::iovec { + iov_base: (&mut marker as *mut u8).cast(), + iov_len: 1, + }; + let mut control = [0usize; 8]; + let mut message: libc::msghdr = unsafe { std::mem::zeroed() }; + message.msg_iov = &mut vector; + message.msg_iovlen = 1; + message.msg_control = control.as_mut_ptr().cast(); + message.msg_controllen = std::mem::size_of_val(&control) as _; + let n = unsafe { libc::recvmsg(socket.as_raw_fd(), &mut message, libc::MSG_CMSG_CLOEXEC) }; + if n < 0 { + return Err(std::io::Error::last_os_error().into()); + } + let mut received = Vec::new(); + unsafe { + let mut header = libc::CMSG_FIRSTHDR(&message); + while !header.is_null() { + if (*header).cmsg_level == libc::SOL_SOCKET && (*header).cmsg_type == libc::SCM_RIGHTS { + let count = ((*header).cmsg_len as usize - libc::CMSG_LEN(0) as usize) + / std::mem::size_of::(); + for index in 0..count { + let fd = + std::ptr::read_unaligned(libc::CMSG_DATA(header).cast::().add(index)); + received.push(OwnedFd::from_raw_fd(fd)); + } + } + header = libc::CMSG_NXTHDR(&message, header); + } + } + if n == 1 + && marker == b'F' + && message.msg_flags & libc::MSG_CTRUNC == 0 + && received.len() == expected + { + return Ok(received); + } + if n == 1 && marker == b'{' && received.is_empty() { + let mut bytes = vec![marker]; + socket.take(LIMIT as u64).read_to_end(&mut bytes)?; + if bytes.len() <= LIMIT { + if let Ok(reply) = serde_json::from_slice::(&bytes) { + if let Some(error) = reply.error { + return Err(Error(error)); + } + } + } + } + Err(Error("Invalid managed program handshake".into())) +} + +pub fn start( + name: &str, + arguments: Vec, + terminal: bool, + working: String, + term: String, + descriptors: &[i32], +) -> Result<(UnixStream, OwnedFd)> { + let mut socket = UnixStream::connect(SOCKET)?; + socket.set_read_timeout(Some(Duration::from_secs(120)))?; + socket.set_write_timeout(Some(Duration::from_secs(5)))?; + let mut request = serde_json::to_vec(&Request::Program { + name: name.into(), + arguments, + terminal, + working, + term, + }) + .map_err(|e| Error(e.to_string()))?; + if request.len() >= LIMIT { + return Err(Error("Program request is too long".into())); + } + request.push(b'\n'); + socket.write_all(&request)?; + let mut ready = [0u8]; + socket.read_exact(&mut ready)?; + if ready != [b'R'] { + let mut bytes = ready.to_vec(); + (&mut socket).take(LIMIT as u64).read_to_end(&mut bytes)?; + let reply: Response = serde_json::from_slice(&bytes) + .map_err(|_| Error("Invalid program handshake".into()))?; + return Err(Error( + reply + .error + .unwrap_or_else(|| "Program launch failed".into()), + )); + } + send_fds(&socket, descriptors)?; + let mut received = receive_fds(&mut socket, 1)?; + socket.set_read_timeout(None)?; + socket.set_nonblocking(true)?; + Ok((socket, received.pop().unwrap())) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::File; + #[test] + fn descriptor_transfer_is_cloexec_and_owned() { + let (sender, mut receiver) = UnixStream::pair().unwrap(); + let original = File::open("/dev/null").unwrap(); + send_fds(&sender, &[original.as_raw_fd()]).unwrap(); + let received = receive_fds(&mut receiver, 1).unwrap().pop().unwrap(); + assert_ne!(received.as_raw_fd(), original.as_raw_fd()); + assert_ne!( + unsafe { libc::fcntl(received.as_raw_fd(), libc::F_GETFD) } & libc::FD_CLOEXEC, + 0 + ); + drop(original); + assert!(File::from(received).metadata().is_ok()); + } +} diff --git a/rust/fds-common/src/lib.rs b/rust/fds-common/src/lib.rs index f464189..b7a8ab0 100644 --- a/rust/fds-common/src/lib.rs +++ b/rust/fds-common/src/lib.rs @@ -1,6 +1,7 @@ //! Shared data contracts. Cartridge contents are data, never startup commands. pub mod boot; pub mod control; +pub mod launch; pub mod machine; pub mod manifest; pub mod software; diff --git a/rust/fds-common/src/software.rs b/rust/fds-common/src/software.rs index 703d529..60126b4 100644 --- a/rust/fds-common/src/software.rs +++ b/rust/fds-common/src/software.rs @@ -1,4 +1,4 @@ -//! Software metadata is descriptive. Bundles never contain privileged build hooks. +//! Software metadata describes immutable installed trees or legacy archives. use crate::{Error, Result, manifest::identifier}; use serde::{Deserialize, Serialize}; @@ -21,6 +21,13 @@ pub struct Software { pub architecture: String, /// GPT partition number, starting at 2 after FDS_METADATA. pub partition: u8, + /// New media executes installed Void package files directly from EROFS. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub installed: bool, + /// Exact XBPS package versions included in this software tree. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub packages: Vec, + #[serde(default, skip_serializing_if = "is_zero")] pub archive_bytes: u64, pub unpacked_bytes: u64, pub entries: u32, @@ -29,6 +36,9 @@ pub struct Software { pub commands: BTreeMap, } impl Software { + pub fn root_path(&self) -> String { + format!("programs/{}", self.id) + } pub fn archive_path(&self) -> String { format!("bundles/{}.tar.xz", self.id) } @@ -41,7 +51,14 @@ impl Software { || !display(&self.version, 32) || !matches!(self.architecture.as_str(), "aarch64" | "any") || !(2..=33).contains(&self.partition) - || !(1..=MAX_ARCHIVE).contains(&self.archive_bytes) + || if self.installed { + self.archive_bytes != 0 + || self.packages.is_empty() + || self.packages.len() > 512 + || self.packages.iter().any(|p| !xbps_identifier(p)) + } else { + !(1..=MAX_ARCHIVE).contains(&self.archive_bytes) || !self.packages.is_empty() + } || self.unpacked_bytes > MAX_UNPACKED || !(1..=MAX_ENTRIES).contains(&self.entries) || self.sha256.len() != 64 @@ -81,15 +98,20 @@ impl Catalogue { Ok(value) } pub fn validate(&self) -> Result<()> { - if self.format != 1 || self.software.is_empty() || self.software.len() > 128 { + if !matches!(self.format, 1 | 2) || self.software.is_empty() || self.software.len() > 128 { return Err(Error( - "Software catalogue requires format 1 and 1..128 software entries".into(), + "Software catalogue requires format 1 or 2 and 1..128 software entries".into(), )); } let mut ids = BTreeSet::new(); let mut partitions = BTreeSet::new(); for software in &self.software { software.validate()?; + if software.installed != (self.format == 2) { + return Err(Error( + "Catalogue format disagrees with software storage layout".into(), + )); + } if !ids.insert(&software.id) { return Err(Error("Duplicate software id".into())); } @@ -118,6 +140,18 @@ impl Catalogue { Ok(result) } } +fn is_zero(value: &u64) -> bool { + *value == 0 +} +pub fn xbps_identifier(value: &str) -> bool { + !value.is_empty() + && value.len() <= 256 + && value.as_bytes()[0].is_ascii_alphanumeric() + && value + .bytes() + .all(|c| c.is_ascii_alphanumeric() || b"._-+~".contains(&c)) + && !value.contains("..") +} pub fn relative(value: &str) -> bool { !value.is_empty() && value.len() <= 1024 @@ -141,6 +175,8 @@ mod tests { version: "1".into(), architecture: "aarch64".into(), partition, + installed: false, + packages: Vec::new(), archive_bytes: 100, unpacked_bytes: 200, entries: 1, @@ -181,4 +217,24 @@ mod tests { } assert!(relative("share/document with spaces.txt")); } + #[test] + fn installed_trees_use_format_two_and_record_xbps_versions() { + let mut entry = software("one", 2); + entry.installed = true; + entry.archive_bytes = 0; + entry.packages = vec!["WindowMaker-0.96.0_1".into(), "libstdc++-14.2.1_1".into()]; + let mut catalogue = Catalogue { + format: 2, + software: vec![entry], + }; + assert_eq!( + Catalogue::parse(&catalogue.to_toml().unwrap()).unwrap(), + catalogue + ); + catalogue.format = 1; + assert!(catalogue.validate().is_err()); + catalogue.format = 2; + catalogue.software[0].packages.clear(); + assert!(catalogue.validate().is_err()); + } } diff --git a/rust/fds-control/Cargo.toml b/rust/fds-control/Cargo.toml new file mode 100644 index 0000000..ccc9121 --- /dev/null +++ b/rust/fds-control/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "fds-control" +version = "0.1.0" +edition = "2024" +license = "MIT" +description = "Native grayscale X11 cartridge control panel" + +[dependencies] +clap.workspace = true +fds-common = { path = "../fds-common" } +libc = "0.2" +serde_json = "1" +x11rb = "=0.13.2" diff --git a/rust/fds-control/src/main.rs b/rust/fds-control/src/main.rs new file mode 100644 index 0000000..28a4111 --- /dev/null +++ b/rust/fds-control/src/main.rs @@ -0,0 +1,859 @@ +//! Core X11 drawing keeps the control panel small, static and free of animation. +use clap::Parser; +use fds_common::{ + Bay, + control::{self, BayState, Request, Response}, + manifest::Class, +}; +use std::{ + error::Error, + io::{Read, Write}, + os::{fd::AsRawFd, unix::net::UnixStream}, + process::{Child, Command}, + sync::mpsc, + time::{Duration, Instant}, +}; +use x11rb::{ + COPY_DEPTH_FROM_PARENT, + connection::Connection, + protocol::{Event, xproto::*}, + rust_connection::RustConnection, + wrapper::ConnectionExt as _, +}; +type Result = std::result::Result>; + +#[derive(Parser)] +#[command( + version, + about = "FDS cartridge control panel for X11", + after_help = "Select a bay to inspect its cartridge. Run opens a terminal; Eject stops its programs and releases the cartridge. Remove media only after SAFE appears." +)] +struct Options { + /// X display; defaults to DISPLAY. + #[arg(long)] + display: Option, +} +#[derive(Clone, Copy, PartialEq)] +enum Focus { + Bays, + Programs, + Run, + Eject, + Rescan, +} +impl Focus { + fn next(self) -> Self { + match self { + Self::Bays => Self::Programs, + Self::Programs => Self::Run, + Self::Run => Self::Eject, + Self::Eject => Self::Rescan, + Self::Rescan => Self::Bays, + } + } +} +#[derive(Clone)] +struct Program { + label: String, + alias: String, +} +struct Model { + bays: Vec, + bay: usize, + program: usize, + focus: Focus, + status: String, + busy: bool, + available: bool, + display: String, +} +impl Model { + fn new() -> Self { + Self { + bays: Vec::new(), + bay: 0, + program: 0, + focus: Focus::Bays, + status: "Connecting to the cartridge service...".into(), + busy: false, + available: false, + display: std::env::var("DISPLAY").unwrap_or_else(|_| ":0".into()), + } + } + fn selected(&self) -> Option<&BayState> { + self.bays + .iter() + .find(|b| u8::from(b.bay) as usize == self.bay + 1) + } + fn programs(&self) -> Vec { + let Some(bay) = self.selected() else { + return Vec::new(); + }; + if bay.state != "mounted_read_only" { + return Vec::new(); + } + bay.commands + .iter() + .map(|command| Program { + label: command.selector.clone(), + alias: command.alias.clone(), + }) + .collect() + } + fn can_eject(&self) -> bool { + self.available + && !self.busy + && self + .selected() + .is_some_and(|b| b.mount.is_some() && b.state != "protected") + } + fn can_use_data(&self) -> bool { + self.available + && !self.busy + && self.selected().is_some_and(|b| { + b.state == "mounted_read_only" + && b.manifest + .as_ref() + .is_some_and(|m| m.cartridge.class == Class::Data) + }) + } + fn can_run(&self) -> bool { + self.available && !self.busy && self.program < self.programs().len() + } + fn select(&mut self, bay: usize) { + if self.bay != bay { + self.bay = bay; + self.program = 0; + } + } +} + +// All labels are bounded before core X11 text requests. Cartridge names never +// become shell source; launches use an argument vector and the validated alias. +fn text_bytes(text: &str, max: usize) -> Vec { + let mut bytes: Vec<_> = text + .chars() + .map(|c| { + if c.is_ascii() && !c.is_control() { + c as u8 + } else { + b'?' + } + }) + .take(max + 1) + .collect(); + if bytes.len() > max { + bytes.truncate(max); + if max >= 3 { + bytes[max - 3..].copy_from_slice(b"..."); + } + } + bytes +} +fn state_label(state: &str) -> String { + state.replace('_', " ").to_uppercase() +} + +struct View { + connection: RustConnection, + window: Window, + gc: Gcontext, + black: u32, + white: u32, + gray: u32, + delete: Atom, + protocols: Atom, + state_atom: Atom, +} +impl View { + fn new(display: Option<&str>) -> Result { + let (connection, index) = x11rb::connect(display)?; + let screen = &connection.setup().roots[index]; + let (black, white) = (screen.black_pixel, screen.white_pixel); + let gray = connection + .alloc_color(screen.default_colormap, 0xcccc, 0xcccc, 0xcccc)? + .reply()? + .pixel; + let window = connection.generate_id()?; + connection + .create_window( + COPY_DEPTH_FROM_PARENT, + window, + screen.root, + 30, + 30, + 900, + 630, + 2, + WindowClass::INPUT_OUTPUT, + 0, + &CreateWindowAux::new() + .background_pixel(white) + .border_pixel(black) + .event_mask( + EventMask::EXPOSURE + | EventMask::BUTTON_PRESS + | EventMask::KEY_PRESS + | EventMask::STRUCTURE_NOTIFY, + ), + )? + .check()?; + connection.change_property8( + PropMode::REPLACE, + window, + AtomEnum::WM_NAME, + AtomEnum::STRING, + b"FDS Control", + )?; + connection.change_property8( + PropMode::REPLACE, + window, + AtomEnum::WM_CLASS, + AtomEnum::STRING, + b"fds-control\0FdsControl\0", + )?; + let protocols = connection + .intern_atom(false, b"WM_PROTOCOLS")? + .reply()? + .atom; + let delete = connection + .intern_atom(false, b"WM_DELETE_WINDOW")? + .reply()? + .atom; + connection.change_property32( + PropMode::REPLACE, + window, + protocols, + AtomEnum::ATOM, + &[delete], + )?; + let state_atom = connection + .intern_atom(false, b"_FDS_CONTROL_STATE")? + .reply()? + .atom; + // Fixed dimensions keep the bitmap-font layout readable and predictable. + let mut hints = x11rb::properties::WmSizeHints::new(); + hints.min_size = Some((900, 630)); + hints.max_size = Some((900, 630)); + hints.set_normal_hints(&connection, window)?; + let font = connection.generate_id()?; + if connection + .open_font( + font, + b"-*-terminus-medium-r-normal--16-*-*-*-*-*-iso10646-1", + )? + .check() + .is_err() + { + connection.open_font(font, b"fixed")?.check()?; + } + let gc = connection.generate_id()?; + connection.create_gc( + gc, + window, + &CreateGCAux::new() + .foreground(black) + .background(white) + .font(font) + .graphics_exposures(0), + )?; + connection.map_window(window)?; + connection.flush()?; + Ok(Self { + connection, + window, + gc, + black, + white, + gray, + delete, + protocols, + state_atom, + }) + } + fn fill(&self, x: i16, y: i16, width: u16, height: u16, color: u32) -> Result<()> { + self.connection + .change_gc(self.gc, &ChangeGCAux::new().foreground(color))?; + self.connection.poly_fill_rectangle( + self.window, + self.gc, + &[Rectangle { + x, + y, + width, + height, + }], + )?; + Ok(()) + } + fn border(&self, x: i16, y: i16, width: u16, height: u16) -> Result<()> { + self.connection + .change_gc(self.gc, &ChangeGCAux::new().foreground(self.black))?; + self.connection.poly_rectangle( + self.window, + self.gc, + &[Rectangle { + x, + y, + width, + height, + }], + )?; + Ok(()) + } + fn text(&self, x: i16, y: i16, text: &str, max: usize, inverted: bool) -> Result<()> { + self.connection.change_gc( + self.gc, + &ChangeGCAux::new().foreground(if inverted { self.white } else { self.black }), + )?; + let bytes = text_bytes(text, max.min(254)); + // poly_text draws only glyphs, preserving the selection/background fill. + let mut data = vec![bytes.len() as u8, 0]; + data.extend(bytes); + self.connection + .poly_text8(self.window, self.gc, x, y, &data)?; + Ok(()) + } + fn button(&self, x: i16, width: u16, label: &str, enabled: bool, focused: bool) -> Result<()> { + self.fill( + x, + 498, + width, + 38, + if enabled && focused { + self.black + } else if enabled { + self.white + } else { + self.gray + }, + )?; + self.border(x, 498, width, 38)?; + self.text( + x + 12, + 522, + label, + (width as usize - 24) / 8, + enabled && focused, + )?; + Ok(()) + } + fn draw(&self, model: &Model) -> Result<()> { + // Expose the same visible state to X11 inspection/accessibility tools. + let state = serde_json::json!({"bay": model.bay + 1, "state": model.selected().map(|b| &b.state), "busy": model.busy, "available": model.available, "commands": model.programs().len(), "status": model.status}); + self.connection.change_property8( + PropMode::REPLACE, + self.window, + self.state_atom, + AtomEnum::STRING, + serde_json::to_string(&state)?.as_bytes(), + )?; + self.fill(0, 0, 900, 630, self.white)?; + self.fill(0, 0, 900, 72, self.black)?; + self.text(22, 29, "FDS / CONTROL", 50, true)?; + self.text(22, 53, "Cartridges and programs", 70, true)?; + self.text(710, 42, "TWELVE BAYS", 22, true)?; + self.text(22, 98, "BAY CARTRIDGE / STATE", 37, false)?; + for index in 0..12 { + let y = 110 + index as i16 * 31; + let selected = model.bay == index; + self.fill( + 20, + y, + 300, + 31, + if selected { self.black } else { self.white }, + )?; + self.border(20, y, 300, 31)?; + let bay = model + .bays + .iter() + .find(|b| u8::from(b.bay) as usize == index + 1); + let label = match bay { + Some(b) => { + if b.state == "empty" { + "Empty".into() + } else if b.state == "safe" { + "SAFE - remove cartridge".into() + } else { + b.name.clone().unwrap_or_else(|| state_label(&b.state)) + } + } + None => "Unavailable".into(), + }; + self.text( + 30, + y + 21, + &format!("{:02} {label}", index + 1), + 35, + selected, + )?; + } + if model.focus == Focus::Bays { + self.border(17, 107, 306, 378)?; + } + self.text(346, 98, &format!("BAY {:02}", model.bay + 1), 60, false)?; + if let Some(bay) = model.selected() { + self.text( + 346, + 130, + bay.name.as_deref().unwrap_or("No cartridge"), + 66, + false, + )?; + self.text(346, 156, &state_label(&bay.state), 66, false)?; + if let Some(manifest) = &bay.manifest { + self.text( + 346, + 184, + &format!("Type: {:?}", manifest.cartridge.class), + 66, + false, + )?; + self.text( + 346, + 207, + &format!("ID: {}", manifest.cartridge.id), + 66, + false, + )?; + } + self.text( + 346, + 235, + &format!("Running processes: {}", bay.consumers), + 66, + false, + )?; + if let Some(detail) = &bay.detail { + self.text(346, 260, detail, 66, false)?; + } + } else { + self.text(346, 132, "Waiting for the cartridge service.", 66, false)?; + } + self.text(346, 292, "PROGRAMS", 66, false)?; + let programs = model.programs(); + let first = model.program.saturating_sub(4); + for row in 0..5 { + let index = first + row; + let y = 304 + row as i16 * 31; + let selected = index == model.program && index < programs.len(); + self.fill( + 346, + y, + 532, + 31, + if selected { self.black } else { self.white }, + )?; + self.border(346, y, 532, 31)?; + if let Some(program) = programs.get(index) { + self.text(356, y + 21, &program.label, 64, selected)?; + } else if row == 0 { + self.text( + 356, + y + 21, + "No software commands on this cartridge", + 64, + false, + )?; + } + } + if model.focus == Focus::Programs { + self.border(343, 301, 538, 161)?; + } + if !programs.is_empty() { + self.text( + 346, + 482, + &format!("Command {} of {}", model.program + 1, programs.len()), + 66, + false, + )?; + } + self.button( + 346, + 192, + if model.can_use_data() { + "Use DATA" + } else { + "Run in terminal" + }, + model.can_run() || model.can_use_data(), + model.focus == Focus::Run, + )?; + self.button( + 554, + 148, + "Safe eject", + model.can_eject(), + model.focus == Focus::Eject, + )?; + self.button( + 718, + 160, + "Rescan", + !model.busy, + model.focus == Focus::Rescan, + )?; + self.text(22, 517, "Remove media only after SAFE.", 37, false)?; + self.fill(20, 552, 858, 44, self.gray)?; + self.text(30, 572, &model.status, 104, false)?; + self.text( + 30, + 589, + &model.status.chars().skip(104).collect::(), + 104, + false, + )?; + self.text( + 22, + 619, + "Arrows: select Tab: focus Enter: activate R: rescan Esc: close", + 106, + false, + )?; + self.connection.flush()?; + Ok(()) + } + fn keysym(&self, keycode: u8) -> Result { + let mapping = self.connection.get_keyboard_mapping(keycode, 1)?.reply()?; + Ok(mapping.keysyms.first().copied().unwrap_or(0)) + } +} + +struct Update { + result: fds_common::Result, + action: Option, +} +struct Worker { + request: mpsc::SyncSender<(Request, Option)>, + response: mpsc::Receiver, + wake: UnixStream, + in_flight: bool, + queued: Option<(Request, Option)>, +} +impl Worker { + fn new() -> Result { + let (tx, rx) = mpsc::sync_channel::<(Request, Option)>(1); + let (updates, response) = mpsc::channel(); + let (wake, mut writer) = UnixStream::pair()?; + wake.set_nonblocking(true)?; + std::thread::spawn(move || { + while let Ok((request, action)) = rx.recv() { + let result = control::request(&request).and_then(|reply| { + let inventory = if matches!(request, Request::Bays) { + reply + } else { + control::request(&Request::Bays)? + }; + if let Request::Eject { bay } = &request { + if !inventory.bays.iter().any(|b| b.bay == *bay && matches!(b.state.as_str(), "safe" | "empty")) { + return Err(fds_common::Error("Bay changed after eject; inspect its current state before removing media".into())); + } + } + Ok(inventory) + }); + if updates.send(Update { result, action }).is_err() + || writer.write_all(b"R").is_err() + { + break; + } + } + }); + Ok(Self { + request: tx, + response, + wake, + in_flight: false, + queued: None, + }) + } + fn send(&mut self, model: &mut Model, request: Request, action: Option) -> Result<()> { + if self.in_flight { + // One user operation may queue behind the background status read. + // Polling never disables controls or discards a click. + if action.is_some() && self.queued.is_none() { + self.queued = Some((request, action)); + model.busy = true; + } + } else { + model.busy = action.is_some(); + self.request.send((request, action))?; + self.in_flight = true; + } + Ok(()) + } +} +fn activate( + focus: Focus, + model: &mut Model, + worker: &mut Worker, + children: &mut Vec, +) -> Result<()> { + match focus { + Focus::Run if model.can_use_data() => { + model.status = "Activating DATA...".into(); + worker.send( + model, + Request::DataUse { + bay: Bay::try_from((model.bay + 1) as u8)?, + }, + Some("DATA is active at /data.".into()), + )?; + } + Focus::Programs | Focus::Run if model.can_run() => { + let program = &model.programs()[model.program]; + match Command::new("/usr/bin/xterm") + .env("DISPLAY", &model.display) + .args([ + "-hold", + "-T", + &program.label, + "-fa", + "Terminus", + "-fs", + "16", + "-bg", + "white", + "-fg", + "black", + "-e", + "/usr/bin/fds-program", + &program.alias, + ]) + .spawn() + { + Ok(child) => { + children.push(child); + model.status = format!( + "Opened {}. Close its terminal when finished.", + program.label + ); + } + Err(error) => model.status = format!("Cannot open terminal: {error}"), + } + } + Focus::Eject if model.can_eject() => { + model.status = format!( + "Releasing bay {:02}; waiting for programs and storage...", + model.bay + 1 + ); + worker.send( + model, + Request::Eject { + bay: Bay::try_from((model.bay + 1) as u8)?, + }, + Some(format!( + "Bay {:02} is SAFE. You may remove the cartridge.", + model.bay + 1 + )), + )?; + } + Focus::Rescan if !model.busy => { + model.status = "Scanning cartridge bays...".into(); + worker.send( + model, + Request::Rescan, + Some("Cartridge inventory refreshed.".into()), + )?; + } + _ => (), + } + Ok(()) +} +fn run() -> Result<()> { + let options = Options::parse(); + let view = View::new(options.display.as_deref())?; + let mut model = Model::new(); + if let Some(display) = options.display { + model.display = display; + } + let mut worker = Worker::new()?; + let mut children: Vec = Vec::new(); + worker.send( + &mut model, + Request::Bays, + Some("Select a bay to inspect its cartridge.".into()), + )?; + let mut refresh = Instant::now() + Duration::from_secs(2); + let mut previous = String::new(); + let mut redraw = true; + let mut painted_busy = false; + loop { + while let Some(event) = view.connection.poll_for_event()? { + match event { + Event::Expose(_) => redraw = true, + Event::ClientMessage(e) + if e.type_ == view.protocols && e.data.as_data32()[0] == view.delete => + { + return Ok(()); + } + Event::DestroyNotify(_) => return Ok(()), + Event::Error(e) => return Err(format!("X11 protocol error: {e:?}").into()), + Event::ButtonPress(e) if e.detail == 1 => { + let (x, y) = (e.event_x, e.event_y); + if (20..320).contains(&x) && (110..482).contains(&y) { + model.select(((y - 110) / 31) as usize); + model.focus = Focus::Bays; + } else if (346..878).contains(&x) && (304..459).contains(&y) { + let index = model.program.saturating_sub(4) + ((y - 304) / 31) as usize; + if index < model.programs().len() { + model.program = index; + model.focus = Focus::Programs; + } + } else if (498..536).contains(&y) { + let focus = if (346..538).contains(&x) { + Some(Focus::Run) + } else if (554..702).contains(&x) { + Some(Focus::Eject) + } else if (718..878).contains(&x) { + Some(Focus::Rescan) + } else { + None + }; + if let Some(focus) = focus { + model.focus = focus; + activate(focus, &mut model, &mut worker, &mut children)?; + } + } + redraw = true; + } + Event::KeyPress(e) => { + match view.keysym(e.detail)? { + 0xff1b => return Ok(()), + 0xff09 => model.focus = model.focus.next(), + 0xff52 | 0xff54 => { + let down = view.keysym(e.detail)? == 0xff54; + if model.focus == Focus::Programs { + let length = model.programs().len(); + if length > 0 { + model.program = if down { + (model.program + 1).min(length - 1) + } else { + model.program.saturating_sub(1) + }; + } + } else { + model.select(if down { + (model.bay + 1).min(11) + } else { + model.bay.saturating_sub(1) + }); + } + } + 0xff0d => activate(model.focus, &mut model, &mut worker, &mut children)?, + 0x72 | 0x52 => { + activate(Focus::Rescan, &mut model, &mut worker, &mut children)? + } + _ => (), + } + redraw = true; + } + _ => (), + } + } + while let Ok(update) = worker.response.try_recv() { + worker.in_flight = false; + if update.action.is_some() { + model.busy = false; + } + redraw |= painted_busy; + match update.result { + Ok(reply) => { + let serialized = serde_json::to_string(&reply.bays)?; + let changed = serialized != previous; + let reconnected = !model.available; + model.available = true; + model.bays = reply.bays; + if model.program >= model.programs().len() { + model.program = 0; + } + if let Some(action) = update.action { + model.status = action; + redraw = true; + } else if changed && !reconnected && !model.busy { + model.status = + "Cartridge inventory updated. Check the selected bay's current state." + .into(); + redraw = true; + } else if reconnected { + model.status = "Cartridge service connected.".into(); + redraw = true; + } + previous = serialized; + redraw |= changed; + } + Err(error) => { + let status = format!("Operation failed: {error}"); + redraw |= status != model.status; + model.status = status; + model.available = false; + } + } + if let Some((request, action)) = worker.queued.take() { + worker.send(&mut model, request, action)?; + } + refresh = Instant::now() + Duration::from_secs(2); + } + children.retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_)))); + if redraw { + view.draw(&model)?; + painted_busy = model.busy; + redraw = false; + } + if Instant::now() >= refresh { + worker.send(&mut model, Request::Bays, None)?; + refresh = Instant::now() + Duration::from_secs(2); + } + let mut fds = [ + libc::pollfd { + fd: view.connection.stream().as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }, + libc::pollfd { + fd: worker.wake.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }, + ]; + let remaining = refresh + .saturating_duration_since(Instant::now()) + .as_millis() + .min(2000) as i32; + if unsafe { libc::poll(fds.as_mut_ptr(), 2, remaining) } < 0 + && std::io::Error::last_os_error().kind() != std::io::ErrorKind::Interrupted + { + return Err(std::io::Error::last_os_error().into()); + } + if fds[1].revents != 0 { + let mut buffer = [0; 64]; + let _ = worker.wake.read(&mut buffer); + } + } +} +fn main() { + if let Err(error) = run() { + eprintln!("fds-control: {error}"); + std::process::exit(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn cartridge_labels_are_bounded_and_cannot_inject_x11_text_items() { + assert_eq!(text_bytes("abc\n\u{ff}def", 20), b"abc??def"); + assert_eq!(text_bytes("abcdefgh", 6), b"abc..."); + assert_eq!(text_bytes("", 10), b""); + } + #[test] + fn disconnected_controls_do_not_allow_operations() { + let model = Model::new(); + assert!(!model.can_eject()); + assert!(!model.can_run()); + use clap::CommandFactory; + Options::command().debug_assert(); + } +} diff --git a/rust/fds-software/Cargo.toml b/rust/fds-software/Cargo.toml index 522153d..0242acf 100644 --- a/rust/fds-software/Cargo.toml +++ b/rust/fds-software/Cargo.toml @@ -3,7 +3,7 @@ name = "fds-software" version = "0.1.0" edition = "2024" license = "MIT" -description = "Bounded software catalogues and verified xz tar bundles" +description = "Verified installed software trees and legacy archive reading" [dependencies] fds-common = { path = "../fds-common" } diff --git a/rust/fds-software/src/archive.rs b/rust/fds-software/src/archive.rs index 0cb601f..71d0067 100644 --- a/rust/fds-software/src/archive.rs +++ b/rust/fds-software/src/archive.rs @@ -388,6 +388,8 @@ mod tests { use super::*; fn metadata() -> Software { Software { + installed: false, + packages: Vec::new(), id: "test.tool".into(), name: "Tool".into(), version: "1".into(), diff --git a/rust/fds-software/src/lib.rs b/rust/fds-software/src/lib.rs index ceae48b..a065567 100644 --- a/rust/fds-software/src/lib.rs +++ b/rust/fds-software/src/lib.rs @@ -1,3 +1,4 @@ -//! Verified software archives shared by workstation and guest tools. +//! Installed software trees and legacy archive verification for host and guest. pub use fds_common::software::*; pub mod archive; +pub mod tree; diff --git a/rust/fds-software/src/tree.rs b/rust/fds-software/src/tree.rs new file mode 100644 index 0000000..49dc01e --- /dev/null +++ b/rust/fds-software/src/tree.rs @@ -0,0 +1,361 @@ +//! Deterministic integrity checks for programs executed directly from EROFS. +use crate::{MAX_ENTRIES, MAX_UNPACKED, Software, relative}; +use fds_common::{Error, Result}; +use sha2::{Digest, Sha256}; +use std::{ + fs::{self, File, OpenOptions}, + io::Read, + os::unix::fs::{OpenOptionsExt, PermissionsExt, symlink}, + path::{Component, Path, PathBuf}, +}; + +#[derive(Debug, PartialEq, Eq)] +pub struct Inventory { + pub bytes: u64, + pub entries: u32, + pub sha256: String, +} + +fn paths(root: &Path, directory: &Path, output: &mut Vec) -> Result<()> { + for entry in fs::read_dir(directory)? { + let entry = entry?; + let path = entry.path(); + let name = path.strip_prefix(root).unwrap(); + if !name.to_str().is_some_and(relative) || output.len() >= MAX_ENTRIES as usize { + return Err(Error( + "Invalid software path or too many installed files".into(), + )); + } + output.push(name.to_owned()); + if entry.file_type()?.is_dir() { + paths(root, &path, output)?; + } + } + Ok(()) +} + +fn link_inside(root: &Path, path: &Path, target: &Path) -> Result<()> { + if target.is_absolute() || target.as_os_str().is_empty() { + return Err(Error( + "Installed software links must be relative to their tree".into(), + )); + } + let mut depth = path + .parent() + .unwrap() + .strip_prefix(root) + .unwrap() + .components() + .count(); + for part in target.components() { + match part { + Component::Normal(_) => depth += 1, + Component::CurDir => (), + Component::ParentDir if depth > 0 => depth -= 1, + _ => return Err(Error("Installed software symlink escapes its tree".into())), + } + } + match path.canonicalize() { + Ok(destination) if !destination.starts_with(root) => { + return Err(Error( + "Installed software symlink resolves outside its tree".into(), + )); + } + Ok(_) => (), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => (), + Err(e) => return Err(e.into()), + } + Ok(()) +} + +pub fn inspect(root: &Path, architecture: &str) -> Result { + if !fs::symlink_metadata(root)?.is_dir() { + return Err(Error( + "Installed software root must be a real directory".into(), + )); + } + let root = root.canonicalize()?; + let mut entries = Vec::new(); + paths(&root, &root, &mut entries)?; + entries.sort(); + let mut hash = Sha256::new(); + let mut bytes = 0u64; + for relative in &entries { + let path = root.join(relative); + let metadata = path.symlink_metadata()?; + let name = relative.as_os_str().as_encoded_bytes(); + hash.update((name.len() as u64).to_le_bytes()); + hash.update(name); + let mode = metadata.permissions().mode(); + if !metadata.is_symlink() && mode & 0o7022 != 0 { + return Err(Error(format!( + "Privileged or group/world-writable software file: {}", + relative.display() + ))); + } + hash.update((mode & 0o777).to_le_bytes()); + if metadata.is_dir() { + hash.update(b"d"); + } else if metadata.is_symlink() { + hash.update(b"l"); + let target = fs::read_link(&path)?; + link_inside(&root, &path, &target)?; + let value = target.as_os_str().as_encoded_bytes(); + hash.update((value.len() as u64).to_le_bytes()); + hash.update(value); + } else if metadata.is_file() { + hash.update(b"f"); + bytes = bytes + .checked_add(metadata.len()) + .ok_or_else(|| Error("Software size overflow".into()))?; + if bytes > MAX_UNPACKED { + return Err(Error("Installed software tree exceeds 1 GiB".into())); + } + hash.update(metadata.len().to_le_bytes()); + let mut file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(&path)?; + let mut prefix = [0u8; 20]; + let n = file.read(&mut prefix)?; + if prefix.starts_with(b"\x7fELF") + && (architecture != "aarch64" + || n < 20 + || prefix[4] != 2 + || prefix[5] != 1 + || prefix[18..20] != [183, 0]) + { + return Err(Error( + "Installed software contains an ELF file for the wrong architecture".into(), + )); + } + hash.update(&prefix[..n]); + let copied = std::io::copy(&mut file, &mut HashWriter(&mut hash))?; + if copied + n as u64 != metadata.len() { + return Err(Error("Installed software changed during inspection".into())); + } + } else { + return Err(Error( + "Installed software permits only files, directories and internal symlinks".into(), + )); + } + } + Ok(Inventory { + bytes, + entries: entries.len() as u32, + sha256: format!("{:x}", hash.finalize()), + }) +} + +struct HashWriter<'a>(&'a mut Sha256); +impl std::io::Write for HashWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.update(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +pub fn verify(root: &Path, software: &Software) -> Result<()> { + software.validate()?; + if !software.installed { + return Err(Error("Expected an installed software tree".into())); + } + let actual = inspect(root, &software.architecture)?; + if actual.sha256 != software.sha256 + || actual.bytes != software.unpacked_bytes + || actual.entries != software.entries + { + return Err(Error( + "Installed software tree digest, size or entry count disagrees with catalogue".into(), + )); + } + let canonical = root.canonicalize()?; + for command in software.commands.values() { + let executable = root.join(command).canonicalize()?; + if !executable.starts_with(&canonical) + || !executable.is_file() + || executable.metadata()?.permissions().mode() & 0o111 == 0 + { + return Err(Error( + "Software command must resolve to an executable within its installed tree".into(), + )); + } + } + Ok(()) +} + +/// XBPS trees contain root-relative symlinks. Relocate those on the workstation +/// so the exact tree can be mounted under /run without pointing into SYSTEM. +pub fn relocate(root: &Path) -> Result<()> { + let mut entries = Vec::new(); + paths(root, root, &mut entries)?; + for name in entries { + let path = root.join(&name); + if path.is_symlink() { + let target = fs::read_link(&path)?; + if target.is_absolute() { + let mut relative = PathBuf::new(); + for _ in name.parent().unwrap().components() { + relative.push(".."); + } + relative.push(target.strip_prefix("/").unwrap()); + fs::remove_file(&path)?; + symlink(relative, &path)?; + } + } + } + Ok(()) +} + +pub fn copy(root: &Path, destination: &Path) -> Result<()> { + fs::create_dir(destination)?; + fs::set_permissions(destination, fs::Permissions::from_mode(0o755))?; + let mut entries = Vec::new(); + paths(root, root, &mut entries)?; + entries.sort(); + for name in entries { + let from = root.join(&name); + let to = destination.join(name); + let metadata = from.symlink_metadata()?; + if metadata.is_symlink() { + symlink(fs::read_link(from)?, to)?; + } else if metadata.is_dir() { + fs::create_dir(&to)?; + fs::set_permissions(to, metadata.permissions())?; + } else if metadata.is_file() { + fs::copy(from, to)?; + } else { + return Err(Error("Unsupported installed software file type".into())); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + struct Fixture(PathBuf); + impl Fixture { + fn new() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + let root = std::env::temp_dir().join(format!( + "fds-tree-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&root).unwrap(); + fs::create_dir_all(root.join("usr/bin")).unwrap(); + fs::write(root.join("usr/bin/tool"), b"#!/bin/sh\necho installed\n").unwrap(); + fs::set_permissions(root.join("usr/bin/tool"), fs::Permissions::from_mode(0o755)) + .unwrap(); + symlink("usr/bin", root.join("bin")).unwrap(); + Self(root) + } + fn software(&self) -> Software { + let found = inspect(&self.0, "aarch64").unwrap(); + Software { + id: "demo.tool".into(), + name: "Tool".into(), + version: "1".into(), + architecture: "aarch64".into(), + partition: 2, + installed: true, + packages: vec!["WindowMaker-0.96.0_1".into()], + archive_bytes: 0, + unpacked_bytes: found.bytes, + entries: found.entries, + sha256: found.sha256, + commands: [("tool".into(), "bin/tool".into())].into(), + } + } + } + impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + #[test] + fn installed_tree_roundtrip_and_tampering() { + let root = Fixture::new(); + let software = root.software(); + verify(&root.0, &software).unwrap(); + let copy_path = root.0.with_extension("copy"); + copy(&root.0, ©_path).unwrap(); + verify(©_path, &software).unwrap(); + fs::remove_dir_all(copy_path).unwrap(); + fs::write(root.0.join("usr/bin/tool"), b"changed").unwrap(); + assert!(verify(&root.0, &software).is_err()); + } + #[test] + fn escaping_links_and_privileged_files_are_rejected() { + let root = Fixture::new(); + symlink("../../../etc/passwd", root.0.join("usr/bin/escape")).unwrap(); + assert!(inspect(&root.0, "aarch64").is_err()); + fs::remove_file(root.0.join("usr/bin/escape")).unwrap(); + fs::set_permissions( + root.0.join("usr/bin/tool"), + fs::Permissions::from_mode(0o4755), + ) + .unwrap(); + assert!(inspect(&root.0, "aarch64").is_err()); + } + #[test] + fn root_relative_xbps_links_are_relocated_and_wrong_elf_is_rejected() { + let root = Fixture::new(); + symlink("/usr/bin/tool", root.0.join("usr/bin/alias")).unwrap(); + assert!(inspect(&root.0, "aarch64").is_err()); + relocate(&root.0).unwrap(); + verify(&root.0, &root.software()).unwrap(); + fs::write( + root.0.join("usr/bin/tool"), + b"\x7fELF\x02\x01\0\0\0\0\0\0\0\0\0\0\0\0\x3e\0", + ) + .unwrap(); + assert!(inspect(&root.0, "aarch64").is_err()); + } +} + +/// Use the package's own glibc loader when present. Static ELFs and scripts +/// execute normally; dynamically linked binaries retain their packaged ABI. +pub fn executable(root: &Path, relative: &str) -> Result> { + let path = root.join(relative); + let mut file = File::open(&path)?; + let mut header = [0u8; 64]; + let n = file.read(&mut header)?; + if n == header.len() && header.starts_with(b"\x7fELF") && header[4] == 2 && header[5] == 1 { + use std::io::{Seek, SeekFrom}; + let offset = u64::from_le_bytes(header[32..40].try_into().unwrap()); + let size = u16::from_le_bytes(header[54..56].try_into().unwrap()) as u64; + let count = u16::from_le_bytes(header[56..58].try_into().unwrap()) as u64; + if size >= 56 && count <= 1024 { + for index in 0..count { + file.seek(SeekFrom::Start( + offset + .checked_add(index * size) + .ok_or_else(|| Error("ELF header overflow".into()))?, + ))?; + let mut program = [0u8; 56]; + file.read_exact(&mut program)?; + if u32::from_le_bytes(program[..4].try_into().unwrap()) == 3 { + for name in ["lib/ld-linux-aarch64.so.1", "usr/lib/ld-linux-aarch64.so.1"] { + let loader = root.join(name); + if loader.is_file() { + return Ok(vec![ + loader.display().to_string(), + "--library-path".into(), + format!("{0}/usr/lib:{0}/lib", root.display()), + path.display().to_string(), + ]); + } + } + break; + } + } + } + } + Ok(vec![path.display().to_string()]) +} diff --git a/rust/fds-workstation/src/cartridge.rs b/rust/fds-workstation/src/cartridge.rs index e4c5252..d77cffe 100644 --- a/rust/fds-workstation/src/cartridge.rs +++ b/rust/fds-workstation/src/cartridge.rs @@ -8,7 +8,7 @@ use fds_common::{ manifest::{Cartridge, Class, Manifest, Media}, read_text, }; -use fds_software::{Catalogue, archive}; +use fds_software::{Catalogue, archive, tree}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::{ @@ -22,7 +22,8 @@ use std::{ #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct Payload { - bundles: Vec, + /// Void software recipes or previously built installed-software directories. + sources: Vec, } #[derive(Deserialize)] #[serde(deny_unknown_fields)] @@ -55,8 +56,16 @@ fn uuid(bytes: &[u8]) -> [u8; 16] { id[8] = (id[8] & 63) | 0x80; id } -pub fn create(recipe: &Path, output: &Path, runner: Option<&Path>) -> Result { +pub fn create( + recipe: &Path, + output: &Path, + runner: Option<&Path>, + options: &crate::software::BuildOptions, +) -> Result { workstation()?; + if output.try_exists()? { + return Err(Error("Cartridge output already exists".into())); + } if unsafe { libc::geteuid() } == 0 { return Err(Error( "Create cartridge images as an ordinary workstation user".into(), @@ -64,12 +73,12 @@ pub fn create(recipe: &Path, output: &Path, runner: Option<&Path>) -> Result) -> Result) -> Result { let mut info = image::inspect(&file, file.metadata()?.len())?; if info.filesystem != "erofs" { return Err(Error( - "Use fds-burn inspect for DATA geometry; this inspector validates software bundles" + "Use fds-burn inspect for DATA geometry; this inspector validates software cartridges" .into(), )); } @@ -235,6 +254,13 @@ pub fn inspect(path: &Path, runner: Option<&Path>) -> Result { } for software in &catalogue.software { let root = &trees[usize::from(software.partition) - 1]; + if software.installed { + if !root.join("programs").symlink_metadata()?.is_dir() { + return Err(Error("Programs must be a real directory".into())); + } + tree::verify(&root.join(software.root_path()), software)?; + continue; + } let directory = root.join("bundles"); if !directory.symlink_metadata()?.is_dir() { return Err(Error("Bundle directory must not be a symlink".into())); @@ -247,23 +273,36 @@ pub fn inspect(path: &Path, runner: Option<&Path>) -> Result { } // Reject unlisted files or software hidden in a payload partition. for (index, root) in trees.iter().enumerate().skip(1) { + let directory = if catalogue.format == 2 { + "programs" + } else { + "bundles" + }; if fs::read_dir(root)?.count() != 1 { - return Err(Error("Payload partitions may contain only bundles/".into())); + return Err(Error(format!( + "Payload partitions may contain only {directory}/" + ))); } - let mut actual: Vec<_> = fs::read_dir(root.join("bundles"))? + let mut actual: Vec<_> = fs::read_dir(root.join(directory))? .map(|e| e.map(|e| e.file_name())) .collect::>()?; let mut expected: Vec<_> = catalogue .software .iter() .filter(|s| usize::from(s.partition) == index + 1) - .map(|s| std::ffi::OsString::from(format!("{}.tar.xz", s.id))) + .map(|s| { + std::ffi::OsString::from(if s.installed { + s.id.clone() + } else { + format!("{}.tar.xz", s.id) + }) + }) .collect(); actual.sort(); expected.sort(); if actual != expected { return Err(Error( - "Payload archive inventory disagrees with metadata".into(), + "Payload software inventory disagrees with metadata".into(), )); } } diff --git a/rust/fds-workstation/src/doctor.rs b/rust/fds-workstation/src/doctor.rs index 0c92ed2..0d01a0e 100644 --- a/rust/fds-workstation/src/doctor.rs +++ b/rust/fds-workstation/src/doctor.rs @@ -46,7 +46,7 @@ pub fn cartridge(runner: Option<&Path>) -> Result { ))); } Ok( - json!({"xz":xz,"mkfs.erofs":mkfs,"fsck.erofs":fsck,"unprivileged_sandbox":"available","cross_compiler":"recipe-specific; use aarch64 output or architecture=any scripts"}), + json!({"xz_legacy_reader":xz,"mkfs.erofs":mkfs,"fsck.erofs":fsck,"unprivileged_sandbox":"available","software_builder":"prepared Void xbps-src checkout with native XBPS tools; builds aarch64 source packages and installs runtime dependencies"}), ) } pub fn emulator(runner: Option<&Path>) -> Result { diff --git a/rust/fds-workstation/src/main.rs b/rust/fds-workstation/src/main.rs index d467db8..eb8adc5 100644 --- a/rust/fds-workstation/src/main.rs +++ b/rust/fds-workstation/src/main.rs @@ -5,18 +5,20 @@ use std::{path::PathBuf, process::ExitCode}; #[derive(Parser)] #[command( version, - about = "Build software bundles and metadata-first cartridge images on Linux" + about = "Build Void source packages and ready-to-run cartridge images on Linux" )] struct Cli { /// Optional wrapper that accepts TOOL followed by its arguments. #[arg(long, global = true)] image_tool_runner: Option, + #[command(flatten)] + build: fds_workstation::software::BuildOptions, #[command(subcommand)] command: Action, } #[derive(Subcommand)] enum Action { - /// Check xz, erofs-utils and the unprivileged filesystem inspection sandbox. + /// Check EROFS tools, the inspection sandbox and legacy xz reader. Doctor, /// Build or package software on the workstation, never on the Pi. Software { @@ -25,7 +27,7 @@ enum Action { }, /// Construct and verify a complete 1+m GPT cartridge image before burning. Create { recipe: PathBuf, output: PathBuf }, - /// Verify GPT, every filesystem, catalogue and xz tarball in a prepared image. + /// Verify GPT, every filesystem, catalogue and installed program tree. Inspect { image: PathBuf }, /// Verify a prepared image and save an image/target-bound write preview. Preview { @@ -45,11 +47,9 @@ enum Action { } #[derive(Subcommand)] enum Software { - /// Execute an explicit trusted workstation build recipe, then package its output. + /// Build a Void source template and install its complete runtime package tree. Build { recipe: PathBuf, output: PathBuf }, - /// Package an existing software tree without running its build command. - Pack { recipe: PathBuf, output: PathBuf }, - /// Check a built software descriptor, archive digest and extracted contents. + /// Check installed package metadata and every program/dependency file. Inspect { directory: PathBuf }, } fn run() -> Result<()> { @@ -76,16 +76,18 @@ fn run() -> Result<()> { } Action::Software { command } => serde_json::to_value(match command { Software::Build { recipe, output } => { - fds_workstation::software::build(&recipe, &output, true)? - } - Software::Pack { recipe, output } => { - fds_workstation::software::build(&recipe, &output, false)? + fds_workstation::software::build(&recipe, &output, &cli.build)? } Software::Inspect { directory } => fds_workstation::software::load(&directory)?, }), - Action::Create { recipe, output } => serde_json::to_value( - fds_workstation::cartridge::create(&recipe, &output, cli.image_tool_runner.as_deref())?, - ), + Action::Create { recipe, output } => { + serde_json::to_value(fds_workstation::cartridge::create( + &recipe, + &output, + cli.image_tool_runner.as_deref(), + &cli.build, + )?) + } Action::Inspect { image } => serde_json::to_value(fds_workstation::cartridge::inspect( &image, cli.image_tool_runner.as_deref(), diff --git a/rust/fds-workstation/src/software.rs b/rust/fds-workstation/src/software.rs index c398308..c796dd7 100644 --- a/rust/fds-workstation/src/software.rs +++ b/rust/fds-workstation/src/software.rs @@ -1,22 +1,34 @@ -use crate::{Work, parent, workstation}; +//! Void source packages are built and installed on the workstation only. +use crate::{Work, image_tool, parent, success, workstation}; use fds_common::{Error, Result, manifest::identifier, read_text}; -use fds_software::{Software, archive}; +use fds_software::{Software, tree}; use serde::Deserialize; use std::{ collections::BTreeMap, - fs::{self, File, OpenOptions}, - os::unix::fs::{OpenOptionsExt, PermissionsExt}, + fs::{self, File}, + os::unix::fs::PermissionsExt, path::{Path, PathBuf}, process::{Command, Stdio}, }; +#[derive(clap::Args, Debug, Clone)] +pub struct BuildOptions { + /// Prepared Void source checkout used by xbps-src (not the workstation OS). + #[arg(long, global = true, default_value = "vendor/void-packages")] + pub void_packages: PathBuf, + /// Optional rootless XBPS wrapper, for example this checkout's tools/in-void. + #[arg(long, global = true)] + pub xbps_tool_runner: Option, + /// Native XBPS executables for xbps-src, when they are not already in PATH. + #[arg(long, global = true)] + pub xbps_bin: Option, +} #[derive(Deserialize)] #[serde(deny_unknown_fields)] -struct Build { - directory: PathBuf, - command: Vec, - #[serde(default)] - environment: BTreeMap, +struct Source { + package: String, + /// Omit to build an existing source package from the selected Void checkout. + template: Option, } #[derive(Deserialize)] #[serde(deny_unknown_fields)] @@ -25,97 +37,215 @@ struct Recipe { id: String, name: String, version: String, - architecture: String, - root: PathBuf, commands: BTreeMap, - build: Option, + source: Source, } -pub fn build(recipe: &Path, output: &Path, compile: bool) -> Result { +fn xbps(options: &BuildOptions, program: &str) -> Command { + let mut command = image_tool(options.xbps_tool_runner.as_deref(), "env"); + command.args(["XBPS_ARCH=aarch64", "XBPS_TARGET_ARCH=aarch64", program]); + if let Some(directory) = &options.xbps_bin { + let mut paths = vec![directory.clone()]; + paths.extend(std::env::split_paths( + &std::env::var_os("PATH").unwrap_or_default(), + )); + if let Ok(path) = std::env::join_paths(paths) { + command.env("PATH", path); + } + } + command +} +pub fn build(recipe: &Path, output: &Path, options: &BuildOptions) -> Result { workstation()?; if unsafe { libc::geteuid() } == 0 { return Err(Error( - "Run software builds as an ordinary workstation user".into(), + "Build Void software as an ordinary workstation user".into(), )); } if output.try_exists()? { return Err(Error("Software output already exists".into())); } let input: Recipe = toml::from_str(&read_text(recipe, 65536)?) - .map_err(|e| Error(format!("Invalid build recipe: {e}")))?; - if input.format != 1 || !identifier(&input.id) { + .map_err(|e| Error(format!("Invalid Void software recipe: {e}")))?; + if input.format != 2 + || !identifier(&input.id) + || !fds_software::xbps_identifier(&input.source.package) + { return Err(Error( - "Build recipe requires format 1 and a valid software id".into(), + "Software recipe requires format 2, a software id and a Void source package name" + .into(), )); } - let base = parent(recipe)?; - if compile { - let build = input.build.ok_or_else(|| { - Error("Build recipe lacks [build]; use software pack for an existing tree".into()) - })?; - let program = build - .command - .first() - .ok_or_else(|| Error("Build command is empty".into()))?; - let directory = base.join(build.directory).canonicalize()?; - crate::success( - Command::new(program) - .args(&build.command[1..]) - .current_dir(directory) - .envs(build.environment) - .env("FDS_TARGET_ARCH", &input.architecture) - .stdin(Stdio::null()), + let checkout = options.void_packages.canonicalize()?; + if !checkout.join("xbps-src").is_file() { + return Err(Error( + "--void-packages must name a prepared Void source checkout".into(), + )); + } + let destination = checkout.join("srcpkgs").join(&input.source.package); + if let Some(template) = input.source.template { + let source = parent(recipe)?.join(template).canonicalize()?; + if !source.join("template").is_file() { + return Err(Error( + "Source package directory must contain a Void template".into(), + )); + } + if source != destination { + let tracked = Command::new("git") + .arg("-C") + .arg(&checkout) + .args(["ls-files", "--"]) + .arg(format!("srcpkgs/{}", input.source.package)) + .output()?; + if !tracked.status.success() || !tracked.stdout.is_empty() { + return Err(Error( + "Custom source packages cannot replace tracked upstream Void files".into(), + )); + } + if destination.exists() { + if !Command::new("diff") + .args(["-qr", "--"]) + .arg(&source) + .arg(&destination) + .stdout(Stdio::null()) + .status()? + .success() + { + return Err(Error(format!( + "Stale generated source overlay {}; preserve local edits and remove that copy before rebuilding", + destination.display() + ))); + } + } else { + tree::copy(&source, &destination)?; + } + } + } + if !destination.join("template").is_file() { + return Err(Error(format!( + "Void source package {} has no template", + input.source.package + ))); + } + // Source templates are trusted build code. Use normal xbps-src dependency + // resolution and cross compilation, including its workstation build hooks. + let mut source_build = Command::new(checkout.join("xbps-src")); + source_build + .args(["-f", "-a", "aarch64", "pkg", &input.source.package]) + .current_dir(&checkout) + .env("XBPS_ARCH", std::env::consts::ARCH) + .stdout(Stdio::from(std::io::stderr())); + if let Some(directory) = &options.xbps_bin { + let mut paths = vec![directory.canonicalize()?]; + paths.extend(std::env::split_paths( + &std::env::var_os("PATH").unwrap_or_default(), + )); + source_build.env( + "PATH", + std::env::join_paths(paths).map_err(|e| Error(e.to_string()))?, + ); + } + success(&mut source_build)?; + let output_parent = parent(output)?; + let work = Work::new(&output_parent)?; + let staged = work.0.join("software"); + let root = staged.join("root"); + let config = work.0.join("config"); + let cache = work.0.join("cache"); + fs::create_dir_all(root.join("var/db/xbps/keys"))?; + fs::create_dir(&config)?; + fs::create_dir(&cache)?; + for entry in fs::read_dir(checkout.join("common/repo-keys"))? { + let entry = entry?; + if entry.path().extension().is_some_and(|e| e == "plist") { + fs::copy( + entry.path(), + root.join("var/db/xbps/keys").join(entry.file_name()), + )?; + } + } + let mut install = xbps(options, "xbps-install"); + install + .args(["-SyU", "--reproducible", "-i", "-C"]) + .arg(&config) + .arg("-r") + .arg(&root) + .arg("-c") + .arg(&cache) + .args(["-R", "https://repo-default.voidlinux.org/current/aarch64"]) + .arg("-R") + .arg(checkout.join("hostdir/binpkgs")) + .arg(&input.source.package); + // A supplied runner provides its own rootless XBPS environment. Otherwise + // root exists only inside a user namespace with this staging area writable. + if options.xbps_tool_runner.is_some() { + success(install.stdout(Stdio::from(std::io::stderr())))?; + } else { + success( + Command::new("bwrap") + .args([ + "--unshare-user", + "--uid", + "0", + "--gid", + "0", + "--ro-bind", + "/", + "/", + "--dev", + "/dev", + "--proc", + "/proc", + "--bind", + ]) + .arg(&work.0) + .arg(&work.0) + .arg(install.get_program()) + .args(install.get_args()) + .envs( + install + .get_envs() + .filter_map(|(key, value)| value.map(|value| (key, value))), + ) + .stdout(Stdio::from(std::io::stderr())), )?; } - let root = base.join(input.root).canonicalize()?; - let output_parent = parent(output)?; - if output_parent.starts_with(&root) { - return Err(Error( - "Software output must be outside its source tree".into(), - )); + let list = xbps(options, "xbps-query") + .arg("-r") + .arg(&root) + .arg("-l") + .output()?; + if !list.status.success() { + return Err(Error("Cannot read installed XBPS package inventory".into())); } - let work = Work::new(&output_parent)?; - let bundle = work.0.join("bundle"); - fs::create_dir(&bundle)?; - let archive_path = bundle.join(format!("{}.tar.xz", input.id)); - let encoded = OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o644) - .open(&archive_path)?; - let mut child = Command::new("xz") - .args(["--compress", "--stdout", "--threads=1", "-6"]) - .stdin(Stdio::piped()) - .stdout(Stdio::from(encoded)) - .stderr(Stdio::inherit()) - .spawn()?; - let stats = archive::write_tar(&root, child.stdin.take().unwrap(), &input.architecture); - if stats.is_err() { - let _ = child.kill(); - } - let status = child.wait()?; - let stats = stats?; - if !status.success() { - return Err(Error("XZ compression failed".into())); - } - let file = archive::open(&archive_path)?; + let mut packages: Vec<_> = String::from_utf8_lossy(&list.stdout) + .lines() + .filter_map(|line| line.split_whitespace().nth(1).map(str::to_owned)) + .collect(); + packages.sort(); + packages.dedup(); + tree::relocate(&root)?; + let inventory = tree::inspect(&root, "aarch64")?; let software = Software { id: input.id, name: input.name, version: input.version, - architecture: input.architecture, + architecture: "aarch64".into(), partition: 2, - archive_bytes: file.metadata()?.len(), - unpacked_bytes: stats.bytes, - entries: stats.entries, - sha256: archive::digest(&file)?, + installed: true, + packages, + archive_bytes: 0, + unpacked_bytes: inventory.bytes, + entries: inventory.entries, + sha256: inventory.sha256, commands: input.commands, }; - archive::verify(&file, &software, None)?; - let descriptor = toml::to_string(&software).map_err(|e| Error(e.to_string()))?; - fs::write(bundle.join("software.toml"), descriptor)?; - fs::set_permissions(&bundle, fs::Permissions::from_mode(0o755))?; - // renameat2(NO_REPLACE) publishes the complete directory without overwriting. - let from = std::ffi::CString::new(bundle.as_os_str().as_encoded_bytes()) + tree::verify(&root, &software)?; + fs::write( + staged.join("software.toml"), + toml::to_string(&software).map_err(|e| Error(e.to_string()))?, + )?; + fs::set_permissions(&staged, fs::Permissions::from_mode(0o755))?; + let from = std::ffi::CString::new(staged.as_os_str().as_encoded_bytes()) .map_err(|e| Error(e.to_string()))?; let to = std::ffi::CString::new(output.as_os_str().as_encoded_bytes()) .map_err(|e| Error(e.to_string()))?; @@ -136,17 +266,14 @@ pub fn build(recipe: &Path, output: &Path, compile: bool) -> Result { } pub fn load(directory: &Path) -> Result { let path = directory.join("software.toml"); - let metadata = fs::symlink_metadata(&path)?; - if !metadata.is_file() { + if !path.symlink_metadata()?.is_file() { return Err(Error("Software descriptor must be a regular file".into())); } let software: Software = toml::from_str(&read_text(&path, 65536)?) .map_err(|e| Error(format!("Invalid software descriptor: {e}")))?; - software.validate()?; - archive::verify( - &archive::open(&directory.join(format!("{}.tar.xz", software.id)))?, - &software, - None, - )?; + if !software.installed { + return Err(Error("New cartridges require installed Void packages; rebuild this legacy bundle from its source template".into())); + } + tree::verify(&directory.join("root"), &software)?; Ok(software) } diff --git a/tests/hardware/README.md b/tests/hardware/README.md index bdf3d9d..b9eda20 100644 --- a/tests/hardware/README.md +++ b/tests/hardware/README.md @@ -1,7 +1,7 @@ # Physical acceptance records Physical tests are deferred until the Pi and its attached hardware are available. -Follow [the complete procedure](../../docs/stress-testing.md), including bay +Follow [the complete procedure](../../docs/developer/stress-testing.md), including bay calibration, repeatable device populations, explicit SAFE handling and separate external timing measurements. diff --git a/tests/integration/clean-checks.py b/tests/integration/clean-checks.py new file mode 100644 index 0000000..45e6ebc --- /dev/null +++ b/tests/integration/clean-checks.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Exercise destructive cleanup only in small, disposable Git repositories.""" +import fcntl +import contextlib +import importlib.machinery +import importlib.util +import io +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +PROJECT = Path(__file__).resolve().parents[2] +SCRIPT = PROJECT / 'tools/clean-builds' +loader = importlib.machinery.SourceFileLoader('clean_builds', str(SCRIPT)) +spec = importlib.util.spec_from_loader(loader.name, loader) +clean = importlib.util.module_from_spec(spec) +loader.exec_module(clean) + + +class CleanupTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix='fds-clean-test.') + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) / 'project' + (self.root / 'tools').mkdir(parents=True) + (self.root / 'out/manifests').mkdir(parents=True) + shutil.copyfile(SCRIPT, self.root / 'tools/clean-builds') + subprocess.run(['git', 'init', '-q', self.root], check=True) + + def file(self, name, content='fixture'): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + return path + + def run_cli(self, *args): + return subprocess.run([sys.executable, self.root / 'tools/clean-builds', *args], + cwd=self.root, text=True, capture_output=True, timeout=30) + + def selected(self): + return {p.relative_to(self.root).as_posix() for p in clean.candidates(self.root)[0]} + + def test_cleanup_and_repeat_preserve_published_and_user_files(self): + old = self.file('out/rootfs-build.OLD123/root/data') + self.file('target/debug/program') + keep = [ + self.file('out/rootfs-build.NEW123/rootfs.tar'), + self.file('out/fds-os-0.1.0/release.img'), + self.file('out/inputs-m12-v5/lock.json'), + self.file('out/rebuild-m12-v5-a/out/image.img'), + self.file('out/my-emulator/data.qcow2'), + self.file('out/emulator/session.json'), + self.file('out/my-cartridge.img'), + self.file('out/cache/rootfs/package.xbps'), + self.file('out/logs/build.log'), + self.file('out/manifests/acceptance.json'), + self.file('out/.gitkeep'), + self.file('.host/xbps/tool'), + self.file('vendor/void-packages/hostdir/download'), + ] + (self.root / 'out/rootfs-cli.tar').symlink_to('rootfs-build.NEW123/rootfs.tar') + preview = self.run_cli('--dry-run') + self.assertEqual(preview.returncode, 0, preview.stderr) + self.assertTrue(old.exists()) + self.assertFalse((self.root / 'out/.clean.lock').exists()) + result = self.run_cli() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(old.exists()) + self.assertFalse((self.root / 'target').exists()) + self.assertTrue(all(p.read_text() == 'fixture' for p in keep)) + self.assertEqual((self.root / 'out/rootfs-cli.tar').read_text(), 'fixture') + repeat = self.run_cli() + self.assertEqual(repeat.returncode, 0, repeat.stderr) + self.assertIn('removed 0 disposable directories', repeat.stdout) + + def test_text_pointers_latest_fixture_markers_and_transitive_links(self): + self.file('out/emu-test.old000/normal/data.qcow2') + self.file('out/emu-test.new000/normal/data.qcow2') + self.file('out/workstation-emulator-current.txt', str(self.root / 'out/emu-test.new000') + '\n') + self.file('out/dasung-s6.old000/compiled/a') + self.file('out/dasung-s6.new000/compiled/a') + self.file('out/manifests/dasung-s6-database.txt', str(self.root / 'out/dasung-s6.new000/compiled')) + self.file('out/workstation-images.new000/acceptance.json') + self.file('out/workstation-images-current.txt', 'out/workstation-images.new000\n') + self.file('out/m9-images.old000/root/program.img') + self.file('out/m9-images.new000/root/program.img') + os.utime(self.root / 'out/m9-images.old000', ns=(1, 1)) + self.file('out/m8-vm.kept00/.fds-keep', '') + self.file('out/m8-vm.other0/disk.img') + (self.root / 'out/m8-vm.kept00/dependency').symlink_to('../m8-vm.other0') + self.assertEqual(self.selected(), { + 'out/emu-test.old000', 'out/dasung-s6.old000', 'out/m9-images.old000'}) + + def test_symlinks_do_not_delete_external_content(self): + outside = Path(self.temporary.name) / 'outside' + outside.mkdir() + (outside / 'data').write_text('precious') + (self.root / 'target').symlink_to(outside) + (self.root / 'out/m8-vm.abcdef').symlink_to(outside) + work = self.file('out/m8-vm.ghijkl/data').parent + (work / 'external').symlink_to(outside) + result = self.run_cli() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(work.exists()) + self.assertTrue((self.root / 'target').is_symlink()) + self.assertEqual((outside / 'data').read_text(), 'precious') + shutil.rmtree(self.root / 'out') + (self.root / 'out').symlink_to(outside) + self.assertNotEqual(self.run_cli().returncode, 0) + self.assertEqual((outside / 'data').read_text(), 'precious') + + def test_tracked_files_are_kept(self): + self.file('out/m2-vm.abcdef/notes') + subprocess.run(['git', '-C', self.root, 'add', 'out/m2-vm.abcdef/notes'], check=True) + self.assertEqual(self.selected(), set()) + + def test_directory_permissions_and_hardlinks(self): + kept = self.file('out/kept-file') + kept.chmod(0o444) + root = self.root / 'out/rootfs-build.abcdef/root' + root.mkdir(parents=True) + os.link(kept, root / 'file') + root.chmod(0o555) + result = self.run_cli() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(root.exists()) + self.assertEqual(kept.stat().st_mode & 0o777, 0o444) + self.assertEqual(kept.read_text(), 'fixture') + + def test_held_build_lock_rejects_without_deletion(self): + old = self.file('out/m2-vm.abcdef/image') + with (self.root / 'out/.rootfs.lock').open('w') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + result = self.run_cli() + self.assertNotEqual(result.returncode, 0) + self.assertIn('locked', result.stderr) + self.assertTrue(old.exists()) + + def test_live_process_rejects_without_deletion(self): + old = self.file('out/m2-vm.abcdef/image') + child = subprocess.Popen( + [sys.executable, '-c', 'import sys; print("ready", flush=True); sys.stdin.read()'], + cwd=old.parent, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + try: + self.assertEqual(child.stdout.readline().strip(), 'ready') + result = self.run_cli() + self.assertNotEqual(result.returncode, 0) + self.assertIn('is using', result.stderr) + self.assertTrue(old.exists()) + finally: + child.communicate(timeout=10) + + def test_mount_rejection_including_escaped_spaces(self): + mount = self.root / 'out/m2-vm.abcdef/a directory' + line = '1 0 0:1 / ' + str(mount).replace(' ', r'\040') + ' rw - tmpfs tmpfs rw\n' + with self.assertRaisesRegex(ValueError, 'mounted path'): + clean.check_mounts([mount.parent], line) + clean.check_mounts([self.root / 'target'], line) + + def test_newly_published_output_aborts_before_any_deletion(self): + old = self.file('out/system-build.abcdef/system.img') + + def publish(_paths): + (self.root / 'out/fds-system-cli.img').symlink_to('system-build.abcdef/system.img') + return 0 + + with patch.object(clean, 'footprint', side_effect=publish), contextlib.redirect_stdout(io.StringIO()): + with self.assertRaisesRegex(ValueError, 'selection changed'): + clean.clean(self.root, False) + self.assertTrue(old.exists()) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/integration/m10-runtime.py b/tests/integration/m10-runtime.py index 4d5a933..1ab82f6 100644 --- a/tests/integration/m10-runtime.py +++ b/tests/integration/m10-runtime.py @@ -250,5 +250,9 @@ with VM(work,'writeback-error',system,extra=['-device',controller]) as vm: vm.qmp('device_del',{'id':'bay2'});bay(vm,2,'empty') vm.send('s6-setuidgid fds fds poweroff');finished(vm) print('PASS: attached-device writeback EIO is retained across restarts and refuses SAFE and shutdown',flush=True) +link=project/'out/m10-vm-latest' +temporary=link.with_suffix('.next') +temporary.symlink_to(work.name) +temporary.replace(link) print(f'PASS: M10 ordered shutdown evidence: {work}') print('SKIP: physical Pi poweroff/reboot, flash-controller durability, battery behavior and sub-second hardware timing targets') diff --git a/tests/integration/m8-runtime.py b/tests/integration/m8-runtime.py index f8ae57f..5e1f2d0 100755 --- a/tests/integration/m8-runtime.py +++ b/tests/integration/m8-runtime.py @@ -53,7 +53,7 @@ def desktop(vm, enabled): return p['desktop']==('windowmaker' if enabled else 'cli') and (p['ready_ns'] is not None if enabled else True) return json.loads(wait(vm,'fds --json profiles',condition))['profiles'] def image(name, replacements): - replacements={**replacements, **{f'usr/bin/{binary}':(project/'out'/binary).read_bytes() for binary in ['fds','fds-cartridged','fds-profile']}} + replacements={**replacements, **{f'usr/bin/{binary}':(project/'out'/binary).read_bytes() for binary in ['fds','fds-program','fds-control','fds-cartridged','fds-profile']}} with tarfile.open(project/'out/rootfs-development.tar') as source, tarfile.open(work/(name+'.tar'),'w',format=tarfile.PAX_FORMAT) as output: assert source.extractfile('usr/share/fds/image-profile').read().strip()==b'development', 'Build PROFILE=development before this test' seen=set() @@ -86,10 +86,13 @@ with VM(work,'probe',probe,extra=['-device',controller,'-device','usb-kbd,bus=xh print(capture(vm,'cat /run/log/xserver/current /run/log/desktop/current /run/log/cartridged/current'),flush=True) raise shell(vm,'test ! -d /home/fds/.cache/fontconfig','M8_PREBUILT_FONT_CACHE') + shell(vm,'cmp /etc/WindowMaker/WindowMaker /usr/share/fds/eink/WindowMaker && cmp /etc/WindowMaker/WMRootMenu /usr/share/fds/eink/WMRootMenu && grep -q "FDS Control" /etc/WindowMaker/WMRootMenu','FDS_GLOBAL_RETRO_DEFAULTS') (work/'activation.json').write_text(json.dumps(p,indent=2)+'\n') assert p['ready_ns']>=p['activation_ns']>0 shell(vm,'export DISPLAY=:0 XAUTHORITY=/run/fds/x11/authority; s6-setuidgid fds xdpyinfo >/tmp/display-info; ! grep -q "COMPOSITE" /tmp/display-info','M8_X11_AUTHENTICATED') shell(vm,'if env XAUTHORITY=/dev/null xdpyinfo >/dev/null 2>&1; then false; else true; fi','M8_X11_REJECTS_NO_COOKIE') + panel=wait(vm,'s6-setuidgid fds xdotool search --onlyvisible --name "^FDS Control$"',lambda s:s.isdigit()) + shell(vm,'test "$(ps -o uid= -C fds-control | xargs)" = 1000','FDS_CONTROL_UNPRIVILEGED') window=wait(vm,'s6-setuidgid fds xdotool search --onlyvisible --name "^FDS Terminal$"',lambda s:s.isdigit()) shell(vm,f's6-setuidgid fds xdotool windowactivate --sync {window} type --clearmodifiers "printf M8_INPUT_WORKED > /home/fds/desktop-input"; s6-setuidgid fds xdotool key --clearmodifiers Return','M8_X11_INPUT_SENT') wait(vm,'cat /home/fds/desktop-input 2>/dev/null',lambda s:s=='M8_INPUT_WORKED') @@ -145,12 +148,58 @@ with VM(work,'cartridges',fixture,extra=['-device',controller,'-netdev','user,id add(env,2);bay(vm,2,'mounted_read_only');desktop(vm,True);remove(2);desktop(vm,False) add(program,4);entry=bay(vm,4,'mounted_read_only');assert entry['mount']=='/run/fds/apps/fds.program.test' shell(vm,'test ! -e /home/fds/program-identity && ! /run/fds/apps/fds.program.test/app/bin/check','M8_NO_AUTORUN') - started=query(vm,'s6-setuidgid fds fds --json run 4 -- check');assert started['started_pid']>1 + shell(vm,'fds profile activate windowmaker','FDS_CONTROL_START_DESKTOP') + desktop(vm,True) + shell(vm,'export DISPLAY=:0 XAUTHORITY=/run/fds/x11/authority','FDS_CONTROL_DISPLAY') + panel=wait(vm,'s6-setuidgid fds xdotool search --onlyvisible --name "^FDS Control$"',lambda s:s.isdigit()) + wait(vm,'s6-setuidgid fds xdotool search --onlyvisible --name "^FDS Terminal$"',lambda s:s.isdigit()) + shell(vm,f's6-setuidgid fds xdotool windowactivate --sync {panel} windowraise {panel} mousemove --window {panel} 70 218 click 1','FDS_CONTROL_SELECT_BAY') + def panel_state(): + output=capture(vm,f's6-setuidgid fds xprop -id {panel} _FDS_CONTROL_STATE') + return json.loads(json.loads(output.split(' = ',1)[1])) + deadline=time.monotonic()+30 + while True: + snapshot=panel_state() + if snapshot['bay']==4 and snapshot['commands']>0 and not snapshot['busy']: break + assert time.monotonic()/dev/null',lambda s:'uid=1000(fds)' in s) + assert query(vm,'fds --json bay 4')['bays'][0]['consumers']>0 + shell(vm,f's6-setuidgid fds xdotool windowactivate --sync {panel}','FDS_CONTROL_SHOW_PANEL') + shell(vm,'s6-setuidgid fds xwd -root -silent -out /home/fds/control.xwd','FDS_CONTROL_SCREENSHOT') + (work/'legacy-control.xwd').write_bytes(gzip.decompress(base64.b64decode(capture(vm,'gzip -c /home/fds/control.xwd | base64 -w0')))) wait(vm,'cat /home/fds/program-identity 2>/dev/null',lambda s:'uid=1000(fds)' in s) assert '/run/fds/apps/fds.program.test/app/lib' in capture(vm,'cat /home/fds/program-paths') shell(vm,'if fds run 4 -- escape; then false; else true; fi','M8_PROGRAM_ESCAPE_REJECTED') - shell(vm,'if fds run 4 -- ../check; then false; else true; fi; fds eject 4','M8_PROGRAM_SAFE') + shell(vm,'if fds run 4 -- ../check; then false; else true; fi','M8_PROGRAM_PATH_REJECTED') + deadline=time.monotonic()+30 + while panel_state()['busy']: + assert time.monotonic()=2 and not snapshot['busy']: break + assert time.monotonic()25I',data[:100]);size,version,fmt,depth,width,height,xoff,order,unit,bitorder,pad,bpp,stride,visual,rm,gm,bm,*_=header +data=(work/'control.xwd').read_bytes();header=struct.unpack('>25I',data[:100]);size,version,fmt,depth,width,height,xoff,order,unit,bitorder,pad,bpp,stride,visual,rm,gm,bm,*_=header assert version==7 and fmt==2 and bpp==32 and (rm,gm,bm)==(0xff0000,0xff00,0xff) offset=size+header[19]*12 raw=bytearray() @@ -183,7 +232,8 @@ for y in range(height): pixel=int.from_bytes(data[offset+y*stride+x*4:offset+y*stride+x*4+4], 'little' if order==0 else 'big') raw.extend(((pixel>>16)&255,(pixel>>8)&255,pixel&255)) def chunk(kind,body):return struct.pack('>I',len(body))+kind+body+struct.pack('>I',zlib.crc32(kind+body)&0xffffffff) -(work/'desktop.png').write_bytes(b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',width,height,8,2,0,0,0))+chunk(b'IDAT',zlib.compress(raw))+chunk(b'IEND',b'')) +(work/'control.png').write_bytes(b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',width,height,8,2,0,0,0))+chunk(b'IDAT',zlib.compress(raw))+chunk(b'IEND',b'')) +(work/'acceptance.json').write_text(json.dumps(dict(status='passed', native_arm_x11=True, panel_uid=1000, global_grayscale_defaults=True, legacy_and_void_programs_launched_by_mouse=True, keyboard_rescan=True, panel_safe_eject=True, desktop_stop_removes_panel=True, physical_display='not tested'),indent=2)+'\n') link=project/'out/m8-vm-latest';temporary=link.with_suffix('.next');temporary.symlink_to(work.name);temporary.replace(link) print(f'PASS: M8 ARM desktop and network verification: {work}') print('SKIP: Pi DRM/VC4 output, physical E-Ink quality and monitor input-to-refresh latency') diff --git a/tests/integration/workstation-emulator.py b/tests/integration/workstation-emulator.py index a8520a6..3a2fa2d 100644 --- a/tests/integration/workstation-emulator.py +++ b/tests/integration/workstation-emulator.py @@ -3,6 +3,8 @@ import argparse import io import json +import hashlib +import lzma import os import pty import selectors @@ -63,6 +65,73 @@ def clean(number): assert f'/run/fds/software/{number:02}/' not in text assert '/run/fds/apps/demo.workstation' not in text +def interactive_program_checks(): + master, slave = pty.openpty() + original = termios.tcgetattr(slave) + console = subprocess.Popen([cli, '--session', str(session), 'console'], stdin=slave, stdout=slave, stderr=slave) + selector = selectors.DefaultSelector(); selector.register(master, selectors.EVENT_READ) + pending = bytearray() + def until(marker): + output = pending; deadline = time.monotonic() + 30 + while marker not in output: + assert time.monotonic() < deadline, output + for _, _ in selector.select(max(0, deadline-time.monotonic())): + chunk = os.read(master, 8192); assert chunk + output.extend(chunk) + end = output.index(marker) + len(marker) + result = bytes(output[:end]); del output[:end] + log.write(repr(result) + '\n'); log.flush() + return result + try: + until(b'FDS> ') + os.write(master, b"stty -g >/tmp/program-tty-before; b01:demo.report:shell -c 'printf \"READY:%s\\n\" tty; read -r line; printf \"REPLY:%s\\n\" \"$line\"'\n") + until(b'READY:tty\r\n') + os.write(master, b'literal interactive input\n') + until(b'REPLY:literal interactive input\r\n') + until(b'FDS> ') + os.write(master, b"b01:demo.report:shell -c 'printf \"SIGNAL:%s\\n\" ready; exec tail -f /dev/null'; printf 'RETURN:%s\\n' \"$?\"\n") + until(b'SIGNAL:ready\r\n') + os.write(master, b'\x03') + until(b'RETURN:130\r\n') + until(b'FDS> ') + os.write(master, b"b01:demo.report:shell -c 'printf \"STOP:%s\\n\" ready; exec tail -f /dev/null'\n") + until(b'STOP:ready\r\n') + os.write(master, b'\x1a') + until(b'Stopped') + until(b'FDS> ') + os.write(master, b'fg\n') + until(b"exec tail -f /dev/null'\r\n") + os.write(master, b'\x03') + until(b'FDS> ') + os.write(master, b"test \"$(stty -g)\" = \"$(cat /tmp/program-tty-before)\" && printf 'RESTORE:%s\\n' passed\n") + until(b'RESTORE:passed\r\n') + os.write(master, b'\x1d') + assert console.wait(timeout=10) == 0 + assert termios.tcgetattr(slave) == original + finally: + if console.poll() is None: console.kill(); console.wait() + selector.close(); os.close(master); os.close(slave) + +# Compatibility fixture only: the public creator never emits archive payloads. +legacy_metadata=work/'legacy-metadata'; (legacy_metadata/'FDS').mkdir(parents=True) +legacy_payload=work/'legacy-payload'; (legacy_payload/'bundles').mkdir(parents=True) +legacy_program=b'#!/bin/sh\nprintf "Legacy reader: %s\\n" "$1"\n' +legacy_tar=io.BytesIO() +with tarfile.open(fileobj=legacy_tar,mode='w',format=tarfile.USTAR_FORMAT) as archive: + directory=tarfile.TarInfo('bin');directory.type=tarfile.DIRTYPE;directory.mode=0o755;archive.addfile(directory) + program=tarfile.TarInfo('bin/legacy');program.mode=0o755;program.size=len(legacy_program);archive.addfile(program,io.BytesIO(legacy_program)) +legacy_bytes=lzma.compress(legacy_tar.getvalue(),format=lzma.FORMAT_XZ) +(legacy_payload/'bundles/legacy.reader.tar.xz').write_bytes(legacy_bytes) +(legacy_metadata/'FDS/CARTRIDGE.TOML').write_text('format=1\n[cartridge]\nid="legacy.fixture"\nname="Legacy reader fixture"\nclass="program"\nversion="1"\n[media]\nwritable=false\n') +(legacy_metadata/'FDS/SOFTWARE.TOML').write_text(f'format=1\n[[software]]\nid="legacy.reader"\nname="Legacy reader"\nversion="1"\narchitecture="any"\npartition=2\narchive_bytes={len(legacy_bytes)}\nunpacked_bytes={len(legacy_program)}\nentries=2\nsha256="{hashlib.sha256(legacy_bytes).hexdigest()}"\n[software.commands]\nlegacy="bin/legacy"\n') +legacy_parts=[] +for name,tree in [('FDS_METADATA',legacy_metadata),('FDS_PAYLOAD02',legacy_payload)]: + filesystem=work/(name+'.erofs') + subprocess.run([str(project/'tools/in-image-tools'),'mkfs.erofs','--quiet','-T','0',str(filesystem),str(tree)],check=True,stdout=log,stderr=log) + legacy_parts.append((name,LINUX_FILESYSTEM,filesystem)) +legacy=work/'legacy.img';gpt(legacy,legacy_parts) +subprocess.run([str(project/'out/workstation/fds-cartridge'),'--image-tool-runner',str(project/'tools/in-image-tools'),'inspect',str(legacy)],check=True,stdout=log,stderr=log) + # A workstation-created disposable DATA fixture, never a host block device. data_root = work / 'data-root' (data_root / 'FDS').mkdir(parents=True) @@ -118,32 +187,59 @@ try: if number in [1, 6, 12]: guest('fds', 'run', number, '--', 'demo.hello:hello') guest('fds', 'run', number, '--', 'demo.report:report') - guest('touch', f'/run/fds/software/{number:02}/cache-demo.hello/unexpected', ok=False) + guest('touch', f'/run/fds/software/{number:02}/payload02/programs/demo.hello/unexpected', ok=False) guest('fds', 'run', number, '--', '../escape', ok=False) guest('fds', 'run', number, '--', 'demo.hello:missing', ok=False) + assert guest('hello', 'literal $(id); and spaces').strip() == 'Hello from an FDS AArch64 software cartridge: literal $(id); and spaces' + assert guest(f'b{number:02}:demo.report:shell', '-c', 'id -u').strip() == '1000' + assert guest('sh', '-c', f'printf "pipe input" | b{number:02}:demo.report:shell -c "cat"') == 'pipe input' + assert guest('sh', '-c', f'b{number:02}:demo.report:shell -c "exit 37"; printf "%s" "$?"') == '37' + assert guest('sh', '-c', f'cd /tmp; b{number:02}:demo.report:shell -c "pwd"').strip() == '/tmp' + assert '/cache-' not in mounts() + assert state['commands'] + if number == 1: + interactive_program_checks() + if number == 6: guest('fds', 'run', number, '--', 'demo.report:report', 'hold') - assert query('bay', number)['bays'][0]['consumers'] > 0 + guest('sh', '-c', 'report hold >/tmp/direct-hold.log 2>&1 & echo $! >/tmp/direct-launcher') + deadline = time.monotonic() + 20 + while query('bay', number)['bays'][0]['consumers'] < 2: + assert time.monotonic() < deadline + assert query('bay', number)['bays'][0]['consumers'] >= 2 invoke('unplug', number) else: invoke('eject', number) state = wait_bay(number, 'empty') assert state['consumers'] == 0 clean(number) + if number == 6: + deadline = time.monotonic() + 20 + while guest('sh', '-c', 'test ! -e /proc/$(cat /tmp/direct-launcher); printf "%s" "$?"').strip() != '0': + assert time.monotonic() < deadline + assert not guest('sh', '-c', 'command -v hello || true').strip() + # Two different cartridges export the same names. Lowest bay wins, and + # fully qualified commands remain available before and after an eject. + invoke('insert', 8, software); wait_bay(8, 'mounted_read_only') + invoke('insert', 2, builder / 'collision.img'); wait_bay(2, 'mounted_read_only') + assert '/02/' in guest('where', 'FDS_APP') + assert '/08/' in guest('b08:demo.report:where', 'FDS_APP') + invoke('eject', 2); wait_bay(2, 'empty') + assert '/08/' in guest('where', 'FDS_APP') + invoke('eject', 8); wait_bay(8, 'empty') remaining = json.loads(invoke('status')) assert remaining['state']['cartridges'] == {} assert len(remaining['block_nodes']) == 2, remaining['block_nodes'] output = guest('cat', '/run/log/cartridged/current') - assert 'FDS cartridge AArch64 hello' in output - assert 'FDS cartridge script report' in output + assert 'Hello from an FDS AArch64 software cartridge' in output + assert 'FDS cartridge system report' in output assert 'uid=1000(fds)' in output (work / 'program-output.log').write_text(output) - invoke('insert', 3, builder / 'archive-corrupt.img') - wait_bay(3, 'mounted_read_only') - failure = guest('fds', 'run', 3, '--', 'demo.hello:hello', ok=False) - assert 'digest' in failure.lower() or 'sha' in failure.lower(), failure - assert '/run/fds/software/03/cache-' not in mounts() - invoke('eject', 3) + invoke('insert', 3, builder / 'program-corrupt.img') + failed = wait_bay(3, 'error') + assert 'digest' in failed['detail'].lower() or 'hash' in failed['detail'].lower(), failed + assert '/run/fds/software/03/' not in mounts() + invoke('unplug', 3) wait_bay(3, 'empty') invoke('insert', 4, builder / 'catalogue-mapping.img') wait_bay(4, 'error') @@ -162,7 +258,13 @@ try: assert digest(data) == original_data assert overlay.is_file() assert len(json.loads(invoke('status'))['block_nodes']) == 2 - # Keep a verified software cache mounted across native shutdown. + invoke('insert', 7, legacy); wait_bay(7, 'mounted_read_only') + assert guest('legacy', 'compatibility').strip() == 'Legacy reader: compatibility' + assert '/run/fds/software/07/cache-legacy.reader' in mounts() + guest('fds', 'run', 7, '--', 'legacy.reader:legacy', 'background') + invoke('eject', 7); wait_bay(7, 'empty'); clean(7) + assert not guest('sh', '-c', 'command -v legacy || true').strip() + # Keep a directly mounted software tree active across native shutdown. invoke('insert', 12, software) wait_bay(12, 'mounted_read_only') guest('fds', 'run', 12, '--', 'demo.report:report', 'hold') @@ -193,15 +295,22 @@ try: assert guest('id', '-u').strip() == '0' invoke('insert', 1, software) wait_bay(1, 'mounted_read_only') - guest('fds', 'run', 1, '--', 'demo.report:report', 'hold') - assert '/run/fds/software/01/cache-demo.report' in mounts() + guest('sh', '-c', 'cd /home/fds; /run/fds/bin/report hold >/tmp/restart-direct.log 2>&1 &') + deadline = time.monotonic() + 20 + while query('bay', 1)['bays'][0]['consumers'] == 0: + assert time.monotonic() < deadline + assert '/run/fds/software/01/payload03' in mounts() + assert '/cache-' not in mounts() guest('s6-rc', '-l', '/run/s6-rc', '-d', 'change', 'cartridged') guest('s6-rc', '-l', '/run/s6-rc', '-u', 'change', 'cartridged') wait_bay(1, 'mounted_read_only') assert query('bay', 1)['bays'][0]['consumers'] == 0 assert '/run/fds/software/01/cache-' not in mounts() guest('fds', 'run', 1, '--', 'demo.hello:hello') - # Extra payload aliases must prevent SAFE; cache is genuinely read-only. + assert 'Hello from' in guest('sh', '-c', 'cd /home/fds; /run/fds/bin/hello') + guest('mkdir', '-m', '700', '/tmp/private-cwd') + assert 'Permission denied' in guest('sh', '-c', 'cd /tmp/private-cwd; /run/fds/bin/hello', ok=False) + # Extra payload aliases must prevent SAFE; program trees are read-only. guest('mkdir', '/run/extra-payload') guest('mount', '--bind', '/run/fds/software/01/payload02', '/run/extra-payload') invoke('eject', 1, ok=False) @@ -216,10 +325,11 @@ finally: assert digest(software) == original_software and digest(data) == original_data record = dict(status='passed', ordinary_guest_uid=1000, public_emulator_cli=True, all_twelve_usb_bays=True, interactive_console_detach_and_terminal_restore=True, both_payload_partitions_executed=True, - shared_partition_executed=True, readonly_cache=True, + shared_partition_executed=True, readonly_program_trees_without_extraction=True, legacy_archive_host_and_guest_reader=True, + direct_path_arguments_pipes_exit_status_cwd=True, interactive_program_io_signals_and_restore=True, command_collision_fallback=True, safe_eject=True, forced_removal_stops_consumer=True, - corrupt_archive_and_catalogue_rejected=True, data_overlay_preserves_source=True, - restart_cleans_cache_and_consumers=True, extra_mount_blocks_safe=True, + corrupt_program_and_catalogue_rejected=True, data_overlay_preserves_source=True, + restart_cleans_mounts_commands_and_consumers=True, extra_mount_blocks_safe=True, native_shutdown_with_active_software=True, physical_pi='not tested') (work / 'acceptance.json').write_text(json.dumps(record, indent=2) + '\n') (project / 'out/workstation-emulator-current.txt').write_text(str(work) + '\n') diff --git a/tests/integration/workstation-images.py b/tests/integration/workstation-images.py index d9f43e7..219351b 100644 --- a/tests/integration/workstation-images.py +++ b/tests/integration/workstation-images.py @@ -14,12 +14,19 @@ from image_formats import gpt, LINUX_FILESYSTEM parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--cli', type=Path, required=True) parser.add_argument('--image-tool-runner', type=Path) -parser.add_argument('--cc', nargs='+', default=['aarch64-linux-gnu-gcc']) +parser.add_argument('--void-packages', type=Path, default=project / 'vendor/void-packages') +parser.add_argument('--xbps-tool-runner', type=Path) +parser.add_argument('--xbps-bin', type=Path) args = parser.parse_args() work = Path(tempfile.mkdtemp(prefix='workstation-images.', dir=project / 'out')) cli = [str(args.cli.resolve())] if args.image_tool_runner: cli += ['--image-tool-runner', str(args.image_tool_runner)] +cli += ['--void-packages', str(args.void_packages.resolve())] +if args.xbps_tool_runner: + cli += ['--xbps-tool-runner', str(args.xbps_tool_runner.resolve())] +if args.xbps_bin: + cli += ['--xbps-bin', str(args.xbps_bin.resolve())] log = (work / 'commands.log').open('w') @@ -36,39 +43,32 @@ def digest(path): return hashlib.file_digest(stream, 'sha256').hexdigest() -(work / 'hello-root/bin').mkdir(parents=True) -(work / 'report-root/bin').mkdir(parents=True) -(work / 'hello.c').write_text('#include \nint main(void) { puts("FDS cartridge AArch64 hello"); return 0; }\n') -script = work / 'report-root/bin/report' -script.write_text('#!/bin/sh\nprintf "FDS cartridge script report\\n"\nid\n[ "${1-}" != hold ] || exec tail -f /dev/null\n') -script.chmod(0o755) -recipe = f'''format=1 -id="demo.hello" -name="AArch64 hello" -version="1.0" -architecture="aarch64" -root="hello-root" -[commands] -hello="bin/hello" -[build] -directory={json.dumps(str(project))} -command={json.dumps([*args.cc, '-O2', str(work / 'hello.c'), '-o', str(work / 'hello-root/bin/hello')])} -''' -(work / 'hello.toml').write_text(recipe) -(work / 'report.toml').write_text('format=1\nid="demo.report"\nname="Script report"\nversion="1.0"\narchitecture="any"\nroot="report-root"\n[commands]\nreport="bin/report"\n') -invoke(['software', 'build', work / 'hello.toml', work / 'hello-bundle']) -invoke(['software', 'pack', work / 'report.toml', work / 'report-bundle']) -# Architecture-independent bundles cannot hide actual ELF executables. -(work / 'wrong-architecture.toml').write_text(recipe.replace('architecture="aarch64"', 'architecture="any"')) -invoke(['software', 'pack', work / 'wrong-architecture.toml', work / 'rejected-architecture'], False) -assert not (work / 'rejected-architecture').exists() -# Source links outside the software root must never become bundle contents. -(work / 'report-root/bin/escape').symlink_to('/etc/passwd') -invoke(['software', 'pack', work / 'report.toml', work / 'rejected-symlink'], False) -assert not (work / 'rejected-symlink').exists() -(work / 'report-root/bin/escape').unlink() -invoke(['software', 'pack', work / 'report.toml', work / 'report-bundle'], False) -(work / 'cartridge.toml').write_text('format=1\nid="demo.workstation"\nname="Workstation software"\nversion="1.0"\n[[payload]]\nbundles=["hello-bundle"]\n[[payload]]\nbundles=["report-bundle"]\n') +for name in ['hello', 'report']: + recipe = project / f'examples/software/{name}/software.toml' + if name == 'report': + text = recipe.read_text().replace('[commands]', '[commands]\nshell="usr/bin/bash"\nwhere="usr/bin/printenv"') + text = text.replace('template = "void"', 'template = ' + json.dumps(str(recipe.parent / 'void'))) + recipe = work / 'report-source.toml'; recipe.write_text(text) + invoke(['software', 'build', recipe, work / f'{name}-installed']) + inspected = json.loads(invoke(['software', 'inspect', work / f'{name}-installed'])) + assert inspected['installed'] and inspected['packages'] + assert f'fds-demo-{name}-1.0_1' in inspected['packages'] + assert any(p.startswith('glibc-') for p in inspected['packages']) + assert (work / f'{name}-installed/root/usr/bin/{name}').is_file() + assert not list((work / f'{name}-installed').glob('*.tar.xz')) +# The actual installed trees, including command links, are integrity checked. +program = work / 'hello-installed/root/usr/bin/hello' +original_program = program.read_bytes() +bad_elf = bytearray(original_program); bad_elf[18:20] = bytes([62, 0]) +program.write_bytes(bad_elf) +invoke(['software', 'inspect', work / 'hello-installed'], False) +program.write_bytes(original_program) +escape = work / 'report-installed/root/usr/bin/escape' +escape.symlink_to('/etc/passwd') +invoke(['software', 'inspect', work / 'report-installed'], False) +escape.unlink() +invoke(['software', 'build', project / 'examples/software/hello/software.toml', work / 'hello-installed'], False) +(work / 'cartridge.toml').write_text('format=2\nid="demo.workstation"\nname="Workstation software"\nversion="1.0"\n[[payload]]\nsources=["hello-installed"]\n[[payload]]\nsources=["report-installed"]\n') image = work / 'software.img' original = json.loads(invoke(['create', work / 'cartridge.toml', image])) assert len(original['image']['partitions']) == 3 @@ -78,18 +78,15 @@ assert [p['name'] for p in observed['partitions']] == ['FDS_METADATA', 'FDS_PAYL invoke(['create', work / 'cartridge.toml', work / 'repeat.img']) assert digest(image) == digest(work / 'repeat.img') invoke(['create', work / 'cartridge.toml', image], False) -shared = (work / 'cartridge.toml').read_text().replace('bundles=["hello-bundle"]\n[[payload]]\nbundles=["report-bundle"]', 'bundles=["hello-bundle","report-bundle"]') +(work / 'collision.toml').write_text((work / 'cartridge.toml').read_text().replace('demo.workstation', 'demo.second')) +invoke(['create', work / 'collision.toml', work / 'collision.img']) +shared = (work / 'cartridge.toml').read_text().replace('sources=["hello-installed"]\n[[payload]]\nsources=["report-installed"]', 'sources=["hello-installed","report-installed"]') (work / 'shared.toml').write_text(shared) grouped = json.loads(invoke(['create', work / 'shared.toml', work / 'shared.img'])) assert len(grouped['image']['partitions']) == 2 assert [s['partition'] for s in grouped['catalogue']['software']] == [2, 2] for name in ['software.img', 'shared.img']: subprocess.run(['sfdisk', '--verify', str(work / name)], check=True, stdout=log, stderr=log) -for directory in ['hello-bundle', 'report-bundle']: - archive = next((work / directory).glob('*.tar.xz')) - subprocess.run(['xz', '--test', str(archive)], check=True) - subprocess.run(['tar', '-tJf', str(archive)], check=True, stdout=log) - size = image.stat().st_size for label, extra in [('exact', 0), ('larger', 8 * 1024 * 1024)]: target = work / (label + '.target') @@ -135,14 +132,14 @@ for part in parts: target.write(source.read(part['bytes'])) filesystems.append(filesystem) tools = [str(args.image_tool_runner.resolve())] if args.image_tool_runner else [] -for case in ['archive-corrupt', 'catalogue-mapping']: - number = 2 if case == 'archive-corrupt' else 1 +for case in ['program-corrupt', 'catalogue-mapping']: + number = 2 if case == 'program-corrupt' else 1 tree = work / (case + '-tree') subprocess.run([*tools, 'fsck.erofs', '--extract=' + str(tree), str(filesystems[number-1])], check=True, stdout=log, stderr=log) if number == 2: - archive = next((tree / 'bundles').glob('*.tar.xz')) - content = bytearray(archive.read_bytes()); content[len(content)//2] ^= 1 - archive.write_bytes(content) + program = tree / 'programs/demo.hello/usr/bin/hello' + content = bytearray(program.read_bytes()); content[len(content)//2] ^= 1 + program.write_bytes(content) else: metadata = tree / 'FDS/SOFTWARE.TOML' metadata.write_text(metadata.read_text().replace('partition = 2', 'partition = 4')) @@ -153,11 +150,11 @@ for case in ['archive-corrupt', 'catalogue-mapping']: gpt(malformed, [(part['name'], LINUX_FILESYSTEM, filesystem) for part, filesystem in zip(parts, selected)]) invoke(['inspect', malformed], False) record = dict(status='passed', work=str(work), cli_sha256=digest(args.cli.resolve()), compiled_aarch64_software=True, - script_bundle=True, elf_in_any_bundle_and_escaping_source_symlink_rejected=True, shared_and_separate_payload_partitions=True, - repeat_image_identical=True, independent_gpt_xz_tar_checks=True, + void_source_packages=True, installed_runtime_dependencies=True, wrong_elf_and_escaping_symlink_rejected=True, shared_and_separate_payload_partitions=True, + repeat_image_identical=True, direct_installed_erofs_programs=True, exact_and_larger_target_readback=True, wrong_confirmation_unchanged=True, changed_source_unchanged_target=True, stale_target_preview_rejected=True, - corrupted_gpt_rejected=True, corrupt_archive_and_catalogue_rejected=True, physical_usb_write='not performed') + corrupted_gpt_rejected=True, corrupt_program_and_catalogue_rejected=True, physical_usb_write='not performed') (work / 'acceptance.json').write_text(json.dumps(record, indent=2) + '\n') (project / 'out/workstation-images-current.txt').write_text(str(work) + '\n') print('PASS: workstation software/image/write acceptance:', work) diff --git a/tools/bootstrap-host b/tools/bootstrap-host index eaedc03..04020c1 100755 --- a/tools/bootstrap-host +++ b/tools/bootstrap-host @@ -3,7 +3,7 @@ source "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" [[ $# == 0 || ( $# == 1 && $1 == --install-deps ) ]] || die 'Usage: tools/bootstrap-host [--install-deps]' [[ $(uname -s) == Linux && $(uname -m) == x86_64 ]] || die 'M0 requires an x86_64 Linux build host' source /etc/os-release -[[ $ID == arch ]] || die 'M0 bootstrap supports Arch Linux; see docs/build-host.md' +[[ $ID == arch ]] || die 'M0 bootstrap supports Arch Linux; see docs/developer/build-host.md' (( EUID != 0 )) || die 'Run as a normal user, not root' [[ ! $FDS_ROOT =~ [[:space:]] ]] || die 'xbps-src requires a checkout path without whitespace' if [[ ${1:-} == --install-deps ]]; then diff --git a/tools/build-fds b/tools/build-fds index b2aab61..9951e32 100755 --- a/tools/build-fds +++ b/tools/build-fds @@ -4,16 +4,16 @@ source "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" cd "$FDS_ROOT" mkdir -p out/logs out/manifests tools/cargo-build --locked --offline --release --target aarch64-unknown-linux-musl \ - -p fds-cli -p fds-stage0 -p fds-boottrace -p fds-cartridged -p fds-burn -p fds-release -for binary in fds fds-stage0 fds-boottrace fds-cartridged fds-profile fds-burn fds-release; do + -p fds-cli -p fds-control -p fds-stage0 -p fds-boottrace -p fds-cartridged -p fds-burn -p fds-release +for binary in fds fds-control fds-program fds-stage0 fds-boottrace fds-cartridged fds-profile fds-burn fds-release; do tools/verify-elf "target/aarch64-unknown-linux-musl/release/$binary" aarch64 static cp "target/aarch64-unknown-linux-musl/release/$binary" "out/$binary" done cp out/fds out/fds-inspect cp out/fds out/fds-eject cp out/fds out/fds-power -sha256sum out/fds-release out/fds-burn out/fds-inspect out/fds-eject out/fds-power out/fds out/fds-stage0 out/fds-boottrace out/fds-cartridged out/fds-profile >out/manifests/fds-tools.sha256 -find rust/fds-software rust/fds-common rust/fds-cli rust/fds-stage0 rust/fds-boottrace rust/fds-cartridged rust/fds-burn rust/fds-release -type f -print0 | sort -z | \ +sha256sum out/fds-control out/fds-program out/fds-release out/fds-burn out/fds-inspect out/fds-eject out/fds-power out/fds out/fds-stage0 out/fds-boottrace out/fds-cartridged out/fds-profile >out/manifests/fds-tools.sha256 +find rust/fds-control rust/fds-software rust/fds-common rust/fds-cli rust/fds-stage0 rust/fds-boottrace rust/fds-cartridged rust/fds-burn rust/fds-release -type f -print0 | sort -z | \ xargs -0 sha256sum >out/manifests/fds-tools-inputs.sha256 sha256sum Cargo.toml Cargo.lock rust-toolchain.toml .cargo/config.toml tools/cargo-build tools/lib.sh tools/build-fds >>out/manifests/fds-tools-inputs.sha256 printf 'PASS: static ARM FDS CLI and stage0 tools built\n' diff --git a/tools/build-fds-package b/tools/build-fds-package index e426ff1..1f0db8b 100755 --- a/tools/build-fds-package +++ b/tools/build-fds-package @@ -7,11 +7,11 @@ tools/build-fds tools/prepare-void input="$FDS_VOID/hostdir/sources/fds-cli-0.1.0" mkdir -p "$input" out/packages -cp out/fds out/fds-boottrace out/fds-burn out/fds-inspect out/fds-eject out/fds-power out/fds-release LICENSE "$input/" +cp out/fds out/fds-control out/fds-program out/fds-boottrace out/fds-burn out/fds-inspect out/fds-eject out/fds-power out/fds-release LICENSE "$input/" notices=$(mktemp -d "$FDS_ROOT/out/rust-notices.XXXXXX") python3 tools/rust-notices "$notices/RUST-NOTICES.txt" cp "$notices/RUST-NOTICES.txt" "$input/" -(cd "$input" && sha256sum fds fds-boottrace fds-burn fds-inspect fds-eject fds-power fds-release LICENSE RUST-NOTICES.txt >SHA256SUMS) +(cd "$input" && sha256sum fds fds-control fds-program fds-boottrace fds-burn fds-inspect fds-eject fds-power fds-release LICENSE RUST-NOTICES.txt >SHA256SUMS) ( cd "$FDS_VOID" xbps_src -a aarch64 clean fds-cli diff --git a/tools/clean-builds b/tools/clean-builds new file mode 100755 index 0000000..bfbe65f --- /dev/null +++ b/tools/clean-builds @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +"""Remove obsolete FDS workspaces, retaining published outputs and saved inputs.""" +import argparse +import contextlib +import fcntl +import os +from pathlib import Path +import re +import shutil +import stat +import subprocess +import sys + + +# Only names produced by repository builders/tests belong here. Never sweep out/*: +# it also holds signed releases, frozen inputs, user cartridges and DATA overlays. +PREFIXES = ( + 'rootfs-build', 'kernel-build', 'system-build', 'initramfs-build', + 'boot-build', 'recovery-build', 'internal-build', + 'eeprom-production', 'eeprom-development', + 'cartridged-source', 'init-source', 'dasung-services', 'dasung-s6', + 'rust-notices', 'verify-package', 'dasung-check', 'dasung-s6-run', + 'm1-checks', 'm2-vm', 'm4-vm', 'm5-vm', 'm6-vm', 'm7-vm', 'm8-vm', + 'm9-images', 'm9-vm', 'm9-system', 'm10-vm', 'm11-vm', 'm11-faults', + 'm12-clock', 'm12-development', 'm12-eeprom', 'm12-internal', + 'm12-recovery', 'm12-release-contracts', 'm12-signing', + 'rust-profiles', 'workstation-images', 'emu-test', +) +GENERATED = re.compile(r'(?:' + '|'.join(PREFIXES) + r')\.[A-Za-z0-9_-]{6,8}') +POINTERS = ( + 'out/workstation-images-current.txt', + 'out/workstation-emulator-current.txt', + 'out/manifests/dasung-s6-database.txt', +) +LOCKS = ('.clean.lock', '.rootfs.lock', '.base-packages.lock', '.image-tools.lock') + + +def within(path, parent): + return path.is_relative_to(parent) + + +def workspace_for(project, path): + """Find the only possible candidate ancestor without scanning every run.""" + try: + parts = path.relative_to(project).parts + except ValueError: + return None + if parts and parts[0] == 'target': + return project / 'target' + if len(parts) >= 2 and parts[0] == 'out': + return project / 'out' / parts[1] + return None + + +def candidates(project): + out = project / 'out' + if out.is_symlink(): + raise ValueError('out is a symlink; refusing to clean an external output tree') + if not out.exists(): + entries = [] + else: + entries = sorted(out.iterdir()) + found = {p for p in entries if GENERATED.fullmatch(p.name) + and p.is_dir() and not p.is_symlink()} + target = project / 'target' + if target.is_dir() and not target.is_symlink(): + found.add(target) + protected = {} + + def retain(path, reason): + # Retain both a lexical target and its resolved target, including a link + # reached through a directory alias. Out-of-tree links never add candidates. + lexical = Path(os.path.abspath(path)) + for value in (lexical, lexical.resolve()): + candidate = workspace_for(project, value) + if candidate in found: + protected.setdefault(candidate, reason) + + # Git-tracked content is never build trash, even if named like a workspace. + tracked = subprocess.check_output( + ['git', '-C', str(project), 'ls-files', '-z', '--', 'out', 'target']) + for name in os.fsdecode(tracked).split('\0'): + if name: + retain(project / name, 'contains tracked content') + for path in found: + if os.path.lexists(path / '.fds-keep'): + protected[path] = '.fds-keep marker' + + manifests = out / 'manifests' + links = entries + (list(manifests.iterdir()) + if manifests.is_dir() and not manifests.is_symlink() else []) + for link in links: + if link.is_symlink(): + retain(link.parent / link.readlink(), f'published link {link.relative_to(project)}') + for name in POINTERS: + pointer = project / name + if pointer.is_file(): + value = pointer.read_text().strip() + if value: + retain(project / value, f'current pointer {name}') + + # These suites consume the newest image fixture by mtime, without a link. + image_runs = [p for p in found if p.name.startswith('m9-images.')] + if image_runs: + protected[max(image_runs, key=lambda p: p.stat().st_mtime_ns)] = 'latest media-image fixture' + + # A retained workspace can contain links to another generated workspace. + scanned = set() + while set(protected) - scanned: + for path in set(protected) - scanned: + scanned.add(path) + for directory, dirs, files in os.walk(path, followlinks=False): + for name in dirs + files: + link = Path(directory) / name + if link.is_symlink(): + retain(link.parent / link.readlink(), f'dependency of {path.name}') + return sorted(found - set(protected)), protected + + +def check_mounts(paths, mountinfo=None): + if mountinfo is None: + mountinfo = Path('/proc/self/mountinfo').read_text() + for line in mountinfo.splitlines(): + value = re.sub(r'\\([0-7]{3})', lambda m: chr(int(m[1], 8)), line.split()[4]) + mount = Path(value) + if any(within(mount, path) for path in paths): + raise ValueError(f'mounted path {mount}; unmount it before cleaning') + + +def check_processes(project, paths): + selected = set(paths) + ancestors = set() + pid = os.getpid() + while pid > 0: + ancestors.add(pid) + try: + status = Path(f'/proc/{pid}/status').read_text() + pid = int(re.search(r'^PPid:\s+(\d+)', status, re.M)[1]) + except FileNotFoundError: + break + for proc in Path('/proc').iterdir(): + if not proc.name.isdigit() or int(proc.name) in ancestors: + continue + command = '' + try: + if proc.stat().st_uid != os.getuid(): + continue + command = (proc / 'cmdline').read_bytes().replace(b'\0', b' ').decode(errors='replace') + cwd = (proc / 'cwd').resolve(strict=True) + # Catch a build before it has opened any candidate output files. + name = (proc / 'comm').read_text().strip() + building = (name in ('make', 'gmake', 'cargo', 'rustc', 'rustup', 'bwrap') + or name.startswith(('qemu-', 'xbps-')) + or re.search(r'(?:tools/|tests/integration/|image/build-)', command)) + if building and (within(cwd, project) or str(project) in command): + raise ValueError(f'active build/test process {proc.name} ({name}); stop it before cleaning') + for handle in [proc / 'cwd', proc / 'exe', *(proc / 'fd').iterdir()]: + try: + value = Path(os.readlink(handle)) + except FileNotFoundError: + continue + if workspace_for(project, value) in selected: + raise ValueError(f'process {proc.name} is using {value}; stop it before cleaning') + except (FileNotFoundError, ProcessLookupError): + continue + except PermissionError as error: + # Desktop session helpers may be nondumpable even for the same UID. + # Refuse an inaccessible process naming this checkout; unrelated + # protected processes must not make cleanup permanently unusable. + if str(project) in command: + raise ValueError(f'cannot inspect project process {proc.name}: {error}') from error + + +@contextlib.contextmanager +def locks(project): + with contextlib.ExitStack() as stack: + out = project / 'out' + if out.is_symlink(): + raise ValueError('out is a symlink; refusing cleanup') + out.mkdir(exist_ok=True) + for name in LOCKS: + fd = os.open(out / name, os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600) + stream = stack.enter_context(os.fdopen(fd, 'w')) + try: + fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + raise ValueError(f'{name} is locked; another build or cleanup is running') from error + yield + + +def footprint(paths): + if not paths: + return 0 + result = subprocess.check_output(['du', '-sx', '-B1', '--', *map(str, paths)], text=True) + # One invocation counts hardlinks once. Reflinks and snapshots can still + # share physical extents, so this is not a promise of filesystem free space. + return sum(int(line.split('\t', 1)[0]) for line in result.splitlines()) + + +def remove_tree(path): + if path.is_symlink() or not path.is_dir(): + raise ValueError(f'workspace changed since preview: {path}') + # Some guest directories are deliberately read-only. Change only directory + # permissions; file modes may belong to hardlinks in retained rootfs trees. + def walk_error(error): + raise error + os.chmod(path, path.stat().st_mode | 0o700, follow_symlinks=False) + for directory, dirs, _ in os.walk(path, topdown=True, followlinks=False, onerror=walk_error): + for item in [Path(directory), *(Path(directory) / name for name in dirs)]: + mode = item.lstat().st_mode + if stat.S_ISDIR(mode) and mode & 0o700 != 0o700: + os.chmod(item, mode | 0o700, follow_symlinks=False) + shutil.rmtree(path) + + +def clean(project, dry_run): + paths, protected = candidates(project) + identities = {path: (path.stat().st_dev, path.stat().st_ino) for path in paths} + check_mounts(paths) + if not dry_run: + check_processes(project, paths) + for path in paths: + print(f'{"WOULD REMOVE" if dry_run else "REMOVE"} {path.relative_to(project)}', flush=True) + amount = footprint(paths) + print(f'{len(paths)} disposable directories; allocated footprint {amount / 1024**3:.2f} GiB.', flush=True) + print('Physical space recovered can be smaller with hardlinks, reflinks or snapshots.', flush=True) + for path, reason in sorted(protected.items()): + print(f'KEEP {path.relative_to(project)} ({reason})', flush=True) + print('Saved releases/inputs/rebuild trees, unknown outputs, caches, out/logs and out/manifests are kept.', flush=True) + if dry_run: + print('Preview only. Run make clean to apply.', flush=True) + return + check_processes(project, paths) + check_mounts(paths) + if candidates(project)[0] != paths or any( + path.is_symlink() or (path.stat().st_dev, path.stat().st_ino) != identities[path] + for path in paths): + raise ValueError('output selection changed during inspection; rerun cleanup after stopping builds') + before = shutil.disk_usage(project).free + for path in paths: + remove_tree(path) + after = shutil.disk_usage(project).free + print(f'PASS: removed {len(paths)} disposable directories; ' + f'filesystem free-space change {(after - before) / 1024**3:+.2f} GiB.', flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--dry-run', action='store_true', help='list the exact cleanup selection without deleting anything') + args = parser.parse_args() + project = Path(__file__).resolve().parents[1] + try: + if args.dry_run: + clean(project, True) + else: + with locks(project): + clean(project, False) + except (OSError, ValueError, subprocess.CalledProcessError) as error: + parser.exit(1, f'ERROR: {error}\n') + + +if __name__ == '__main__': + main() diff --git a/tools/finalize-rootfs b/tools/finalize-rootfs index f713f62..2d3bba3 100755 --- a/tools/finalize-rootfs +++ b/tools/finalize-rootfs @@ -6,6 +6,11 @@ export LC_ALL=C sed -i 's/^#en_US.UTF-8 UTF-8[[:space:]]*$/en_US.UTF-8 UTF-8/' /etc/default/libc-locales grep -qx 'en_US.UTF-8 UTF-8' /etc/default/libc-locales xbps-reconfigure -fa +# Install FDS defaults globally so plain wmaker also uses the grayscale theme. +# Home-directory preferences continue to override these image defaults. +for name in WindowMaker WMRootMenu WMWindowAttributes; do + install -m644 "/usr/share/fds/eink/$name" "/etc/WindowMaker/$name" +done # The upstream CA package suppresses updater errors and exits successfully. # Require the actual generator to succeed instead of trusting that wrapper. update-ca-certificates --fresh diff --git a/tools/lib.sh b/tools/lib.sh index f36af26..c5e0336 100644 --- a/tools/lib.sh +++ b/tools/lib.sh @@ -7,7 +7,7 @@ FDS_VOID="$FDS_ROOT/vendor/void-packages" FDS_XBPS="$FDS_ROOT/.host/xbps/usr/bin" die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } -need() { command -v "$1" >/dev/null || die "Missing host command: $1 (see docs/build-host.md)"; } +need() { command -v "$1" >/dev/null || die "Missing host command: $1 (see docs/developer/build-host.md)"; } check_void_pin() { local pin actual diff --git a/tools/release-readme.md b/tools/release-readme.md index 89c992f..eafb73f 100644 --- a/tools/release-readme.md +++ b/tools/release-readme.md @@ -75,5 +75,5 @@ Use new directories and paths without whitespace. The restored build has its recorded modes preserved by `--same-permissions`; `--no-same-owner` keeps the files owned by the ordinary build user. Its compiler and package environment use their own caches and have no network access. Host prerequisites and additional comparison -instructions are in [Offline rebuilds](source/docs/reproducible-builds.md). +instructions are in [Offline rebuilds](source/docs/developer/reproducible-builds.md). The private signing key is not included in this release or its input archive. diff --git a/tools/rootfs-audit b/tools/rootfs-audit index 28b677b..d1dde68 100755 --- a/tools/rootfs-audit +++ b/tools/rootfs-audit @@ -40,6 +40,11 @@ def audit(root, configured=True): assert profile in ("cli", "development", "recovery"), "Unknown image profile" for optional in ("desktop", "xserver", "desktop-session", "network", "dhcp"): assert not (root / f"etc/s6-rc/source/boot/contents.d/{optional}").exists(), "Optional service in boot bundle" + if configured: + for name in ("WindowMaker", "WMRootMenu", "WMWindowAttributes"): + assert (root / "etc/WindowMaker" / name).read_bytes() == (root / "usr/share/fds/eink" / name).read_bytes(), "FDS grayscale defaults must be global" + for name in ("fds-program", "fds-control"): + assert (root / "usr/bin" / name).is_file(), f"Missing FDS interface: {name}" if profile == "development": assert "xorg-server-xvfb" in db and "xdotool" in db, "Missing development display diagnostics" for package in "gcc glibc-devel make cmake meson ninja pkg-config rust cargo git gdb strace vim".split(): diff --git a/tools/rootfs-runtime-check b/tools/rootfs-runtime-check index 35ed3b8..9a81de4 100755 --- a/tools/rootfs-runtime-check +++ b/tools/rootfs-runtime-check @@ -10,7 +10,7 @@ root=$(realpath -e -- "$1") "$FDS_ROOT/tools/verify-elf" "$root/usr/bin/fds-boottrace" aarch64 static "$FDS_ROOT/tools/verify-elf" "$root/usr/bin/fds-cartridged" aarch64 static "$FDS_ROOT/tools/verify-elf" "$root/usr/bin/fds-profile" aarch64 static -for binary in fds-burn fds-inspect fds-eject fds-power fds-release; do +for binary in fds-control fds-program fds-burn fds-inspect fds-eject fds-power fds-release; do "$FDS_ROOT/tools/verify-elf" "$root/usr/bin/$binary" aarch64 static done "$FDS_ROOT/tools/in-rootfs" "$root" --direct /usr/bin/xbps-pkgdb -a diff --git a/tools/run-system-vm b/tools/run-system-vm index 7f18161..fce160a 100755 --- a/tools/run-system-vm +++ b/tools/run-system-vm @@ -3,7 +3,7 @@ source "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" [[ $# == 0 ]] || die 'Usage: tools/run-system-vm' cd "$FDS_ROOT" for file in out/kernel/boot/kernel_2712.img out/fds-initramfs.img out/fds-system-cli.img; do - [[ -s $file ]] || die "Build required input first: $file (see docs/boot.md)" + [[ -s $file ]] || die "Build required input first: $file (see docs/developer/boot.md)" done printf 'FDS user console VM. Press Ctrl-a, then x to close it. Temporary home is discarded.\n' exec tools/in-void qemu-system-aarch64 \