From 6bfcce80706371f827fedd8766b71bdc12d1c8e2 Mon Sep 17 00:00:00 2001 From: Felis Date: Wed, 23 Sep 2026 03:12:45 +0800 Subject: [PATCH] FDS/OS 1.0 fixes --- AGENTS.md | 3 +- docs/cleanup.md | 3 +- docs/developer/README.md | 2 + docs/developer/build-host.md | 22 ++-- docs/developer/master-plan.md | 5 +- docs/developer/reproducible-builds.md | 7 ++ docs/developer/troubleshooting.md | 6 +- docs/developer/void-workspace.md | 77 +++++++++++++ docs/getting-started.md | 3 + docs/software-format.md | 5 +- docs/workstation.md | 8 +- rust/fds-workstation/src/software.rs | 14 ++- tests/integration/clean-checks.py | 3 +- tests/integration/m0-checks | 48 +++++++-- tests/integration/m12-release-build.py | 42 ++++++++ tests/integration/workstation-emulator.py | 10 ++ tests/integration/workstation-images.py | 14 ++- tools/bootstrap-host | 6 +- tools/frozen-inputs | 14 ++- tools/lib.sh | 21 ++-- tools/prepare-void | 3 +- tools/prepare-void-workspace | 125 ++++++++++++++++++++++ tools/smoke-test | 2 +- 23 files changed, 403 insertions(+), 40 deletions(-) create mode 100644 docs/developer/void-workspace.md create mode 100755 tools/prepare-void-workspace diff --git a/AGENTS.md b/AGENTS.md index 67dc5be..b4db0c0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,8 @@ claim Pi boot or physical power-cycle recovery is verified. commands and options with typed parsers; do not hand-parse argument vectors. - No systemd, runit, BusyBox, shell PID1, boot-time cache compilation, or sleeps to hide races in the target system. -- Keep upstream Void files unchanged; pin its submodule and use `packages/` overlays. +- Keep `vendor/void-packages` clean and pinned. Use `.host/void-packages` for + generated build state; keep authoritative FDS overlays in `packages/`. - Explain every new dependency and keep documentation accurate. - Run `make bootstrap`, `make smoke-test`, and `make check` for M0 build changes. Report skips and missing hardware honestly. Do not invent boot benchmarks. diff --git a/docs/cleanup.md b/docs/cleanup.md index 42affff..bf36657 100644 --- a/docs/cleanup.md +++ b/docs/cleanup.md @@ -44,7 +44,8 @@ Unknown names are left alone, including ad hoc diagnostic experiments. - User-created cartridge images, installed software trees and personal emulator sessions such as `out/my-emulator/`, including their persistent DATA overlays. - `out/logs/`, `out/manifests/`, packages, downloads, `out/cache/`, `.host/`, the - Void checkout/build container, and Cargo/Rustup caches outside this checkout. + writable Void checkout/build container under `.host/void-packages/`, and + Cargo/Rustup caches outside this checkout. - Git-tracked files and any recognized workspace containing a `.fds-keep` entry. To retain an older generated workspace for investigation, put a marker in it diff --git a/docs/developer/README.md b/docs/developer/README.md index 40ef332..198bd44 100644 --- a/docs/developer/README.md +++ b/docs/developer/README.md @@ -10,6 +10,8 @@ the inputs named in those reports; a changed source tree needs fresh checks. ## Design and build references +- [Clean upstream and writable Void workspace](void-workspace.md) + - [Master plan](master-plan.md) - [Build host and dependencies](build-host.md) - [Development workflow](development.md) diff --git a/docs/developer/build-host.md b/docs/developer/build-host.md index a3bab75..18fbaa0 100644 --- a/docs/developer/build-host.md +++ b/docs/developer/build-host.md @@ -112,7 +112,7 @@ equivalent of: ```sh export PATH="$PWD/.host/xbps/usr/bin:$PATH" export XBPS_ARCH=x86_64 -cd vendor/void-packages +cd .host/void-packages ./xbps-src -A x86_64 binary-bootstrap ``` @@ -131,7 +131,7 @@ To reproduce the two builds individually from the repository root: ``` The latter command sets the local XBPS PATH, checks the Void pin, and executes -this command within the submodule: +this command within the writable build checkout: ```sh ./xbps-src -a aarch64 pkg hello @@ -186,9 +186,11 @@ tests accumulate. [Cleanup](cleanup.md) explains the retained current images, release/input archives and caches. Full image builds and VM suites need tens of GiB per working tree, beyond the initial M0 smoke test's requirements. +- `vendor/void-packages/`: clean, pinned upstream submodule; no generated overlays. +- `.host/void-packages/`: independent local build checkout at the same pin. - `.host/xbps/`: project-local host tools, not installed into `/usr`. -- `vendor/void-packages/masterdir-x86_64/`: glibc build container. -- `vendor/void-packages/hostdir/`: download caches, cross packages, and build output. +- `.host/void-packages/masterdir-x86_64/`: glibc build container. +- `.host/void-packages/hostdir/`: download caches, cross packages, and build output. - `out/packages/`: exported aarch64 test package and its repository index. - `out/manifests/void-build-packages.txt`: packages remaining installed after the build. Cross toolchains and temporary dependencies removed by xbps-src remain @@ -197,12 +199,20 @@ GiB per working tree, beyond the initial M0 smoke test's requirements. - `out/manifests/artifacts.sha256`: validation artifact digests. - `out/logs/`: actual command logs, including logs from failed attempts. -Place FDS package overlays at `packages/fds-*/template`. `tools/prepare-void` -copies only directories containing a template and refuses to replace tracked +Place FDS package overlays at `packages/fds-*/template`. `tools/prepare-void-workspace` +clones the pinned upstream locally without shared Git objects or network access. +`tools/prepare-void` copies directories containing a template into that writable checkout and refuses to replace tracked upstream packages. These copies are visible inside the build container. Base, base-files, init, CLI, kernel and Dasung overlays are active; future directories without templates remain inactive. +Bootstrap migrates matching old generated copies, configuration, `hostdir` and +`masterdir-*` out of the submodule using renames, preserving cached downloads +and container files. Unknown files, staged edits, changed copies or destination +conflicts stop migration for explicit reconciliation. Python 3 is used for +checked migration and locking; it is already a documented FDS host prerequisite. +The workstation builder also rejects submodules as writable build targets. + If an overlay changes and preparation reports a stale copy, inspect and preserve any unique edits before moving the specific generated copy out of `srcpkgs/` and retrying. Modified upstream tracked files cause immediate failure. Conflicting diff --git a/docs/developer/master-plan.md b/docs/developer/master-plan.md index 7765497..4df0090 100644 --- a/docs/developer/master-plan.md +++ b/docs/developer/master-plan.md @@ -453,7 +453,10 @@ Record: VOID_PACKAGES_COMMIT ``` -Inject FDS packages through an overlay. +Keep this submodule clean. Prepare a writable checkout at `.host/void-packages` +from the same pinned commit. Inject FDS packages from `packages/` into that +build checkout; keep generated sources, caches and build containers outside +the submodule. Required relationship: diff --git a/docs/developer/reproducible-builds.md b/docs/developer/reproducible-builds.md index f23a17b..3809233 100644 --- a/docs/developer/reproducible-builds.md +++ b/docs/developer/reproducible-builds.md @@ -16,6 +16,13 @@ Normal builds still use Void's signed rolling repositories. A frozen build uses the selected package files and build environment from one explicit snapshot. It does not silently consult a newer repository if an input is missing. +Current builds keep the upstream reference in `vendor/void-packages/` and use +`.host/void-packages/` for overlays and cached build state. New snapshots and +restores use this separate layout. Restoring an older +snapshot retains its historical layout and saved build scripts; its files and +acceptance are not rewritten. See [the workspace change](void-workspace.md) for +fresh validation of this migration. + ## What is frozen `tools/frozen-inputs create NEW_DIRECTORY` records: diff --git a/docs/developer/troubleshooting.md b/docs/developer/troubleshooting.md index 9aaf9c4..f15f59c 100644 --- a/docs/developer/troubleshooting.md +++ b/docs/developer/troubleshooting.md @@ -94,14 +94,14 @@ normal. Never use a broad reset or clean command without checking what it remove For `etc/conf differs from config/xbps-src.conf`, compare the two: ```sh -diff -u vendor/void-packages/etc/conf config/xbps-src.conf +diff -u .host/void-packages/etc/conf config/xbps-src.conf ``` Put the desired settings in `config/xbps-src.conf`, then follow [the explicit synchronization procedure](development.md#change-build-configuration-deliberately). A `diff` exit status of 1 means differences were found, not that the comparison failed. -For `Stale overlay`, compare the named `vendor/void-packages/srcpkgs/fds-*` +For `Stale overlay`, compare the named `.host/void-packages/srcpkgs/fds-*` directory with its source under `packages/`. Preserve any unique edits in the source first. Move that specific generated copy into a backup directory under `out/`, outside `srcpkgs/`, then run `./tools/prepare-void` again. Do not move or @@ -121,7 +121,7 @@ space before assuming that a quiet download is a deadlock: ```sh tail -n 40 out/logs/bootstrap.log tail -n 40 out/logs/xbps-hello.log -du -sh out/downloads vendor/void-packages/hostdir +du -sh out/downloads .host/void-packages/hostdir df -h . ``` diff --git a/docs/developer/void-workspace.md b/docs/developer/void-workspace.md new file mode 100644 index 0000000..aa4b379 --- /dev/null +++ b/docs/developer/void-workspace.md @@ -0,0 +1,77 @@ +# Clean upstream and writable Void workspace + +FDS keeps upstream source separate from its build state: + +| Path | Ownership and purpose | +| --- | --- | +| `vendor/void-packages/` | Clean upstream Git submodule at `VOID_PACKAGES_COMMIT` | +| `packages/` | Authoritative FDS base-package sources, including Dasung | +| `examples/software/*/void/` | Authoritative example PROGRAM package sources | +| `.host/void-packages/` | Generated writable checkout, package copies, local configuration, caches and build container | + +`make bootstrap` creates the build checkout locally at the pinned commit. It +copies Git objects independently; there is no network fetch for this step, no +Git alternates dependency and no hardlinks into the upstream source. Normal +package bootstrap still uses its configured repositories. + +`tools/prepare-void` copies FDS package overlays into the writable checkout and +retains the existing checks against replacing upstream packages or overwriting +changed generated copies. Workstation builds default to the same checkout. +Explicitly selecting a Git submodule with `--void-packages` is rejected before +source files or build state can be written there. + +The workspace helper migrates old, matching generated files and build caches by +renaming them. It compares every file, mode and symlink in each known source +copy first. Unknown upstream files, staged changes, independent edits and +conflicting destinations stop migration. Run builds and migration sequentially. +The workspace lock prevents two preparations from changing the layout together. +Python is already a build-host prerequisite; this introduces no new dependency. + +For this checkout, the 58 staged additions matched the main-repository sources. +Their local commit `47580322e801ef195c9bf279fd1fe4d8a1f723e9` and both staged +patches were saved under `.host/backups/void-overlays-20260922-134611/` before +unstaging those copies. The parent gitlink was restored to upstream commit +`02a3cbc132c3c4a3a9d59e9b98f517af5dd11cd1`, matching the declared pin. All eleven +package directories, `hostdir`, `masterdir-x86_64` and `etc/conf` were moved into +the build checkout. No upstream tracked source was changed. + +An explicit future pin update advances the writable checkout without replacing +its caches. Git refuses updates that would overwrite local files. Continue to +record the parent gitlink and `VOID_PACKAGES_COMMIT` together. + +Frozen-input creation uses the separate build checkout. New restores recreate +that layout. Older saved projects retain their +original layout and scripts; the frozen 0.1.0 release is untouched. Restore +compatibility tests do not imply a new reproducibility or physical-hardware claim. + +## Validation + +Fresh checks for this change passed: + +- `make bootstrap`, `make smoke-test`, and `make check`: migrated container use, + ARM/glibc package compilation, static-musl compilation, clean upstream state, + exact overlay/cache preservation, shallow pin updates, rejection of independent + edits and source conflicts, and cleanup protections. +- `make workstation-test`: `out/workstation-images.ckkodex6`; real Void source + builds from `.host/void-packages`, image validation and confirmed file write/readback. + Selecting the upstream submodule as a build target is rejected without writes. +- `make emulator-test`: `out/emu-test.op5azhlm`; full twelve-bay PROGRAM/DATA, + foreground terminal, eject/unplug, legacy reading and service-restart checks. +- `make release-contract-test`: `out/m12-release-contracts.wmsfrcl0`; both separate + and legacy frozen-layout restores, retained source caches, local offline + repositories and existing tamper/release-comparison checks. +- Documentation links, formatting, Python syntax and a final empty upstream Git + status, including ignored files, were checked. + +Logs are in `out/logs/void-workspace/`; source and log hashes, pins, native binary +hashes and fixture results are recorded in `out/manifests/void-workspace-acceptance.json`. +The optional host-native ARM execution shortcut was skipped in smoke-test. ARM +execution was exercised by the full emulator suite. Physical hardware was not used. + +The first emulator attempt (`out/emu-test.463emq8u`) failed on an empty cartridge +IPC response during initial discovery in its admin VM. The complete rerun passed +without target-system changes. SYSTEM, kernel and initramfs hashes matched the +previous accepted inputs. The cause of that first response failure was not +established; its command/serial logs are retained. Future failures now collect +live service diagnostics before VM shutdown, without retrying failed commands +or relaxing their acceptance conditions. diff --git a/docs/getting-started.md b/docs/getting-started.md index 0101954..ef2dbb2 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -34,6 +34,9 @@ make check ``` Bootstrap prepares pinned Rust and XBPS tools and the Void build container. +The upstream reference stays clean in `vendor/void-packages/`. Generated package +copies, build configuration and caches live in `.host/void-packages/`; edit +FDS package sources in `packages/`, not in either generated checkout. The smoke test checks the AArch64 build toolchains; `make check` validates the build guardrails. Stop and resolve any failed command before continuing. Logs are in `out/logs/`. Repeated preparation reuses verified downloads and caches. diff --git a/docs/software-format.md b/docs/software-format.md index 8dc864a..aaf1fc1 100644 --- a/docs/software-format.md +++ b/docs/software-format.md @@ -26,7 +26,10 @@ normal Void `template` and any `files/` or `patches/` directories it uses. Paths are relative to this recipe. The [included hello package](../examples/software/hello/void/template) is a complete example. -Custom sources are copied into an untracked `srcpkgs/PACKAGE` directory. Existing +Custom sources are copied into `srcpkgs/PACKAGE` inside the selected writable +build checkout. FDS uses `.host/void-packages/`; its `vendor/void-packages/` +submodule remains unchanged. The builder rejects a Git submodule as its build +checkout. Existing tracked Void sources are never overwritten. If a generated source copy differs, inspect it, preserve any independent edits, then remove that generated copy and retry. An unchanged copy can be reused. diff --git a/docs/workstation.md b/docs/workstation.md index 017d4b3..64fdae2 100644 --- a/docs/workstation.md +++ b/docs/workstation.md @@ -18,7 +18,7 @@ export PATH="$PWD/out/workstation:$PATH" | Prerequisite | Used for | | --- | --- | -| Prepared Void `void-packages` checkout and native XBPS utilities | Build source packages and install their AArch64 runtime dependencies | +| Prepared writable Void `void-packages` checkout and native XBPS utilities | Build source packages and install their AArch64 runtime dependencies | | `mkfs.erofs`, `fsck.erofs` | Create and inspect read-only payload filesystems | | `bwrap` | Inspect filesystems and install package trees in a user namespace | | `xz` | Read older software cartridges and package/source archives | @@ -31,12 +31,14 @@ share the same Void checkout. Prepare a Void source checkout using its [upstream instructions](https://github.com/void-linux/void-packages/blob/02a3cbc132c3c4a3a9d59e9b98f517af5dd11cd1/README.md). Complete `xbps-src binary-bootstrap` for the workstation architecture before -using it. The workstation itself need not run Void. Select that checkout with +using it. Keep this writable checkout outside Git submodules. The workstation itself need not run Void. Select that checkout with `--void-packages /path/to/void-packages`; native `xbps-*` tools must be in PATH, or selected with `--xbps-bin /path/to/bin`. Builds target AArch64/glibc. For the complete FDS build workstation, [bootstrap](getting-started.md) already -prepares `vendor/void-packages` and `.host/xbps/usr/bin`. Reuse its local tools: +keeps `vendor/void-packages` as a clean upstream reference and prepares the +writable `.host/void-packages` checkout plus `.host/xbps/usr/bin`. The native +builder defaults to `.host/void-packages`. Reuse its local tools: ```sh ./tools/prepare-image-tools diff --git a/rust/fds-workstation/src/software.rs b/rust/fds-workstation/src/software.rs index c796dd7..93561b6 100644 --- a/rust/fds-workstation/src/software.rs +++ b/rust/fds-workstation/src/software.rs @@ -13,8 +13,8 @@ use std::{ #[derive(clap::Args, Debug, Clone)] pub struct BuildOptions { - /// Prepared Void source checkout used by xbps-src (not the workstation OS). - #[arg(long, global = true, default_value = "vendor/void-packages")] + /// Writable Void build checkout; keep it outside Git submodules. + #[arg(long, global = true, default_value = ".host/void-packages")] pub void_packages: PathBuf, /// Optional rootless XBPS wrapper, for example this checkout's tools/in-void. #[arg(long, global = true)] @@ -81,6 +81,16 @@ pub fn build(recipe: &Path, output: &Path, options: &BuildOptions) -> Result"$fake/xbps-src" mkdir -p "$fake/srcpkgs/fds-upstream" "$fake/etc" "$scratch/repo/config" "$scratch/repo/packages/fds-fixture" printf '# upstream\n' >"$fake/srcpkgs/fds-upstream/template" -printf 'etc/conf\n' >"$fake/.gitignore" +printf 'etc/conf\nhostdir/\nmasterdir-*/\n' >"$fake/.gitignore" cp config/xbps-src.conf "$scratch/repo/config/" printf '# inert overlay fixture\n' >"$scratch/repo/packages/fds-fixture/template" git -C "$fake" add . git -C "$fake" -c user.name=FDS -c user.email=test@example.invalid commit -qm fixture git -C "$fake" rev-parse HEAD >"$scratch/repo/VOID_PACKAGES_COMMIT" -bash -c 'source "$1"; check_void_pin' _ "$scratch/repo/tools/lib.sh" +bash -c 'source "$1"; check_void_source' _ "$scratch/repo/tools/lib.sh" printf 'PASS: accepts exact clean Void commit\n' +# Existing generated files and caches move without copying their payloads. +cp -a "$scratch/repo/packages/fds-fixture" "$fake/srcpkgs/" +cp config/xbps-src.conf "$fake/etc/conf" +mkdir -p "$fake/hostdir" "$fake/masterdir-x86_64" +printf 'cached input\n' >"$fake/hostdir/cache" +cache_inode=$(stat -c %i "$fake/hostdir/cache") "$scratch/repo/tools/prepare-void" -cmp "$scratch/repo/packages/fds-fixture/template" "$fake/srcpkgs/fds-fixture/template" +build="$scratch/repo/.host/void-packages" +[[ -z $(git -C "$fake" status --porcelain --untracked-files=all) ]] || die 'Migration dirtied upstream' +[[ ! -e $fake/hostdir && ! -e $fake/etc/conf && ! -e $fake/srcpkgs/fds-fixture ]] || die 'Migration left build state in upstream' +[[ $(stat -c %i "$build/hostdir/cache") == "$cache_inode" ]] || die 'Migration copied cache data' +[[ ! -f $build/.git/objects/info/alternates ]] || die 'Build checkout depends on upstream object storage' +bash -c 'source "$1"; check_void_pin' _ "$scratch/repo/tools/lib.sh" +cmp "$scratch/repo/packages/fds-fixture/template" "$build/srcpkgs/fds-fixture/template" "$scratch/repo/tools/prepare-void" -printf 'PASS: overlay copied and repeat preparation is idempotent\n' +printf 'PASS: overlays and caches moved outside clean upstream; preparation is idempotent\n' printf '# changed\n' >>"$scratch/repo/packages/fds-fixture/template" expect_failure 'stale generated overlay' "$scratch/repo/tools/prepare-void" -cp "$fake/srcpkgs/fds-fixture/template" "$scratch/repo/packages/fds-fixture/template" +cp "$build/srcpkgs/fds-fixture/template" "$scratch/repo/packages/fds-fixture/template" +# Preserve unexpected upstream additions and independently edited legacy copies. +printf 'personal source\n' >"$fake/personal-file" +expect_failure 'unknown upstream addition' "$scratch/repo/tools/prepare-void" +[[ -f $fake/personal-file ]] || die 'Unknown file was removed' +rm "$fake/personal-file" +cp -a "$scratch/repo/packages/fds-fixture" "$fake/srcpkgs/" +printf '# independent edit\n' >>"$fake/srcpkgs/fds-fixture/template" +expect_failure 'edited legacy overlay' "$scratch/repo/tools/prepare-void" +grep -q 'independent edit' "$fake/srcpkgs/fds-fixture/template" +rm -r "$fake/srcpkgs/fds-fixture" +# An explicit upstream pin update advances the build checkout and keeps caches. +printf 'new upstream revision\n' >"$fake/README" +git -C "$fake" add README +git -C "$fake" -c user.name=FDS -c user.email=test@example.invalid commit -qm update +git -C "$fake" rev-parse HEAD >"$scratch/repo/VOID_PACKAGES_COMMIT" +# Production initializes upstream with depth 1; updating that boundary must work. +git -C "$fake" rev-parse HEAD >"$fake/.git/shallow" +expect_failure 'outdated build checkout' bash -c 'source "$1"; check_void_pin' _ "$scratch/repo/tools/lib.sh" +"$scratch/repo/tools/prepare-void" +[[ $(stat -c %i "$build/hostdir/cache") == "$cache_inode" ]] || die 'Pin update replaced cache' +[[ $(git -C "$build" rev-parse HEAD) == "$(cat "$scratch/repo/VOID_PACKAGES_COMMIT")" ]] || die 'Build pin not advanced' mkdir -p "$scratch/repo/packages/fds-upstream" printf '# collision\n' >"$scratch/repo/packages/fds-upstream/template" expect_failure 'overlay replacing upstream package' "$scratch/repo/tools/prepare-void" -printf '# changed config\n' >>"$fake/etc/conf" +rm -r "$scratch/repo/packages/fds-upstream" +printf '# changed config\n' >>"$build/etc/conf" expect_failure 'overwriting local xbps configuration' "$scratch/repo/tools/prepare-void" printf '\n# dirty\n' >>"$fake/xbps-src" expect_failure 'modified Void source' bash -c 'source "$1"; check_void_pin' _ "$scratch/repo/tools/lib.sh" diff --git a/tests/integration/m12-release-build.py b/tests/integration/m12-release-build.py index bc857e3..e281364 100644 --- a/tests/integration/m12-release-build.py +++ b/tests/integration/m12-release-build.py @@ -131,4 +131,46 @@ command[-1] = linked result = invoke(*command, ok=False) assert 'not a symlink' in result.stderr print(f'PASS: release output preservation and private-key symlink rejection: {work}') + +# Restore small real Git bundles through both workspace layouts. These fixtures +# exercise path selection and cache preservation, not an operating-system build. +upstream = work / 'upstream' +upstream.mkdir() +invoke('git', 'init', '-q', upstream) +(upstream / 'xbps-src').write_text('# inert source fixture\n') +(upstream / '.gitignore').write_text('hostdir/\nmasterdir-*/\netc/conf\netc/xbps.d/custom/\n') +invoke('git', '-C', upstream, 'add', '.') +invoke('git', '-C', upstream, '-c', 'user.name=FDS', '-c', 'user.email=test@example.invalid', 'commit', '-qm', 'fixture') +pin = invoke('git', '-C', upstream, 'rev-parse', 'HEAD').stdout.strip() +epoch = int(invoke('git', '-C', upstream, 'show', '-s', '--format=%ct', 'HEAD').stdout) +for layout in ('separate', 'legacy'): + fixture = work / ('restore-input-' + layout) + (fixture / 'project/tools').mkdir(parents=True) + (fixture / 'project/config').mkdir() + (fixture / 'project/VOID_PACKAGES_COMMIT').write_text(pin + '\n') + (fixture / 'project/config/xbps-src.conf').write_text('# fixture\n') + if layout == 'separate': + shutil.copy2(project / 'tools/prepare-void-workspace', fixture / 'project/tools/prepare-void-workspace') + invoke('git', '-C', upstream, 'bundle', 'create', fixture / 'void.bundle', 'HEAD') + for name in ('masterdir/etc/xbps.d', 'sources', 'repositories/build', + 'repositories/cli', 'repositories/development', 'repositories/recovery', + 'xbps', 'image-tools', 'eeprom', 'cache', 'downloads', 'cargo-vendor', 'rust-toolchain'): + (fixture / name).mkdir(parents=True, exist_ok=True) + (fixture / 'sources/retained.tar.gz').write_bytes(b'cached source fixture') + restore_lock = dict(lock, void_commit=pin, source_epoch=epoch) + restore_lock['files'] = frozen.inventory(fixture) + restore_lock['source_sha256'] = frozen.source_digest(restore_lock['files']) + (fixture / 'lock.json').write_text(json.dumps(restore_lock)) + destination = work / ('restored-' + layout) + invoke(verify, 'restore', fixture, destination) + build = destination / ('.host/void-packages' if layout == 'separate' else 'vendor/void-packages') + assert (build / 'hostdir/sources/retained.tar.gz').read_bytes() == b'cached source fixture' + assert (build / 'masterdir-x86_64/etc/xbps.d/05-fds-frozen-local.conf').read_text() == 'repository=/host/frozen-repository\n' + assert invoke('git', '-C', build, 'rev-parse', 'HEAD').stdout.strip() == pin + if layout == 'separate': + reference = destination / 'vendor/void-packages' + assert not (reference / 'hostdir').exists() + assert not invoke('git', '-C', reference, 'status', '--porcelain', '--untracked-files=all').stdout.strip() + invoke(verify, 'verify', fixture) +print('PASS: separate-workspace and legacy frozen restores retain source caches and offline repositories', flush=True) print('NOTE: these are small contract fixtures; actual source-to-image offline reproduction remains a separate gate') diff --git a/tests/integration/workstation-emulator.py b/tests/integration/workstation-emulator.py index 3a2fa2d..fc67ea5 100644 --- a/tests/integration/workstation-emulator.py +++ b/tests/integration/workstation-emulator.py @@ -39,6 +39,16 @@ def invoke(*command, ok=True): result = subprocess.run([cli, '--session', str(session), *map(str, command)], capture_output=True, text=True, timeout=240) log.write(repr(command) + '\n' + result.stdout + result.stderr) log.flush() + if (result.returncode == 0) != ok and command and command[0] == 'guest': + # Preserve live service evidence before the finally block stops the VM. + # Do not retry or relax the failed command's acceptance condition. + diagnostics = subprocess.run([ + cli, '--session', str(session), 'guest', '--', 'sh', '-c', + 'cat /run/log/cartridged/current; s6-svstat /run/service/cartridged; ' + 'dmesg | tail -40', + ], capture_output=True, text=True, timeout=60) + log.write('Failure diagnostics:\n' + diagnostics.stdout + diagnostics.stderr) + log.flush() assert (result.returncode == 0) == ok, (command, result.returncode, result.stdout, result.stderr) return result.stdout diff --git a/tests/integration/workstation-images.py b/tests/integration/workstation-images.py index 219351b..0fd9c5b 100644 --- a/tests/integration/workstation-images.py +++ b/tests/integration/workstation-images.py @@ -14,7 +14,7 @@ from image_formats import gpt, LINUX_FILESYSTEM parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--cli', type=Path, required=True) parser.add_argument('--image-tool-runner', type=Path) -parser.add_argument('--void-packages', type=Path, default=project / 'vendor/void-packages') +parser.add_argument('--void-packages', type=Path, default=project / '.host/void-packages') parser.add_argument('--xbps-tool-runner', type=Path) parser.add_argument('--xbps-bin', type=Path) args = parser.parse_args() @@ -43,6 +43,15 @@ def digest(path): return hashlib.file_digest(stream, 'sha256').hexdigest() +# A user-provided submodule path must fail before copying any package sources. +upstream = project / 'vendor/void-packages' +assert not subprocess.check_output(['git', '-C', str(upstream), 'status', '--porcelain', '--untracked-files=all']) +rejected = subprocess.run([cli[0], '--void-packages', str(upstream), 'software', 'build', + str(project / 'examples/software/hello/software.toml'), str(work / 'forbidden-build')], + capture_output=True, text=True) +assert rejected.returncode != 0 and 'outside Git submodules' in rejected.stderr, rejected.stderr +assert not (work / 'forbidden-build').exists() + for name in ['hello', 'report']: recipe = project / f'examples/software/{name}/software.toml' if name == 'report': @@ -149,7 +158,8 @@ for case in ['program-corrupt', 'catalogue-mapping']: malformed = work / (case + '.img') gpt(malformed, [(part['name'], LINUX_FILESYSTEM, filesystem) for part, filesystem in zip(parts, selected)]) invoke(['inspect', malformed], False) -record = dict(status='passed', work=str(work), cli_sha256=digest(args.cli.resolve()), compiled_aarch64_software=True, +assert not subprocess.check_output(['git', '-C', str(upstream), 'status', '--porcelain', '--untracked-files=all']) +record = dict(source_checkout_clean=True, submodule_build_rejected=True, status='passed', work=str(work), cli_sha256=digest(args.cli.resolve()), compiled_aarch64_software=True, void_source_packages=True, installed_runtime_dependencies=True, wrong_elf_and_escaping_symlink_rejected=True, shared_and_separate_payload_partitions=True, repeat_image_identical=True, direct_installed_erofs_programs=True, exact_and_larger_target_readback=True, wrong_confirmation_unchanged=True, diff --git a/tools/bootstrap-host b/tools/bootstrap-host index 04020c1..685d7df 100755 --- a/tools/bootstrap-host +++ b/tools/bootstrap-host @@ -8,10 +8,10 @@ source /etc/os-release [[ ! $FDS_ROOT =~ [[:space:]] ]] || die 'xbps-src requires a checkout path without whitespace' if [[ ${1:-} == --install-deps ]]; then sudo pacman -S --needed bash coreutils binutils git curl make file tar xz gzip zstd \ - bubblewrap rustup ca-certificates findutils diffutils grep sed gawk util-linux + bubblewrap rustup ca-certificates findutils diffutils grep sed gawk util-linux python fi for cmd in bash git curl make file readelf ldd tar xz gzip zstd sha256sum bwrap rustup \ - find diff grep sed awk flock install; do + find diff grep sed awk flock install python3; do need "$cmd" done # Fail before downloading if unprivileged build containers cannot run. @@ -21,7 +21,7 @@ mkdir -p out/downloads out/logs .host if [[ ${FDS_OFFLINE:-0} != 1 ]]; then git -c http.lowSpeedLimit=1000 -c http.lowSpeedTime=60 submodule update --init --depth 1 vendor/void-packages fi -check_void_pin +check_void_source source config/host-tools.conf archive="out/downloads/${XBPS_STATIC_URL##*/}" if [[ ! -f $archive ]]; then diff --git a/tools/frozen-inputs b/tools/frozen-inputs index a0e9d5a..3354432 100755 --- a/tools/frozen-inputs +++ b/tools/frozen-inputs @@ -79,7 +79,13 @@ def check_pin(): if pin != actual: raise ValueError('Void checkout differs from its pin') run('git', '-C', void, 'diff', '--exit-code', 'HEAD', '--') - return void, pin + build = project / '.host/void-packages' + if subprocess.check_output(['git', '-C', str(build), 'rev-parse', 'HEAD'], text=True).strip() != pin: + raise ValueError('Void build checkout differs from its pin') + run('git', '-C', build, 'diff', '--exit-code', 'HEAD', '--') + if subprocess.check_output(['git', '-C', str(void), 'status', '--porcelain', '--untracked-files=all'], text=True).strip(): + raise ValueError('Void upstream submodule is dirty') + return build, pin def add_package(source, destination): @@ -245,6 +251,12 @@ def restore(source, destination): epoch = int(subprocess.check_output(['git', '-C', str(void), 'show', '-s', '--format=%ct', 'HEAD'])) if epoch != lock['source_epoch']: raise ValueError('Restored Void commit timestamp differs from the snapshot') + # Older frozen projects expect caches under vendor; their saved sources and + # release inputs remain unchanged. New projects use the separate workspace. + prepare = destination / 'tools/prepare-void-workspace' + if prepare.is_file(): + run(sys.executable, prepare) + void = destination / '.host/void-packages' copy(source / 'masterdir', void / 'masterdir-x86_64') copy(source / 'sources', void / 'hostdir/sources') copy(source / 'repositories/build', void / 'hostdir/frozen-repository') diff --git a/tools/lib.sh b/tools/lib.sh index c5e0336..f049b85 100644 --- a/tools/lib.sh +++ b/tools/lib.sh @@ -3,22 +3,31 @@ set -euo pipefail export LC_ALL=C FDS_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) -FDS_VOID="$FDS_ROOT/vendor/void-packages" +FDS_VOID_SOURCE="$FDS_ROOT/vendor/void-packages" +FDS_VOID="$FDS_ROOT/.host/void-packages" FDS_XBPS="$FDS_ROOT/.host/xbps/usr/bin" die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } need() { command -v "$1" >/dev/null || die "Missing host command: $1 (see docs/developer/build-host.md)"; } -check_void_pin() { +check_void_source() { local pin actual pin=$(cat "$FDS_ROOT/VOID_PACKAGES_COMMIT") [[ $pin =~ ^[0-9a-f]{40}$ ]] || die 'Invalid VOID_PACKAGES_COMMIT' - [[ -f $FDS_VOID/xbps-src ]] || die 'Void submodule is missing; run make bootstrap' - actual=$(git -C "$FDS_VOID" rev-parse HEAD) + [[ -f $FDS_VOID_SOURCE/xbps-src ]] || die 'Void submodule is missing; run make bootstrap' + actual=$(git -C "$FDS_VOID_SOURCE" rev-parse HEAD) [[ $actual == "$pin" ]] || die "Void checkout mismatch: expected $pin, found $actual" - git -C "$FDS_VOID" diff --quiet HEAD -- || die 'Tracked Void files were changed; keep FDS changes in packages/' + git -C "$FDS_VOID_SOURCE" diff --quiet HEAD -- || die 'Tracked Void files were changed; keep FDS changes in packages/' export SOURCE_DATE_EPOCH - SOURCE_DATE_EPOCH=$(git -C "$FDS_VOID" show -s --format=%ct HEAD) + SOURCE_DATE_EPOCH=$(git -C "$FDS_VOID_SOURCE" show -s --format=%ct HEAD) +} + +check_void_pin() { + check_void_source + [[ -z $(git -C "$FDS_VOID_SOURCE" status --porcelain --untracked-files=all) ]] || die 'Upstream Void submodule is dirty; run make bootstrap to migrate known build copies' + [[ -f $FDS_VOID/xbps-src ]] || die 'Void build checkout is missing; run make bootstrap' + [[ $(git -C "$FDS_VOID" rev-parse HEAD) == "$(cat "$FDS_ROOT/VOID_PACKAGES_COMMIT")" ]] || die 'Void build checkout pin changed; run make bootstrap' + git -C "$FDS_VOID" diff --quiet HEAD -- || die 'Tracked build-checkout files changed; keep FDS sources in packages/' } # Keep the upstream entry point unchanged. Offline builds retain normal diff --git a/tools/prepare-void b/tools/prepare-void index 603e5b3..b7d97d8 100755 --- a/tools/prepare-void +++ b/tools/prepare-void @@ -1,10 +1,11 @@ #!/usr/bin/env bash source "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" [[ $# == 0 ]] || die 'Usage: tools/prepare-void' +"$FDS_ROOT/tools/prepare-void-workspace" check_void_pin # xbps-src reads this ignored local configuration instead of user-global settings. if [[ -f $FDS_VOID/etc/conf ]] && ! cmp -s "$FDS_ROOT/config/xbps-src.conf" "$FDS_VOID/etc/conf"; then - die 'vendor/void-packages/etc/conf differs from config/xbps-src.conf; reconcile it explicitly' + die '.host/void-packages/etc/conf differs from config/xbps-src.conf; reconcile it explicitly' fi cp -- "$FDS_ROOT/config/xbps-src.conf" "$FDS_VOID/etc/conf" # Only directories containing a template are active overlays. diff --git a/tools/prepare-void-workspace b/tools/prepare-void-workspace new file mode 100755 index 0000000..641391c --- /dev/null +++ b/tools/prepare-void-workspace @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Prepare a writable Void build checkout outside the upstream submodule.""" +import fcntl +import hashlib +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile + + +def git(directory, *arguments): + return subprocess.check_output(['git', '-C', str(directory), *arguments], text=True).strip() + + +def inventory(root): + result = {} + for path in sorted(root.rglob('*')): + name = path.relative_to(root).as_posix() + if path.is_symlink(): + result[name] = ('link', os.readlink(path)) + elif path.is_file(): + with path.open('rb') as stream: + result[name] = ('file', path.stat().st_mode & 0o777, hashlib.file_digest(stream, 'sha256').hexdigest()) + elif path.is_dir(): + result[name] = ('directory',) + else: + raise ValueError(f'Unexpected special source file: {path}') + return result + + +def prepare(project): + source = project / 'vendor/void-packages' + workspace = project / '.host/void-packages' + pin = (project / 'VOID_PACKAGES_COMMIT').read_text().strip() + if not re.fullmatch('[0-9a-f]{40}', pin) or git(source, 'rev-parse', 'HEAD') != pin: + raise ValueError('Upstream Void checkout differs from VOID_PACKAGES_COMMIT') + if git(source, 'diff', 'HEAD', '--'): + raise ValueError('Upstream Void has staged or tracked changes; preserve them outside the submodule first') + if source.is_symlink() or workspace.is_symlink(): + raise ValueError('Void source and build checkout must be real directories') + + # Migrate only known generated copies after comparing every entry. Unknown + # or independently edited source files are never deleted or overwritten. + overlays = {path.parent.name: path.parent for path in (project / 'packages').glob('*/template')} + for name in ('hello', 'report'): + example = project / f'examples/software/{name}/void' + if (example / 'template').is_file(): + overlays[f'fds-demo-{name}'] = example + moves = [] + duplicates = [] + known = set() + for package, original in overlays.items(): + old = source / 'srcpkgs' / package + if not old.exists() and not old.is_symlink(): + continue + if git(source, 'ls-tree', 'HEAD', f'srcpkgs/{package}'): + raise ValueError(f'FDS overlay collides with upstream package: {package}') + if old.is_symlink() or not old.is_dir() or inventory(old) != inventory(original): + raise ValueError(f'Preserve independent edits in {old} before migrating it') + known.add(f'srcpkgs/{package}/') + new = workspace / 'srcpkgs' / package + if new.exists() or new.is_symlink(): + if new.is_symlink() or not new.is_dir() or inventory(new) != inventory(original): + raise ValueError(f'Conflicting generated overlay: {new}') + duplicates.append(old) + else: + moves.append((old, new)) + for name in git(source, 'ls-files', '--others', '--exclude-standard').splitlines(): + if not any(name.startswith(prefix) for prefix in known): + raise ValueError(f'Untracked upstream file must be moved outside the submodule: {name}') + for old in [source / 'hostdir', *sorted(source.glob('masterdir-*')), source / 'etc/conf']: + if not old.exists() and not old.is_symlink(): + continue + new = workspace / old.relative_to(source) + if old.is_symlink() or new.exists() or new.is_symlink(): + raise ValueError(f'Cannot migrate {old}: symlink or existing destination {new}; reconcile explicitly') + if old == source / 'etc/conf' and old.read_bytes() != (project / 'config/xbps-src.conf').read_bytes(): + raise ValueError('Old Void configuration differs from config/xbps-src.conf; reconcile explicitly') + moves.append((old, new)) + + if not workspace.exists(): + # Independent objects keep this cache usable without Git alternates or + # hardlinks back into the submodule. No network access is needed. + with tempfile.TemporaryDirectory(prefix='void-checkout.', dir=workspace.parent) as temporary: + checkout = Path(temporary) / 'checkout' + subprocess.run(['git', 'clone', '--quiet', '--no-hardlinks', '--no-checkout', str(source), str(checkout)], check=True) + subprocess.run(['git', '-C', str(checkout), 'checkout', '--quiet', '--detach', pin], check=True) + subprocess.run(['git', '-C', str(checkout), 'remote', 'remove', 'origin'], check=True) + checkout.rename(workspace) + else: + if not (workspace / '.git').is_dir() or git(workspace, 'diff', 'HEAD', '--'): + raise ValueError('Build checkout has tracked changes; preserve them before preparing it') + if git(workspace, 'rev-parse', 'HEAD') != pin: + subprocess.run(['git', '-C', str(workspace), 'fetch', '--quiet', '--update-shallow', str(source), pin], check=True) + # Git refuses an update that would overwrite local files. + subprocess.run(['git', '-C', str(workspace), 'checkout', '--quiet', '--detach', pin], check=True) + for old, new in moves: + new.parent.mkdir(parents=True, exist_ok=True) + old.rename(new) + print(f'Moved {old.relative_to(project)} to {new.relative_to(project)}', flush=True) + for old in duplicates: + shutil.rmtree(old) + if git(source, 'status', '--porcelain', '--untracked-files=all'): + raise ValueError('Upstream submodule is not clean after preparation') + print('PASS: clean upstream reference; writable build checkout in .host/void-packages', flush=True) + + +def main(): + project = Path(__file__).resolve().parents[1] + if len(sys.argv) != 1: + sys.exit('Usage: tools/prepare-void-workspace') + try: + (project / '.host').mkdir(exist_ok=True) + with (project / '.host/void-workspace.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + prepare(project) + except (OSError, ValueError, subprocess.CalledProcessError) as error: + sys.exit(f'ERROR: {error}') + + +if __name__ == '__main__': + main() diff --git a/tools/smoke-test b/tools/smoke-test index 8d946df..e43c620 100755 --- a/tools/smoke-test +++ b/tools/smoke-test @@ -42,7 +42,7 @@ else printf 'SKIP: ARM execution (optional qemu-aarch64 not installed); ELF verification passed\n' fi XBPS_ARCH=x86_64 xbps-query -r "$FDS_VOID/masterdir-x86_64" -l >out/manifests/void-build-packages.txt -find vendor/void-packages/hostdir -type f -name '*.xbps' -print0 \ +find "$FDS_VOID/hostdir" -type f -name '*.xbps' -print0 \ | sort -z | xargs -0 -r sha256sum >out/manifests/void-package-inputs.sha256 sha256sum "$package" out/fds-smoketest >out/manifests/artifacts.sha256 printf 'PASS: M0 smoke test complete\n'