FDS/OS 1.0 fixes

tools/bootstrap-host

@@ -8,10 +8,10 @@ source /etc/os-release
 [[ ! $FDS_ROOT =~ [[:space:]] ]] || die 'xbps-src requires a checkout path without whitespace'
 if [[ ${1:-} == --install-deps ]]; then
     sudo pacman -S --needed bash coreutils binutils git curl make file tar xz gzip zstd \
-        bubblewrap rustup ca-certificates findutils diffutils grep sed gawk util-linux
+        bubblewrap rustup ca-certificates findutils diffutils grep sed gawk util-linux python
 fi
 for cmd in bash git curl make file readelf ldd tar xz gzip zstd sha256sum bwrap rustup \
-    find diff grep sed awk flock install; do
+    find diff grep sed awk flock install python3; do
     need "$cmd"
 done
 # Fail before downloading if unprivileged build containers cannot run.
@@ -21,7 +21,7 @@ mkdir -p out/downloads out/logs .host
 if [[ ${FDS_OFFLINE:-0} != 1 ]]; then
     git -c http.lowSpeedLimit=1000 -c http.lowSpeedTime=60 submodule update --init --depth 1 vendor/void-packages
 fi
-check_void_pin
+check_void_source
 source config/host-tools.conf
 archive="out/downloads/${XBPS_STATIC_URL##*/}"
 if [[ ! -f $archive ]]; then

tools/frozen-inputs

@@ -79,7 +79,13 @@ def check_pin():
     if pin != actual:
         raise ValueError('Void checkout differs from its pin')
     run('git', '-C', void, 'diff', '--exit-code', 'HEAD', '--')
-    return void, pin
+    build = project / '.host/void-packages'
+    if subprocess.check_output(['git', '-C', str(build), 'rev-parse', 'HEAD'], text=True).strip() != pin:
+        raise ValueError('Void build checkout differs from its pin')
+    run('git', '-C', build, 'diff', '--exit-code', 'HEAD', '--')
+    if subprocess.check_output(['git', '-C', str(void), 'status', '--porcelain', '--untracked-files=all'], text=True).strip():
+        raise ValueError('Void upstream submodule is dirty')
+    return build, pin
 
 
 def add_package(source, destination):
@@ -245,6 +251,12 @@ def restore(source, destination):
     epoch = int(subprocess.check_output(['git', '-C', str(void), 'show', '-s', '--format=%ct', 'HEAD']))
     if epoch != lock['source_epoch']:
         raise ValueError('Restored Void commit timestamp differs from the snapshot')
+    # Older frozen projects expect caches under vendor; their saved sources and
+    # release inputs remain unchanged. New projects use the separate workspace.
+    prepare = destination / 'tools/prepare-void-workspace'
+    if prepare.is_file():
+        run(sys.executable, prepare)
+        void = destination / '.host/void-packages'
     copy(source / 'masterdir', void / 'masterdir-x86_64')
     copy(source / 'sources', void / 'hostdir/sources')
     copy(source / 'repositories/build', void / 'hostdir/frozen-repository')

tools/lib.sh

@@ -3,22 +3,31 @@
 set -euo pipefail
 export LC_ALL=C
 FDS_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)
-FDS_VOID="$FDS_ROOT/vendor/void-packages"
+FDS_VOID_SOURCE="$FDS_ROOT/vendor/void-packages"
+FDS_VOID="$FDS_ROOT/.host/void-packages"
 FDS_XBPS="$FDS_ROOT/.host/xbps/usr/bin"
 
 die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
 need() { command -v "$1" >/dev/null || die "Missing host command: $1 (see docs/developer/build-host.md)"; }
 
-check_void_pin() {
+check_void_source() {
     local pin actual
     pin=$(cat "$FDS_ROOT/VOID_PACKAGES_COMMIT")
     [[ $pin =~ ^[0-9a-f]{40}$ ]] || die 'Invalid VOID_PACKAGES_COMMIT'
-    [[ -f $FDS_VOID/xbps-src ]] || die 'Void submodule is missing; run make bootstrap'
-    actual=$(git -C "$FDS_VOID" rev-parse HEAD)
+    [[ -f $FDS_VOID_SOURCE/xbps-src ]] || die 'Void submodule is missing; run make bootstrap'
+    actual=$(git -C "$FDS_VOID_SOURCE" rev-parse HEAD)
     [[ $actual == "$pin" ]] || die "Void checkout mismatch: expected $pin, found $actual"
-    git -C "$FDS_VOID" diff --quiet HEAD -- || die 'Tracked Void files were changed; keep FDS changes in packages/'
+    git -C "$FDS_VOID_SOURCE" diff --quiet HEAD -- || die 'Tracked Void files were changed; keep FDS changes in packages/'
     export SOURCE_DATE_EPOCH
-    SOURCE_DATE_EPOCH=$(git -C "$FDS_VOID" show -s --format=%ct HEAD)
+    SOURCE_DATE_EPOCH=$(git -C "$FDS_VOID_SOURCE" show -s --format=%ct HEAD)
+}
+
+check_void_pin() {
+    check_void_source
+    [[ -z $(git -C "$FDS_VOID_SOURCE" status --porcelain --untracked-files=all) ]] || die 'Upstream Void submodule is dirty; run make bootstrap to migrate known build copies'
+    [[ -f $FDS_VOID/xbps-src ]] || die 'Void build checkout is missing; run make bootstrap'
+    [[ $(git -C "$FDS_VOID" rev-parse HEAD) == "$(cat "$FDS_ROOT/VOID_PACKAGES_COMMIT")" ]] || die 'Void build checkout pin changed; run make bootstrap'
+    git -C "$FDS_VOID" diff --quiet HEAD -- || die 'Tracked build-checkout files changed; keep FDS sources in packages/'
 }
 
 # Keep the upstream entry point unchanged. Offline builds retain normal

tools/prepare-void

@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
 source "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
 [[ $# == 0 ]] || die 'Usage: tools/prepare-void'
+"$FDS_ROOT/tools/prepare-void-workspace"
 check_void_pin
 # xbps-src reads this ignored local configuration instead of user-global settings.
 if [[ -f $FDS_VOID/etc/conf ]] && ! cmp -s "$FDS_ROOT/config/xbps-src.conf" "$FDS_VOID/etc/conf"; then
-    die 'vendor/void-packages/etc/conf differs from config/xbps-src.conf; reconcile it explicitly'
+    die '.host/void-packages/etc/conf differs from config/xbps-src.conf; reconcile it explicitly'
 fi
 cp -- "$FDS_ROOT/config/xbps-src.conf" "$FDS_VOID/etc/conf"
 # Only directories containing a template are active overlays.

tools/prepare-void-workspace

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""Prepare a writable Void build checkout outside the upstream submodule."""
+import fcntl
+import hashlib
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def git(directory, *arguments):
+    return subprocess.check_output(['git', '-C', str(directory), *arguments], text=True).strip()
+
+
+def inventory(root):
+    result = {}
+    for path in sorted(root.rglob('*')):
+        name = path.relative_to(root).as_posix()
+        if path.is_symlink():
+            result[name] = ('link', os.readlink(path))
+        elif path.is_file():
+            with path.open('rb') as stream:
+                result[name] = ('file', path.stat().st_mode & 0o777, hashlib.file_digest(stream, 'sha256').hexdigest())
+        elif path.is_dir():
+            result[name] = ('directory',)
+        else:
+            raise ValueError(f'Unexpected special source file: {path}')
+    return result
+
+
+def prepare(project):
+    source = project / 'vendor/void-packages'
+    workspace = project / '.host/void-packages'
+    pin = (project / 'VOID_PACKAGES_COMMIT').read_text().strip()
+    if not re.fullmatch('[0-9a-f]{40}', pin) or git(source, 'rev-parse', 'HEAD') != pin:
+        raise ValueError('Upstream Void checkout differs from VOID_PACKAGES_COMMIT')
+    if git(source, 'diff', 'HEAD', '--'):
+        raise ValueError('Upstream Void has staged or tracked changes; preserve them outside the submodule first')
+    if source.is_symlink() or workspace.is_symlink():
+        raise ValueError('Void source and build checkout must be real directories')
+
+    # Migrate only known generated copies after comparing every entry. Unknown
+    # or independently edited source files are never deleted or overwritten.
+    overlays = {path.parent.name: path.parent for path in (project / 'packages').glob('*/template')}
+    for name in ('hello', 'report'):
+        example = project / f'examples/software/{name}/void'
+        if (example / 'template').is_file():
+            overlays[f'fds-demo-{name}'] = example
+    moves = []
+    duplicates = []
+    known = set()
+    for package, original in overlays.items():
+        old = source / 'srcpkgs' / package
+        if not old.exists() and not old.is_symlink():
+            continue
+        if git(source, 'ls-tree', 'HEAD', f'srcpkgs/{package}'):
+            raise ValueError(f'FDS overlay collides with upstream package: {package}')
+        if old.is_symlink() or not old.is_dir() or inventory(old) != inventory(original):
+            raise ValueError(f'Preserve independent edits in {old} before migrating it')
+        known.add(f'srcpkgs/{package}/')
+        new = workspace / 'srcpkgs' / package
+        if new.exists() or new.is_symlink():
+            if new.is_symlink() or not new.is_dir() or inventory(new) != inventory(original):
+                raise ValueError(f'Conflicting generated overlay: {new}')
+            duplicates.append(old)
+        else:
+            moves.append((old, new))
+    for name in git(source, 'ls-files', '--others', '--exclude-standard').splitlines():
+        if not any(name.startswith(prefix) for prefix in known):
+            raise ValueError(f'Untracked upstream file must be moved outside the submodule: {name}')
+    for old in [source / 'hostdir', *sorted(source.glob('masterdir-*')), source / 'etc/conf']:
+        if not old.exists() and not old.is_symlink():
+            continue
+        new = workspace / old.relative_to(source)
+        if old.is_symlink() or new.exists() or new.is_symlink():
+            raise ValueError(f'Cannot migrate {old}: symlink or existing destination {new}; reconcile explicitly')
+        if old == source / 'etc/conf' and old.read_bytes() != (project / 'config/xbps-src.conf').read_bytes():
+            raise ValueError('Old Void configuration differs from config/xbps-src.conf; reconcile explicitly')
+        moves.append((old, new))
+
+    if not workspace.exists():
+        # Independent objects keep this cache usable without Git alternates or
+        # hardlinks back into the submodule. No network access is needed.
+        with tempfile.TemporaryDirectory(prefix='void-checkout.', dir=workspace.parent) as temporary:
+            checkout = Path(temporary) / 'checkout'
+            subprocess.run(['git', 'clone', '--quiet', '--no-hardlinks', '--no-checkout', str(source), str(checkout)], check=True)
+            subprocess.run(['git', '-C', str(checkout), 'checkout', '--quiet', '--detach', pin], check=True)
+            subprocess.run(['git', '-C', str(checkout), 'remote', 'remove', 'origin'], check=True)
+            checkout.rename(workspace)
+    else:
+        if not (workspace / '.git').is_dir() or git(workspace, 'diff', 'HEAD', '--'):
+            raise ValueError('Build checkout has tracked changes; preserve them before preparing it')
+        if git(workspace, 'rev-parse', 'HEAD') != pin:
+            subprocess.run(['git', '-C', str(workspace), 'fetch', '--quiet', '--update-shallow', str(source), pin], check=True)
+            # Git refuses an update that would overwrite local files.
+            subprocess.run(['git', '-C', str(workspace), 'checkout', '--quiet', '--detach', pin], check=True)
+    for old, new in moves:
+        new.parent.mkdir(parents=True, exist_ok=True)
+        old.rename(new)
+        print(f'Moved {old.relative_to(project)} to {new.relative_to(project)}', flush=True)
+    for old in duplicates:
+        shutil.rmtree(old)
+    if git(source, 'status', '--porcelain', '--untracked-files=all'):
+        raise ValueError('Upstream submodule is not clean after preparation')
+    print('PASS: clean upstream reference; writable build checkout in .host/void-packages', flush=True)
+
+
+def main():
+    project = Path(__file__).resolve().parents[1]
+    if len(sys.argv) != 1:
+        sys.exit('Usage: tools/prepare-void-workspace')
+    try:
+        (project / '.host').mkdir(exist_ok=True)
+        with (project / '.host/void-workspace.lock').open('a') as lock:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            prepare(project)
+    except (OSError, ValueError, subprocess.CalledProcessError) as error:
+        sys.exit(f'ERROR: {error}')
+
+
+if __name__ == '__main__':
+    main()

tools/smoke-test

@@ -42,7 +42,7 @@ else
     printf 'SKIP: ARM execution (optional qemu-aarch64 not installed); ELF verification passed\n'
 fi
 XBPS_ARCH=x86_64 xbps-query -r "$FDS_VOID/masterdir-x86_64" -l >out/manifests/void-build-packages.txt
-find vendor/void-packages/hostdir -type f -name '*.xbps' -print0 \
+find "$FDS_VOID/hostdir" -type f -name '*.xbps' -print0 \
     | sort -z | xargs -0 -r sha256sum >out/manifests/void-package-inputs.sha256
 sha256sum "$package" out/fds-smoketest >out/manifests/artifacts.sha256
 printf 'PASS: M0 smoke test complete\n'