update docs
This commit is contained in:
@@ -3,7 +3,7 @@ source "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
|
||||
[[ $# == 0 || ( $# == 1 && $1 == --install-deps ) ]] || die 'Usage: tools/bootstrap-host [--install-deps]'
|
||||
[[ $(uname -s) == Linux && $(uname -m) == x86_64 ]] || die 'M0 requires an x86_64 Linux build host'
|
||||
source /etc/os-release
|
||||
[[ $ID == arch ]] || die 'M0 bootstrap supports Arch Linux; see docs/build-host.md'
|
||||
[[ $ID == arch ]] || die 'M0 bootstrap supports Arch Linux; see docs/developer/build-host.md'
|
||||
(( EUID != 0 )) || die 'Run as a normal user, not root'
|
||||
[[ ! $FDS_ROOT =~ [[:space:]] ]] || die 'xbps-src requires a checkout path without whitespace'
|
||||
if [[ ${1:-} == --install-deps ]]; then
|
||||
|
||||
+4
-4
@@ -4,16 +4,16 @@ source "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
|
||||
cd "$FDS_ROOT"
|
||||
mkdir -p out/logs out/manifests
|
||||
tools/cargo-build --locked --offline --release --target aarch64-unknown-linux-musl \
|
||||
-p fds-cli -p fds-stage0 -p fds-boottrace -p fds-cartridged -p fds-burn -p fds-release
|
||||
for binary in fds fds-stage0 fds-boottrace fds-cartridged fds-profile fds-burn fds-release; do
|
||||
-p fds-cli -p fds-control -p fds-stage0 -p fds-boottrace -p fds-cartridged -p fds-burn -p fds-release
|
||||
for binary in fds fds-control fds-program fds-stage0 fds-boottrace fds-cartridged fds-profile fds-burn fds-release; do
|
||||
tools/verify-elf "target/aarch64-unknown-linux-musl/release/$binary" aarch64 static
|
||||
cp "target/aarch64-unknown-linux-musl/release/$binary" "out/$binary"
|
||||
done
|
||||
cp out/fds out/fds-inspect
|
||||
cp out/fds out/fds-eject
|
||||
cp out/fds out/fds-power
|
||||
sha256sum out/fds-release out/fds-burn out/fds-inspect out/fds-eject out/fds-power out/fds out/fds-stage0 out/fds-boottrace out/fds-cartridged out/fds-profile >out/manifests/fds-tools.sha256
|
||||
find rust/fds-software rust/fds-common rust/fds-cli rust/fds-stage0 rust/fds-boottrace rust/fds-cartridged rust/fds-burn rust/fds-release -type f -print0 | sort -z | \
|
||||
sha256sum out/fds-control out/fds-program out/fds-release out/fds-burn out/fds-inspect out/fds-eject out/fds-power out/fds out/fds-stage0 out/fds-boottrace out/fds-cartridged out/fds-profile >out/manifests/fds-tools.sha256
|
||||
find rust/fds-control rust/fds-software rust/fds-common rust/fds-cli rust/fds-stage0 rust/fds-boottrace rust/fds-cartridged rust/fds-burn rust/fds-release -type f -print0 | sort -z | \
|
||||
xargs -0 sha256sum >out/manifests/fds-tools-inputs.sha256
|
||||
sha256sum Cargo.toml Cargo.lock rust-toolchain.toml .cargo/config.toml tools/cargo-build tools/lib.sh tools/build-fds >>out/manifests/fds-tools-inputs.sha256
|
||||
printf 'PASS: static ARM FDS CLI and stage0 tools built\n'
|
||||
|
||||
@@ -7,11 +7,11 @@ tools/build-fds
|
||||
tools/prepare-void
|
||||
input="$FDS_VOID/hostdir/sources/fds-cli-0.1.0"
|
||||
mkdir -p "$input" out/packages
|
||||
cp out/fds out/fds-boottrace out/fds-burn out/fds-inspect out/fds-eject out/fds-power out/fds-release LICENSE "$input/"
|
||||
cp out/fds out/fds-control out/fds-program out/fds-boottrace out/fds-burn out/fds-inspect out/fds-eject out/fds-power out/fds-release LICENSE "$input/"
|
||||
notices=$(mktemp -d "$FDS_ROOT/out/rust-notices.XXXXXX")
|
||||
python3 tools/rust-notices "$notices/RUST-NOTICES.txt"
|
||||
cp "$notices/RUST-NOTICES.txt" "$input/"
|
||||
(cd "$input" && sha256sum fds fds-boottrace fds-burn fds-inspect fds-eject fds-power fds-release LICENSE RUST-NOTICES.txt >SHA256SUMS)
|
||||
(cd "$input" && sha256sum fds fds-control fds-program fds-boottrace fds-burn fds-inspect fds-eject fds-power fds-release LICENSE RUST-NOTICES.txt >SHA256SUMS)
|
||||
(
|
||||
cd "$FDS_VOID"
|
||||
xbps_src -a aarch64 clean fds-cli
|
||||
|
||||
Executable
+263
@@ -0,0 +1,263 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Remove obsolete FDS workspaces, retaining published outputs and saved inputs."""
|
||||
import argparse
|
||||
import contextlib
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
# Only names produced by repository builders/tests belong here. Never sweep out/*:
|
||||
# it also holds signed releases, frozen inputs, user cartridges and DATA overlays.
|
||||
PREFIXES = (
|
||||
'rootfs-build', 'kernel-build', 'system-build', 'initramfs-build',
|
||||
'boot-build', 'recovery-build', 'internal-build',
|
||||
'eeprom-production', 'eeprom-development',
|
||||
'cartridged-source', 'init-source', 'dasung-services', 'dasung-s6',
|
||||
'rust-notices', 'verify-package', 'dasung-check', 'dasung-s6-run',
|
||||
'm1-checks', 'm2-vm', 'm4-vm', 'm5-vm', 'm6-vm', 'm7-vm', 'm8-vm',
|
||||
'm9-images', 'm9-vm', 'm9-system', 'm10-vm', 'm11-vm', 'm11-faults',
|
||||
'm12-clock', 'm12-development', 'm12-eeprom', 'm12-internal',
|
||||
'm12-recovery', 'm12-release-contracts', 'm12-signing',
|
||||
'rust-profiles', 'workstation-images', 'emu-test',
|
||||
)
|
||||
GENERATED = re.compile(r'(?:' + '|'.join(PREFIXES) + r')\.[A-Za-z0-9_-]{6,8}')
|
||||
POINTERS = (
|
||||
'out/workstation-images-current.txt',
|
||||
'out/workstation-emulator-current.txt',
|
||||
'out/manifests/dasung-s6-database.txt',
|
||||
)
|
||||
LOCKS = ('.clean.lock', '.rootfs.lock', '.base-packages.lock', '.image-tools.lock')
|
||||
|
||||
|
||||
def within(path, parent):
|
||||
return path.is_relative_to(parent)
|
||||
|
||||
|
||||
def workspace_for(project, path):
|
||||
"""Find the only possible candidate ancestor without scanning every run."""
|
||||
try:
|
||||
parts = path.relative_to(project).parts
|
||||
except ValueError:
|
||||
return None
|
||||
if parts and parts[0] == 'target':
|
||||
return project / 'target'
|
||||
if len(parts) >= 2 and parts[0] == 'out':
|
||||
return project / 'out' / parts[1]
|
||||
return None
|
||||
|
||||
|
||||
def candidates(project):
|
||||
out = project / 'out'
|
||||
if out.is_symlink():
|
||||
raise ValueError('out is a symlink; refusing to clean an external output tree')
|
||||
if not out.exists():
|
||||
entries = []
|
||||
else:
|
||||
entries = sorted(out.iterdir())
|
||||
found = {p for p in entries if GENERATED.fullmatch(p.name)
|
||||
and p.is_dir() and not p.is_symlink()}
|
||||
target = project / 'target'
|
||||
if target.is_dir() and not target.is_symlink():
|
||||
found.add(target)
|
||||
protected = {}
|
||||
|
||||
def retain(path, reason):
|
||||
# Retain both a lexical target and its resolved target, including a link
|
||||
# reached through a directory alias. Out-of-tree links never add candidates.
|
||||
lexical = Path(os.path.abspath(path))
|
||||
for value in (lexical, lexical.resolve()):
|
||||
candidate = workspace_for(project, value)
|
||||
if candidate in found:
|
||||
protected.setdefault(candidate, reason)
|
||||
|
||||
# Git-tracked content is never build trash, even if named like a workspace.
|
||||
tracked = subprocess.check_output(
|
||||
['git', '-C', str(project), 'ls-files', '-z', '--', 'out', 'target'])
|
||||
for name in os.fsdecode(tracked).split('\0'):
|
||||
if name:
|
||||
retain(project / name, 'contains tracked content')
|
||||
for path in found:
|
||||
if os.path.lexists(path / '.fds-keep'):
|
||||
protected[path] = '.fds-keep marker'
|
||||
|
||||
manifests = out / 'manifests'
|
||||
links = entries + (list(manifests.iterdir())
|
||||
if manifests.is_dir() and not manifests.is_symlink() else [])
|
||||
for link in links:
|
||||
if link.is_symlink():
|
||||
retain(link.parent / link.readlink(), f'published link {link.relative_to(project)}')
|
||||
for name in POINTERS:
|
||||
pointer = project / name
|
||||
if pointer.is_file():
|
||||
value = pointer.read_text().strip()
|
||||
if value:
|
||||
retain(project / value, f'current pointer {name}')
|
||||
|
||||
# These suites consume the newest image fixture by mtime, without a link.
|
||||
image_runs = [p for p in found if p.name.startswith('m9-images.')]
|
||||
if image_runs:
|
||||
protected[max(image_runs, key=lambda p: p.stat().st_mtime_ns)] = 'latest media-image fixture'
|
||||
|
||||
# A retained workspace can contain links to another generated workspace.
|
||||
scanned = set()
|
||||
while set(protected) - scanned:
|
||||
for path in set(protected) - scanned:
|
||||
scanned.add(path)
|
||||
for directory, dirs, files in os.walk(path, followlinks=False):
|
||||
for name in dirs + files:
|
||||
link = Path(directory) / name
|
||||
if link.is_symlink():
|
||||
retain(link.parent / link.readlink(), f'dependency of {path.name}')
|
||||
return sorted(found - set(protected)), protected
|
||||
|
||||
|
||||
def check_mounts(paths, mountinfo=None):
|
||||
if mountinfo is None:
|
||||
mountinfo = Path('/proc/self/mountinfo').read_text()
|
||||
for line in mountinfo.splitlines():
|
||||
value = re.sub(r'\\([0-7]{3})', lambda m: chr(int(m[1], 8)), line.split()[4])
|
||||
mount = Path(value)
|
||||
if any(within(mount, path) for path in paths):
|
||||
raise ValueError(f'mounted path {mount}; unmount it before cleaning')
|
||||
|
||||
|
||||
def check_processes(project, paths):
|
||||
selected = set(paths)
|
||||
ancestors = set()
|
||||
pid = os.getpid()
|
||||
while pid > 0:
|
||||
ancestors.add(pid)
|
||||
try:
|
||||
status = Path(f'/proc/{pid}/status').read_text()
|
||||
pid = int(re.search(r'^PPid:\s+(\d+)', status, re.M)[1])
|
||||
except FileNotFoundError:
|
||||
break
|
||||
for proc in Path('/proc').iterdir():
|
||||
if not proc.name.isdigit() or int(proc.name) in ancestors:
|
||||
continue
|
||||
command = ''
|
||||
try:
|
||||
if proc.stat().st_uid != os.getuid():
|
||||
continue
|
||||
command = (proc / 'cmdline').read_bytes().replace(b'\0', b' ').decode(errors='replace')
|
||||
cwd = (proc / 'cwd').resolve(strict=True)
|
||||
# Catch a build before it has opened any candidate output files.
|
||||
name = (proc / 'comm').read_text().strip()
|
||||
building = (name in ('make', 'gmake', 'cargo', 'rustc', 'rustup', 'bwrap')
|
||||
or name.startswith(('qemu-', 'xbps-'))
|
||||
or re.search(r'(?:tools/|tests/integration/|image/build-)', command))
|
||||
if building and (within(cwd, project) or str(project) in command):
|
||||
raise ValueError(f'active build/test process {proc.name} ({name}); stop it before cleaning')
|
||||
for handle in [proc / 'cwd', proc / 'exe', *(proc / 'fd').iterdir()]:
|
||||
try:
|
||||
value = Path(os.readlink(handle))
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
if workspace_for(project, value) in selected:
|
||||
raise ValueError(f'process {proc.name} is using {value}; stop it before cleaning')
|
||||
except (FileNotFoundError, ProcessLookupError):
|
||||
continue
|
||||
except PermissionError as error:
|
||||
# Desktop session helpers may be nondumpable even for the same UID.
|
||||
# Refuse an inaccessible process naming this checkout; unrelated
|
||||
# protected processes must not make cleanup permanently unusable.
|
||||
if str(project) in command:
|
||||
raise ValueError(f'cannot inspect project process {proc.name}: {error}') from error
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def locks(project):
|
||||
with contextlib.ExitStack() as stack:
|
||||
out = project / 'out'
|
||||
if out.is_symlink():
|
||||
raise ValueError('out is a symlink; refusing cleanup')
|
||||
out.mkdir(exist_ok=True)
|
||||
for name in LOCKS:
|
||||
fd = os.open(out / name, os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
|
||||
stream = stack.enter_context(os.fdopen(fd, 'w'))
|
||||
try:
|
||||
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError as error:
|
||||
raise ValueError(f'{name} is locked; another build or cleanup is running') from error
|
||||
yield
|
||||
|
||||
|
||||
def footprint(paths):
|
||||
if not paths:
|
||||
return 0
|
||||
result = subprocess.check_output(['du', '-sx', '-B1', '--', *map(str, paths)], text=True)
|
||||
# One invocation counts hardlinks once. Reflinks and snapshots can still
|
||||
# share physical extents, so this is not a promise of filesystem free space.
|
||||
return sum(int(line.split('\t', 1)[0]) for line in result.splitlines())
|
||||
|
||||
|
||||
def remove_tree(path):
|
||||
if path.is_symlink() or not path.is_dir():
|
||||
raise ValueError(f'workspace changed since preview: {path}')
|
||||
# Some guest directories are deliberately read-only. Change only directory
|
||||
# permissions; file modes may belong to hardlinks in retained rootfs trees.
|
||||
def walk_error(error):
|
||||
raise error
|
||||
os.chmod(path, path.stat().st_mode | 0o700, follow_symlinks=False)
|
||||
for directory, dirs, _ in os.walk(path, topdown=True, followlinks=False, onerror=walk_error):
|
||||
for item in [Path(directory), *(Path(directory) / name for name in dirs)]:
|
||||
mode = item.lstat().st_mode
|
||||
if stat.S_ISDIR(mode) and mode & 0o700 != 0o700:
|
||||
os.chmod(item, mode | 0o700, follow_symlinks=False)
|
||||
shutil.rmtree(path)
|
||||
|
||||
|
||||
def clean(project, dry_run):
|
||||
paths, protected = candidates(project)
|
||||
identities = {path: (path.stat().st_dev, path.stat().st_ino) for path in paths}
|
||||
check_mounts(paths)
|
||||
if not dry_run:
|
||||
check_processes(project, paths)
|
||||
for path in paths:
|
||||
print(f'{"WOULD REMOVE" if dry_run else "REMOVE"} {path.relative_to(project)}', flush=True)
|
||||
amount = footprint(paths)
|
||||
print(f'{len(paths)} disposable directories; allocated footprint {amount / 1024**3:.2f} GiB.', flush=True)
|
||||
print('Physical space recovered can be smaller with hardlinks, reflinks or snapshots.', flush=True)
|
||||
for path, reason in sorted(protected.items()):
|
||||
print(f'KEEP {path.relative_to(project)} ({reason})', flush=True)
|
||||
print('Saved releases/inputs/rebuild trees, unknown outputs, caches, out/logs and out/manifests are kept.', flush=True)
|
||||
if dry_run:
|
||||
print('Preview only. Run make clean to apply.', flush=True)
|
||||
return
|
||||
check_processes(project, paths)
|
||||
check_mounts(paths)
|
||||
if candidates(project)[0] != paths or any(
|
||||
path.is_symlink() or (path.stat().st_dev, path.stat().st_ino) != identities[path]
|
||||
for path in paths):
|
||||
raise ValueError('output selection changed during inspection; rerun cleanup after stopping builds')
|
||||
before = shutil.disk_usage(project).free
|
||||
for path in paths:
|
||||
remove_tree(path)
|
||||
after = shutil.disk_usage(project).free
|
||||
print(f'PASS: removed {len(paths)} disposable directories; '
|
||||
f'filesystem free-space change {(after - before) / 1024**3:+.2f} GiB.', flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--dry-run', action='store_true', help='list the exact cleanup selection without deleting anything')
|
||||
args = parser.parse_args()
|
||||
project = Path(__file__).resolve().parents[1]
|
||||
try:
|
||||
if args.dry_run:
|
||||
clean(project, True)
|
||||
else:
|
||||
with locks(project):
|
||||
clean(project, False)
|
||||
except (OSError, ValueError, subprocess.CalledProcessError) as error:
|
||||
parser.exit(1, f'ERROR: {error}\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -6,6 +6,11 @@ export LC_ALL=C
|
||||
sed -i 's/^#en_US.UTF-8 UTF-8[[:space:]]*$/en_US.UTF-8 UTF-8/' /etc/default/libc-locales
|
||||
grep -qx 'en_US.UTF-8 UTF-8' /etc/default/libc-locales
|
||||
xbps-reconfigure -fa
|
||||
# Install FDS defaults globally so plain wmaker also uses the grayscale theme.
|
||||
# Home-directory preferences continue to override these image defaults.
|
||||
for name in WindowMaker WMRootMenu WMWindowAttributes; do
|
||||
install -m644 "/usr/share/fds/eink/$name" "/etc/WindowMaker/$name"
|
||||
done
|
||||
# The upstream CA package suppresses updater errors and exits successfully.
|
||||
# Require the actual generator to succeed instead of trusting that wrapper.
|
||||
update-ca-certificates --fresh
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ FDS_VOID="$FDS_ROOT/vendor/void-packages"
|
||||
FDS_XBPS="$FDS_ROOT/.host/xbps/usr/bin"
|
||||
|
||||
die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
need() { command -v "$1" >/dev/null || die "Missing host command: $1 (see docs/build-host.md)"; }
|
||||
need() { command -v "$1" >/dev/null || die "Missing host command: $1 (see docs/developer/build-host.md)"; }
|
||||
|
||||
check_void_pin() {
|
||||
local pin actual
|
||||
|
||||
@@ -75,5 +75,5 @@ Use new directories and paths without whitespace. The restored build has its
|
||||
recorded modes preserved by `--same-permissions`; `--no-same-owner` keeps the
|
||||
files owned by the ordinary build user. Its compiler and package environment use
|
||||
their own caches and have no network access. Host prerequisites and additional comparison
|
||||
instructions are in [Offline rebuilds](source/docs/reproducible-builds.md).
|
||||
instructions are in [Offline rebuilds](source/docs/developer/reproducible-builds.md).
|
||||
The private signing key is not included in this release or its input archive.
|
||||
|
||||
@@ -40,6 +40,11 @@ def audit(root, configured=True):
|
||||
assert profile in ("cli", "development", "recovery"), "Unknown image profile"
|
||||
for optional in ("desktop", "xserver", "desktop-session", "network", "dhcp"):
|
||||
assert not (root / f"etc/s6-rc/source/boot/contents.d/{optional}").exists(), "Optional service in boot bundle"
|
||||
if configured:
|
||||
for name in ("WindowMaker", "WMRootMenu", "WMWindowAttributes"):
|
||||
assert (root / "etc/WindowMaker" / name).read_bytes() == (root / "usr/share/fds/eink" / name).read_bytes(), "FDS grayscale defaults must be global"
|
||||
for name in ("fds-program", "fds-control"):
|
||||
assert (root / "usr/bin" / name).is_file(), f"Missing FDS interface: {name}"
|
||||
if profile == "development":
|
||||
assert "xorg-server-xvfb" in db and "xdotool" in db, "Missing development display diagnostics"
|
||||
for package in "gcc glibc-devel make cmake meson ninja pkg-config rust cargo git gdb strace vim".split():
|
||||
|
||||
@@ -10,7 +10,7 @@ root=$(realpath -e -- "$1")
|
||||
"$FDS_ROOT/tools/verify-elf" "$root/usr/bin/fds-boottrace" aarch64 static
|
||||
"$FDS_ROOT/tools/verify-elf" "$root/usr/bin/fds-cartridged" aarch64 static
|
||||
"$FDS_ROOT/tools/verify-elf" "$root/usr/bin/fds-profile" aarch64 static
|
||||
for binary in fds-burn fds-inspect fds-eject fds-power fds-release; do
|
||||
for binary in fds-control fds-program fds-burn fds-inspect fds-eject fds-power fds-release; do
|
||||
"$FDS_ROOT/tools/verify-elf" "$root/usr/bin/$binary" aarch64 static
|
||||
done
|
||||
"$FDS_ROOT/tools/in-rootfs" "$root" --direct /usr/bin/xbps-pkgdb -a
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ source "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
|
||||
[[ $# == 0 ]] || die 'Usage: tools/run-system-vm'
|
||||
cd "$FDS_ROOT"
|
||||
for file in out/kernel/boot/kernel_2712.img out/fds-initramfs.img out/fds-system-cli.img; do
|
||||
[[ -s $file ]] || die "Build required input first: $file (see docs/boot.md)"
|
||||
[[ -s $file ]] || die "Build required input first: $file (see docs/developer/boot.md)"
|
||||
done
|
||||
printf 'FDS user console VM. Press Ctrl-a, then x to close it. Temporary home is discarded.\n'
|
||||
exec tools/in-void qemu-system-aarch64 \
|
||||
|
||||
Reference in New Issue
Block a user