felis
f4bc28043a
Accept native SD cards for internal settings alongside legacy NVMe, reject USB ancestry and ambiguous disks, and preserve read-only boot loading with explicit recovery writes. Require built-in MMC drivers, validate cached kernel configuration, and provide SD-only, SD-first and USB-first EEPROM profiles. Expose typed create and inspect commands through fds-flash, reuse the existing cartridge creation code, and document the optional e2fsprogs package dependency. Extend storage, EEPROM, flashing and hardware-test coverage and advance the wiki reference to the published SD guide. Validation: rootfs checks for CLI/development, boot matrix, internal storage, EEPROM, flash, workstation, emulator, make check and wiki checks passed. The full internal suite passed on an unchanged rerun after one unexplained VM shutdown stall. Standalone init-test was blocked by its unavailable pinned upstream kernel. Physical Pi checks remain pending.
269 lines
11 KiB
Python
Executable File
269 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Remove obsolete FDS workspaces, retaining published outputs and saved inputs."""
|
|
import argparse
|
|
import contextlib
|
|
import fcntl
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
# Only names produced by repository builders/tests belong here. Never sweep out/*:
|
|
# it also holds signed releases, frozen inputs, user cartridges and DATA overlays.
|
|
PREFIXES = (
|
|
'rootfs-build', 'kernel-build', 'system-build', 'initramfs-build',
|
|
'boot-build', 'recovery-build', 'internal-build',
|
|
'eeprom-production', 'eeprom-development', 'eeprom-maintenance',
|
|
'cartridged-source', 'init-source', 'dasung-services', 'dasung-s6',
|
|
'rust-notices', 'verify-package', 'dasung-check', 'dasung-s6-run',
|
|
'm1-checks', 'm2-vm', 'm4-vm', 'm5-vm', 'm6-vm', 'm7-vm', 'm8-vm',
|
|
'm9-images', 'm9-vm', 'm9-system', 'm10-vm', 'm11-vm', 'm11-faults',
|
|
'm12-clock', 'm12-development', 'm12-eeprom', 'm12-internal',
|
|
'm12-recovery', 'm12-release-contracts', 'm12-signing',
|
|
'rust-profiles', 'workstation-images', 'workstation-flash', 'workstation-flash-vm',
|
|
'emu-test', 'arch-package-test', 'version-image-test',
|
|
)
|
|
GENERATED = re.compile(r'(?:' + '|'.join(PREFIXES) + r')\.[A-Za-z0-9_-]{6,8}')
|
|
POINTERS = (
|
|
'out/arch-package-current.txt',
|
|
'out/version-image-current.txt',
|
|
'out/workstation-images-current.txt',
|
|
'out/workstation-emulator-current.txt',
|
|
'out/workstation-flash-current.txt',
|
|
'out/workstation-flash-vm-current.txt',
|
|
'out/manifests/dasung-s6-database.txt',
|
|
)
|
|
LOCKS = ('.clean.lock', '.rootfs.lock', '.base-packages.lock', '.image-tools.lock', '.workstation-package.lock')
|
|
|
|
|
|
def within(path, parent):
|
|
return path.is_relative_to(parent)
|
|
|
|
|
|
def workspace_for(project, path):
|
|
"""Find the only possible candidate ancestor without scanning every run."""
|
|
try:
|
|
parts = path.relative_to(project).parts
|
|
except ValueError:
|
|
return None
|
|
if parts and parts[0] == 'target':
|
|
return project / 'target'
|
|
if len(parts) >= 2 and parts[0] == 'out':
|
|
return project / 'out' / parts[1]
|
|
return None
|
|
|
|
|
|
def candidates(project):
|
|
out = project / 'out'
|
|
if out.is_symlink():
|
|
raise ValueError('out is a symlink; refusing to clean an external output tree')
|
|
if not out.exists():
|
|
entries = []
|
|
else:
|
|
entries = sorted(out.iterdir())
|
|
found = {p for p in entries if GENERATED.fullmatch(p.name)
|
|
and p.is_dir() and not p.is_symlink()}
|
|
target = project / 'target'
|
|
if target.is_dir() and not target.is_symlink():
|
|
found.add(target)
|
|
protected = {}
|
|
|
|
def retain(path, reason):
|
|
# Retain both a lexical target and its resolved target, including a link
|
|
# reached through a directory alias. Out-of-tree links never add candidates.
|
|
lexical = Path(os.path.abspath(path))
|
|
for value in (lexical, lexical.resolve()):
|
|
candidate = workspace_for(project, value)
|
|
if candidate in found:
|
|
protected.setdefault(candidate, reason)
|
|
|
|
# Git-tracked content is never build trash, even if named like a workspace.
|
|
tracked = subprocess.check_output(
|
|
['git', '-C', str(project), 'ls-files', '-z', '--', 'out', 'target'])
|
|
for name in os.fsdecode(tracked).split('\0'):
|
|
if name:
|
|
retain(project / name, 'contains tracked content')
|
|
for path in found:
|
|
if os.path.lexists(path / '.fds-keep'):
|
|
protected[path] = '.fds-keep marker'
|
|
|
|
manifests = out / 'manifests'
|
|
links = entries + (list(manifests.iterdir())
|
|
if manifests.is_dir() and not manifests.is_symlink() else [])
|
|
for link in links:
|
|
if link.is_symlink():
|
|
retain(link.parent / link.readlink(), f'published link {link.relative_to(project)}')
|
|
for name in POINTERS:
|
|
pointer = project / name
|
|
if pointer.is_file():
|
|
value = pointer.read_text().strip()
|
|
if value:
|
|
retain(project / value, f'current pointer {name}')
|
|
|
|
# These suites consume the newest image fixture by mtime, without a link.
|
|
image_runs = [p for p in found if p.name.startswith('m9-images.')]
|
|
if image_runs:
|
|
protected[max(image_runs, key=lambda p: p.stat().st_mtime_ns)] = 'latest media-image fixture'
|
|
|
|
# A retained workspace can contain links to another generated workspace.
|
|
scanned = set()
|
|
while set(protected) - scanned:
|
|
for path in set(protected) - scanned:
|
|
scanned.add(path)
|
|
for directory, dirs, files in os.walk(path, followlinks=False):
|
|
for name in dirs + files:
|
|
link = Path(directory) / name
|
|
if link.is_symlink():
|
|
retain(link.parent / link.readlink(), f'dependency of {path.name}')
|
|
return sorted(found - set(protected)), protected
|
|
|
|
|
|
def check_mounts(paths, mountinfo=None):
|
|
if mountinfo is None:
|
|
mountinfo = Path('/proc/self/mountinfo').read_text()
|
|
for line in mountinfo.splitlines():
|
|
value = re.sub(r'\\([0-7]{3})', lambda m: chr(int(m[1], 8)), line.split()[4])
|
|
mount = Path(value)
|
|
if any(within(mount, path) for path in paths):
|
|
raise ValueError(f'mounted path {mount}; unmount it before cleaning')
|
|
|
|
|
|
def check_processes(project, paths):
|
|
selected = set(paths)
|
|
ancestors = set()
|
|
pid = os.getpid()
|
|
while pid > 0:
|
|
ancestors.add(pid)
|
|
try:
|
|
status = Path(f'/proc/{pid}/status').read_text()
|
|
pid = int(re.search(r'^PPid:\s+(\d+)', status, re.M)[1])
|
|
except FileNotFoundError:
|
|
break
|
|
for proc in Path('/proc').iterdir():
|
|
if not proc.name.isdigit() or int(proc.name) in ancestors:
|
|
continue
|
|
command = ''
|
|
try:
|
|
if proc.stat().st_uid != os.getuid():
|
|
continue
|
|
command = (proc / 'cmdline').read_bytes().replace(b'\0', b' ').decode(errors='replace')
|
|
cwd = (proc / 'cwd').resolve(strict=True)
|
|
# Catch a build before it has opened any candidate output files.
|
|
name = (proc / 'comm').read_text().strip()
|
|
building = (name in ('make', 'gmake', 'cargo', 'rustc', 'rustup', 'bwrap')
|
|
or name.startswith(('qemu-', 'xbps-'))
|
|
or re.search(r'(?:tools/|tests/integration/|image/build-)', command))
|
|
if building and (within(cwd, project) or str(project) in command):
|
|
raise ValueError(f'active build/test process {proc.name} ({name}); stop it before cleaning')
|
|
for handle in [proc / 'cwd', proc / 'exe', *(proc / 'fd').iterdir()]:
|
|
try:
|
|
value = Path(os.readlink(handle))
|
|
except FileNotFoundError:
|
|
continue
|
|
if workspace_for(project, value) in selected:
|
|
raise ValueError(f'process {proc.name} is using {value}; stop it before cleaning')
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
continue
|
|
except PermissionError as error:
|
|
# Desktop session helpers may be nondumpable even for the same UID.
|
|
# Refuse an inaccessible process naming this checkout; unrelated
|
|
# protected processes must not make cleanup permanently unusable.
|
|
if str(project) in command:
|
|
raise ValueError(f'cannot inspect project process {proc.name}: {error}') from error
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def locks(project):
|
|
with contextlib.ExitStack() as stack:
|
|
out = project / 'out'
|
|
if out.is_symlink():
|
|
raise ValueError('out is a symlink; refusing cleanup')
|
|
out.mkdir(exist_ok=True)
|
|
for name in LOCKS:
|
|
fd = os.open(out / name, os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
|
|
stream = stack.enter_context(os.fdopen(fd, 'w'))
|
|
try:
|
|
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError as error:
|
|
raise ValueError(f'{name} is locked; another build or cleanup is running') from error
|
|
yield
|
|
|
|
|
|
def footprint(paths):
|
|
if not paths:
|
|
return 0
|
|
result = subprocess.check_output(['du', '-sx', '-B1', '--', *map(str, paths)], text=True)
|
|
# One invocation counts hardlinks once. Reflinks and snapshots can still
|
|
# share physical extents, so this is not a promise of filesystem free space.
|
|
return sum(int(line.split('\t', 1)[0]) for line in result.splitlines())
|
|
|
|
|
|
def remove_tree(path):
|
|
if path.is_symlink() or not path.is_dir():
|
|
raise ValueError(f'workspace changed since preview: {path}')
|
|
# Some guest directories are deliberately read-only. Change only directory
|
|
# permissions; file modes may belong to hardlinks in retained rootfs trees.
|
|
def walk_error(error):
|
|
raise error
|
|
os.chmod(path, path.stat().st_mode | 0o700, follow_symlinks=False)
|
|
for directory, dirs, _ in os.walk(path, topdown=True, followlinks=False, onerror=walk_error):
|
|
for item in [Path(directory), *(Path(directory) / name for name in dirs)]:
|
|
mode = item.lstat().st_mode
|
|
if stat.S_ISDIR(mode) and mode & 0o700 != 0o700:
|
|
os.chmod(item, mode | 0o700, follow_symlinks=False)
|
|
shutil.rmtree(path)
|
|
|
|
|
|
def clean(project, dry_run):
|
|
paths, protected = candidates(project)
|
|
identities = {path: (path.stat().st_dev, path.stat().st_ino) for path in paths}
|
|
check_mounts(paths)
|
|
if not dry_run:
|
|
check_processes(project, paths)
|
|
for path in paths:
|
|
print(f'{"WOULD REMOVE" if dry_run else "REMOVE"} {path.relative_to(project)}', flush=True)
|
|
amount = footprint(paths)
|
|
print(f'{len(paths)} disposable directories; allocated footprint {amount / 1024**3:.2f} GiB.', flush=True)
|
|
print('Physical space recovered can be smaller with hardlinks, reflinks or snapshots.', flush=True)
|
|
for path, reason in sorted(protected.items()):
|
|
print(f'KEEP {path.relative_to(project)} ({reason})', flush=True)
|
|
print('Saved releases/inputs/rebuild trees, unknown outputs, caches, out/logs and out/manifests are kept.', flush=True)
|
|
if dry_run:
|
|
print('Preview only. Run make clean to apply.', flush=True)
|
|
return
|
|
check_processes(project, paths)
|
|
check_mounts(paths)
|
|
if candidates(project)[0] != paths or any(
|
|
path.is_symlink() or (path.stat().st_dev, path.stat().st_ino) != identities[path]
|
|
for path in paths):
|
|
raise ValueError('output selection changed during inspection; rerun cleanup after stopping builds')
|
|
before = shutil.disk_usage(project).free
|
|
for path in paths:
|
|
remove_tree(path)
|
|
after = shutil.disk_usage(project).free
|
|
print(f'PASS: removed {len(paths)} disposable directories; '
|
|
f'filesystem free-space change {(after - before) / 1024**3:+.2f} GiB.', flush=True)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--dry-run', action='store_true', help='list the exact cleanup selection without deleting anything')
|
|
args = parser.parse_args()
|
|
project = Path(__file__).resolve().parents[1]
|
|
try:
|
|
if args.dry_run:
|
|
clean(project, True)
|
|
else:
|
|
with locks(project):
|
|
clean(project, False)
|
|
except (OSError, ValueError, subprocess.CalledProcessError) as error:
|
|
parser.exit(1, f'ERROR: {error}\n')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|