FDS/OS 1.0
This commit is contained in:
@@ -0,0 +1,299 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Boot independent recovery, preserve media by default, repair and replace media."""
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
project = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(project / 'tools'))
|
||||
from image_formats import digest, gpt, LINUX_FILESYSTEM
|
||||
from vm_test import VM
|
||||
|
||||
work = Path(tempfile.mkdtemp(prefix='m12-recovery.', dir=project / 'out'))
|
||||
rootfs = (project / 'out/rootfs-recovery.tar').resolve(strict=True)
|
||||
recovery = (project / 'out/fds-recovery.img').resolve(strict=True)
|
||||
source_system = (project / 'out/fds-system-cli.img').resolve(strict=True)
|
||||
(work / 'inputs.sha256').write_text(''.join(f'{digest(p)} {p}\n' for p in [rootfs, recovery, source_system, project / 'out/fds-initramfs.img', project / 'out/kernel/boot/kernel_2712.img']))
|
||||
controller = 'qemu-xhci,id=xhci,addr=05.0'
|
||||
# Build two independent EROFS outputs and require identical payload bytes.
|
||||
for name in ['repeat-a', 'repeat-b']:
|
||||
output = work / name
|
||||
output.mkdir()
|
||||
with (work / (name + '.log')).open('wb') as log:
|
||||
subprocess.run([str(project / 'image/build-recovery'), '--rootfs', str(rootfs), '--output-directory', str(output)], check=True, stdout=log, stderr=subprocess.STDOUT)
|
||||
assert digest(work / 'repeat-a/recovery.erofs') == digest(recovery) == digest(work / 'repeat-b/recovery.erofs')
|
||||
base = work / 'recovery-base.img'
|
||||
gpt(base, [('FDS_RECOVERY', LINUX_FILESYSTEM, recovery)])
|
||||
|
||||
def query(vm, args):
|
||||
try:
|
||||
return json.loads(vm.capture('fds --json ' + shlex.join(args), timeout=600 if args[0] == 'burn' else 180))
|
||||
except AssertionError:
|
||||
(work / 'last-recovery-checker.log').write_text(vm.capture('cat /run/fds/recovery/*.log 2>/dev/null || true') + '\n')
|
||||
raise
|
||||
|
||||
def wait(operation, condition, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
result = operation()
|
||||
if condition(result):
|
||||
return result
|
||||
assert time.monotonic() < deadline, result
|
||||
|
||||
def bay(vm, n, state):
|
||||
try:
|
||||
return wait(lambda: query(vm, ['bay', str(n)])['bays'][0], lambda row: row['state'] == state)
|
||||
except AssertionError:
|
||||
(work / (vm.name + '-failure-state.log')).write_text(vm.capture('cat /run/fds/ejected/* /run/fds/data-sessions/* 2>/dev/null; cat /proc/self/mountinfo; fds inspect BAY02; tail -n 80 /run/log/cartridged/current; dmesg | tail -n 40') + '\n')
|
||||
raise
|
||||
|
||||
def enter(vm, damaged=False):
|
||||
vm.expect(rb'SYSTEM CANNOT BE USED' if damaged else rb'FDS_SYSTEM MEDIA NOT PRESENT')
|
||||
vm.send('recovery')
|
||||
vm.expect(rb'RECOVERY# ')
|
||||
assert vm.capture('id -u') == '0'
|
||||
assert vm.capture('cat /usr/share/fds/image-profile') == 'recovery'
|
||||
wait(lambda: vm.capture('test -S /run/fds/control.sock && echo ready || echo pending'), lambda value: value == 'ready')
|
||||
|
||||
def finish(vm):
|
||||
vm.send('fds poweroff')
|
||||
vm.expect(rb'reboot: Power down', timeout=120)
|
||||
assert vm.child.wait(timeout=20) == 0
|
||||
|
||||
with VM(work, 'without-system', base, extra=['-device', controller, '-device', 'usb-kbd,bus=xhci.0,port=4']) as vm:
|
||||
enter(vm)
|
||||
info = query(vm, ['info'])
|
||||
assert info['pid1'].endswith('s6-svscan') and info['target'] == 'aarch64 static-musl'
|
||||
vm.capture('test "$(findmnt -nr -o FSTYPE /)" = erofs && findmnt -nr -o OPTIONS / | grep -qw ro')
|
||||
vm.capture('touch /etc/recovery-write-probe', ok=False)
|
||||
vm.capture('pgrep -x dasungd && command -v bash ls mount e2fsck xbps-query fds-burn fds-inspect && test -s /usr/share/doc/fds/recovery.md')
|
||||
report = wait(lambda: query(vm, ['topology']), lambda row: any('03' in d['interfaces'] for d in row['unmapped']))
|
||||
hub = next(d['topology'] for d in report['unmapped'] if '03' in d['interfaces']).rsplit('/', 1)[0]
|
||||
vm.capture('bash /usr/share/fds/capture-hardware /tmp/recovery-capture && cd /tmp/recovery-capture && sha256sum -c SHA256SUMS')
|
||||
(work / 'capture-status.tsv').write_text(vm.capture('cat /tmp/recovery-capture/status.tsv') + '\n')
|
||||
(work / 'boot-without-system.json').write_text(json.dumps(query(vm, ['boot-profile']), indent=2) + '\n')
|
||||
finish(vm)
|
||||
print('PASS: separate reproducible recovery payload; missing-SYSTEM selection, read-only root, native s6, local root console, base Dasung and diagnostic capture', flush=True)
|
||||
|
||||
config = ''
|
||||
for name, identity in [('usb2', hub), ('usb3', hub.replace(':usb2', ':usb3'))]:
|
||||
config += f'[{name}]\nhub={json.dumps(identity)}\n[{name}.ports]\n' + ''.join(f'{n}={n}\n' for n in range(1, 13))
|
||||
# Only the virtual bay calibration differs from the independently packaged root.
|
||||
with tarfile.open(rootfs) as source, tarfile.open(work / 'calibrated.tar', 'w', format=tarfile.PAX_FORMAT) as output:
|
||||
found = False
|
||||
for member in source:
|
||||
if member.name == 'etc/fds/bays.toml':
|
||||
found = True
|
||||
data = config.encode()
|
||||
member.size = len(data)
|
||||
output.addfile(member, io.BytesIO(data))
|
||||
else:
|
||||
output.addfile(member, source.extractfile(member) if member.isfile() else None)
|
||||
assert found
|
||||
(work / 'calibrated').mkdir()
|
||||
with (work / 'calibrated.log').open('wb') as log:
|
||||
subprocess.run([str(project / 'image/build-recovery'), '--rootfs', str(work / 'calibrated.tar'), '--output-directory', str(work / 'calibrated')], check=True, stdout=log, stderr=subprocess.STDOUT)
|
||||
calibrated = work / 'calibrated.img'
|
||||
gpt(calibrated, [('FDS_RECOVERY', LINUX_FILESYSTEM, work / 'calibrated/recovery.erofs')])
|
||||
|
||||
bad = work / 'bad-system.erofs'
|
||||
bad.write_bytes(bytes(4096))
|
||||
bad_disk = work / 'bad-system.img'
|
||||
gpt(bad_disk, [('FDS_SYSTEM', LINUX_FILESYSTEM, bad)])
|
||||
with VM(work, 'damaged-system', base, extra=['-device', controller, '-drive', f'file={bad_disk},if=none,id=bad,format=raw,readonly=on', '-device', 'usb-storage,id=badusb,drive=bad,bus=xhci.0,port=2']) as vm:
|
||||
enter(vm, damaged=True)
|
||||
vm.capture('test "$(findmnt -nr -o FSTYPE /)" = erofs')
|
||||
finish(vm)
|
||||
print('PASS: unreadable SYSTEM does not prevent explicit independent recovery', flush=True)
|
||||
|
||||
def data_image(name, size=128, system=False, damaged=False):
|
||||
root = work / (name + '-files')
|
||||
(root / 'FDS').mkdir(parents=True)
|
||||
(root / 'FDS/CARTRIDGE.TOML').write_text(f'format=1\n[cartridge]\nid="fds.recovery.{name}"\nname="RECOVERY {name.upper()}"\nclass="data"\nversion="1"\n[media]\nwritable=true\n')
|
||||
(root / 'precious.txt').write_text('Keep this DATA payload unchanged.\n')
|
||||
if system:
|
||||
shutil.copyfile(source_system, root / 'replacement.img')
|
||||
fs = work / (name + '.ext4')
|
||||
with fs.open('xb') as stream:
|
||||
stream.truncate(size * 1024 * 1024)
|
||||
subprocess.run(['mke2fs', '-q', '-t', 'ext4', '-F', '-b', '4096', '-E', 'root_owner=1000:1000,lazy_itable_init=0,lazy_journal_init=0', '-d', str(root), str(fs)], check=True)
|
||||
if damaged:
|
||||
with (work / (name + '-corrupt.log')).open('wb') as log:
|
||||
subprocess.run(['debugfs', '-w', '-R', 'set_inode_field /precious.txt links_count 3', str(fs)], check=True, stdout=log, stderr=subprocess.STDOUT)
|
||||
checked = subprocess.run(['e2fsck', '-f', '-n', str(fs)], stdout=log, stderr=subprocess.STDOUT)
|
||||
assert checked.returncode == 4, checked.returncode
|
||||
disk = work / (name + '.img')
|
||||
layout = gpt(disk, [('FDS_DATA', LINUX_FILESYSTEM, fs)])
|
||||
return disk, layout
|
||||
|
||||
fault, fault_layout = data_image('fault', damaged=True)
|
||||
source, _ = data_image('source', size=768, system=True)
|
||||
clean, _ = data_image('clean')
|
||||
environment_files = work / 'environment-files'
|
||||
(environment_files / 'FDS').mkdir(parents=True)
|
||||
(environment_files / 'FDS/CARTRIDGE.TOML').write_text('format=1\n[cartridge]\nid="fds.recovery.environment"\nname="RECOVERY ENVIRONMENT TEST"\nclass="environment"\nversion="1"\n[media]\nwritable=false\n[activation]\nprofile="windowmaker"\n')
|
||||
environment_fs = work / 'environment.erofs'
|
||||
subprocess.run([str(project / 'tools/in-image-tools'), 'mkfs.erofs', '--quiet', '-b', '4096', '-T', '0', str(environment_fs), str(environment_files)], check=True)
|
||||
environment = work / 'environment.img'
|
||||
gpt(environment, [('FDS_ENVIRONMENT', LINUX_FILESYSTEM, environment_fs)])
|
||||
original = {p: digest(p) for p in [fault, source, clean, environment]}
|
||||
|
||||
with VM(work, 'maintenance', calibrated, extra=['-device', controller, '-netdev', 'user,id=recoverynet,restrict=on']) as vm:
|
||||
enter(vm)
|
||||
def attach(path, node, port, readonly=False):
|
||||
vm.qmp('blockdev-add', {'driver': 'raw', 'node-name': node, 'read-only': readonly, 'file': {'driver': 'file', 'filename': str(path)}})
|
||||
vm.qmp('device_add', {'driver': 'usb-storage', 'id': node + 'usb', 'drive': node, 'bus': 'xhci.0', 'port': str(port), 'serial': 'RECOVERY-' + node.upper()})
|
||||
def remove(node, port):
|
||||
vm.qmp('device_del', {'id': node + 'usb'})
|
||||
bay(vm, port, 'empty')
|
||||
attach(fault, 'fault', 2)
|
||||
attach(environment, 'environment', 3, True)
|
||||
vm.qmp('device_add', {'driver': 'usb-net', 'id': 'networkusb', 'netdev': 'recoverynet', 'bus': 'xhci.0', 'port': '4'})
|
||||
bay(vm, 2, 'mounted_read_only')
|
||||
bay(vm, 3, 'mounted_read_only')
|
||||
bay(vm, 4, 'hardware')
|
||||
vm.capture('test ! -e /data/FDS && ! pgrep -x Xorg && ! pgrep -x dhcpcd')
|
||||
profiles = query(vm, ['profiles'])['profiles']
|
||||
assert profiles['desktop'] == 'cli' and profiles['network'] == []
|
||||
vm.capture('s6-setuidgid fds fds recovery check 2', ok=False)
|
||||
preview = query(vm, ['recovery', 'repair', '2'])
|
||||
assert preview['checked'] is False and preview['confirmation']
|
||||
vm.capture('fds recovery repair 2 --confirm wrong', ok=False)
|
||||
assert digest(fault) == original[fault]
|
||||
vm.capture('cd /run/fds/media/02 && fds recovery check 2', ok=False)
|
||||
assert bay(vm, 2, 'mounted_read_only')['mount'] == '/run/fds/media/02'
|
||||
result = vm.capture('fds recovery check 2', ok=False)
|
||||
assert 'e2fsck returned 4' in result, result
|
||||
assert digest(fault) == original[fault], 'Read-only check changed DATA'
|
||||
fault_state = bay(vm, 2, 'error')
|
||||
vm.capture('s6-rc -l /run/s6-rc -d change cartridged && s6-rc -l /run/s6-rc -u change cartridged')
|
||||
assert 'recovery' in bay(vm, 2, 'error')['detail'].lower()
|
||||
vm.capture('fds data use 2', ok=False)
|
||||
repair = query(vm, ['recovery', 'repair', '2', '--confirm', preview['confirmation']])
|
||||
assert repair['checked'] and repair['repaired']
|
||||
(work / 'repair-result.json').write_text(json.dumps(repair, indent=2) + '\n')
|
||||
(work / 'repair.log').write_text(vm.capture('cat ' + shlex.quote(repair['log'])) + '\n')
|
||||
bay(vm, 2, 'safe')
|
||||
remove('fault', 2)
|
||||
attach(fault, 'repaired', 2)
|
||||
bay(vm, 2, 'mounted_read_only')
|
||||
assert vm.capture('cat /run/fds/media/02/precious.txt') == 'Keep this DATA payload unchanged.'
|
||||
vm.capture('fds recovery repair 2 --confirm ' + shlex.quote(preview['confirmation']), ok=False)
|
||||
checked = query(vm, ['recovery', 'check', '2'])
|
||||
assert checked['checked'] and not checked['repaired']
|
||||
remove('repaired', 2)
|
||||
remove('environment', 3)
|
||||
vm.qmp('device_del', {'id': 'networkusb'})
|
||||
bay(vm, 4, 'empty')
|
||||
print('PASS: no automatic DATA/GUI/network activation; root-only recovery, busy and stale-confirmation rejection, unchanged read-only check, actual ext4 repair, restart quarantine and payload preservation', flush=True)
|
||||
|
||||
attach(clean, 'clean', 2)
|
||||
bay(vm, 2, 'mounted_read_only')
|
||||
checked = query(vm, ['recovery', 'check', '2'])
|
||||
assert checked['checked'] and digest(clean) == original[clean]
|
||||
remove('clean', 2)
|
||||
attach(source, 'source', 1, True)
|
||||
bay(vm, 1, 'mounted_read_only')
|
||||
target = work / 'replacement-target.img'
|
||||
with target.open('xb') as stream:
|
||||
stream.truncate(640 * 1024 * 1024)
|
||||
attach(target, 'target', 4)
|
||||
bay(vm, 4, 'unrecognized_storage')
|
||||
wait(lambda: vm.capture('fds --json inspect BAY04 2>/dev/null || printf pending'), lambda value: value.startswith('{'))
|
||||
job = query(vm, ['burn', 'system', '/run/fds/media/01/replacement.img', 'BAY04'])
|
||||
completed = query(vm, ['burn', 'confirm', job['id'], job['confirmation']])
|
||||
assert completed['phase'] == 'complete'
|
||||
(work / 'burn-result.json').write_text(json.dumps(completed, indent=2) + '\n')
|
||||
remove('target', 4)
|
||||
# Attach an actual SYSTEM only after recovery has booted, then inspect it.
|
||||
attach(target, 'new-system', 4, True)
|
||||
inspected = bay(vm, 4, 'mounted_read_only')
|
||||
assert inspected['manifest']['cartridge']['class'] == 'system'
|
||||
vm.capture('fds recovery repair 4', ok=False)
|
||||
vm.capture('fds eject 4')
|
||||
vm.capture('test -z \"$(losetup -l -n -O NAME)\"')
|
||||
finish(vm)
|
||||
print('PASS: clean DATA check leaves bytes unchanged; recovery writes and verifies replacement SYSTEM, inspects it, excludes it from DATA repair, and shuts down through native s6', flush=True)
|
||||
|
||||
# Kill the real supervised checker while storage reads are throttled. The
|
||||
# throttle is a fault-observation fixture, never a production readiness delay.
|
||||
interrupted, _ = data_image('interrupted')
|
||||
interrupted_before = digest(interrupted)
|
||||
with VM(work, 'checker-parent-death', calibrated, extra=['-device', controller]) as vm:
|
||||
enter(vm)
|
||||
vm.qmp('blockdev-add', {'driver': 'raw', 'node-name': 'interruptdisk', 'file': {'driver': 'file', 'filename': str(interrupted)}})
|
||||
vm.qmp('object-add', {'qom-type': 'throttle-group', 'id': 'recoverylimit', 'limits': {}})
|
||||
vm.qmp('blockdev-add', {'driver': 'throttle', 'node-name': 'interruptlimited', 'throttle-group': 'recoverylimit', 'file': 'interruptdisk'})
|
||||
vm.qmp('device_add', {'driver': 'usb-storage', 'id': 'interruptusb', 'drive': 'interruptlimited', 'bus': 'xhci.0', 'port': '2', 'serial': 'RECOVERY-INTERRUPT'})
|
||||
bay(vm, 2, 'mounted_read_only')
|
||||
vm.capture('echo 3 >/proc/sys/vm/drop_caches')
|
||||
vm.qmp('qom-set', {'path': '/objects/recoverylimit', 'property': 'limits', 'value': {'bps-read': 16384}})
|
||||
vm.capture('fds recovery check 2 >/tmp/interrupted-check.log 2>&1 & echo $! >/tmp/check-client.pid')
|
||||
checker = wait(lambda: vm.capture('pgrep -x e2fsck || true'), lambda value: value.isdigit(), timeout=180)
|
||||
daemon = vm.capture('pgrep -x fds-cartridged')
|
||||
vm.capture('kill -KILL ' + daemon)
|
||||
vm.qmp('qom-set', {'path': '/objects/recoverylimit', 'property': 'limits', 'value': {'bps-read': 0}})
|
||||
wait(lambda: vm.capture('pgrep -x e2fsck || true'), lambda value: not value)
|
||||
wait(lambda: vm.capture('test -S /run/fds/control.sock && pgrep -x fds-cartridged || true'), lambda value: value.isdigit() and value != daemon)
|
||||
wait(lambda: vm.capture('fds --json bay 2 2>/dev/null || printf pending'), lambda value: value.startswith('{'))
|
||||
state = bay(vm, 2, 'error')
|
||||
assert 'recovery' in state['detail'].lower(), state
|
||||
vm.capture('fds data use 2', ok=False)
|
||||
assert digest(interrupted) == interrupted_before
|
||||
# A successful explicit retry clears only this insertion's incomplete check.
|
||||
checked = query(vm, ['recovery', 'check', '2'])
|
||||
assert checked['checked'] and not checked['repaired']
|
||||
bay(vm, 2, 'safe')
|
||||
wait(lambda: vm.capture('losetup -l -n -O NAME'), lambda value: not value)
|
||||
disk = vm.capture("""lsblk -dnpo NAME,SERIAL | awk '$2=="RECOVERY-INTERRUPT" {print $1}'""")
|
||||
assert disk.startswith('/dev/') and len(disk.splitlines()) == 1
|
||||
# eudev's short-lived blkid probes can legitimately make BLKRRPART busy.
|
||||
# Retry that observed condition, never wait for a guessed time interval.
|
||||
def reread():
|
||||
result = vm.capture('blockdev --rereadpt ' + shlex.quote(disk) + ' >/tmp/reread-result 2>&1; status=$?; cat /tmp/reread-result; printf "STATUS:%s" "$status"')
|
||||
assert result == 'STATUS:0' or 'Device or resource busy' in result, result
|
||||
return result == 'STATUS:0'
|
||||
for iteration in range(20):
|
||||
wait(reread, bool, timeout=20)
|
||||
query(vm, ['rescan'])
|
||||
bay(vm, 2, 'safe')
|
||||
bay(vm, 2, 'safe')
|
||||
vm.capture('s6-rc -l /run/s6-rc -d change cartridged && s6-rc -l /run/s6-rc -u change cartridged')
|
||||
bay(vm, 2, 'safe')
|
||||
(work / 'checker-interruption.json').write_text(json.dumps({'checker_pid': checker, 'daemon_pid': daemon, 'quarantine': state, 'retry': checked}, indent=2) + '\n')
|
||||
vm.capture('test -z \"$(losetup -l -n -O NAME)\"')
|
||||
finish(vm)
|
||||
print('PASS: real checker interruption kills the child, retains quarantine across daemon restart, preserves DATA bytes and allows an explicit verified retry', flush=True)
|
||||
|
||||
# Filesystem validation outside the guest and a separate boot of the written OS.
|
||||
part = fault_layout['partitions'][0]
|
||||
verified = work / 'repaired.ext4'
|
||||
with fault.open('rb') as src, verified.open('wb') as dst:
|
||||
src.seek(part['start'] * 512)
|
||||
remaining = part['payload_bytes']
|
||||
while remaining:
|
||||
chunk = src.read(min(remaining, 1024 * 1024))
|
||||
assert chunk
|
||||
dst.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
with (work / 'independent-fsck.log').open('wb') as log:
|
||||
subprocess.run(['e2fsck', '-f', '-n', str(verified)], check=True, stdout=log, stderr=subprocess.STDOUT)
|
||||
for path in [source, clean, environment]:
|
||||
assert digest(path) == original[path], f'Unexpected media change: {path}'
|
||||
with VM(work, 'boot-replacement', target, system_usb=True) as vm:
|
||||
vm.expect(rb'FDS> ')
|
||||
assert vm.capture('id -u') == '1000'
|
||||
assert query(vm, ['info'])['pid1'].endswith('s6-svscan')
|
||||
finish(vm)
|
||||
link = project / 'out/m12-recovery-latest.next'
|
||||
link.symlink_to(work.name)
|
||||
link.replace(project / 'out/m12-recovery-latest')
|
||||
print(f'PASS: independent fsck and ordinary-user boot of the SYSTEM written by recovery: {work}', flush=True)
|
||||
print('SKIP: physical Pi boot, display, power-loss and media-controller durability')
|
||||
Reference in New Issue
Block a user