300 lines
17 KiB
Python
300 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Boot independent recovery, preserve media by default, repair and replace media."""
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
|
|
project = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(project / 'tools'))
|
|
from image_formats import digest, gpt, LINUX_FILESYSTEM
|
|
from vm_test import VM
|
|
|
|
work = Path(tempfile.mkdtemp(prefix='m12-recovery.', dir=project / 'out'))
|
|
rootfs = (project / 'out/rootfs-recovery.tar').resolve(strict=True)
|
|
recovery = (project / 'out/fds-recovery.img').resolve(strict=True)
|
|
source_system = (project / 'out/fds-system-cli.img').resolve(strict=True)
|
|
(work / 'inputs.sha256').write_text(''.join(f'{digest(p)} {p}\n' for p in [rootfs, recovery, source_system, project / 'out/fds-initramfs.img', project / 'out/kernel/boot/kernel_2712.img']))
|
|
controller = 'qemu-xhci,id=xhci,addr=05.0'
|
|
# Build two independent EROFS outputs and require identical payload bytes.
|
|
for name in ['repeat-a', 'repeat-b']:
|
|
output = work / name
|
|
output.mkdir()
|
|
with (work / (name + '.log')).open('wb') as log:
|
|
subprocess.run([str(project / 'image/build-recovery'), '--rootfs', str(rootfs), '--output-directory', str(output)], check=True, stdout=log, stderr=subprocess.STDOUT)
|
|
assert digest(work / 'repeat-a/recovery.erofs') == digest(recovery) == digest(work / 'repeat-b/recovery.erofs')
|
|
base = work / 'recovery-base.img'
|
|
gpt(base, [('FDS_RECOVERY', LINUX_FILESYSTEM, recovery)])
|
|
|
|
def query(vm, args):
|
|
try:
|
|
return json.loads(vm.capture('fds --json ' + shlex.join(args), timeout=600 if args[0] == 'burn' else 180))
|
|
except AssertionError:
|
|
(work / 'last-recovery-checker.log').write_text(vm.capture('cat /run/fds/recovery/*.log 2>/dev/null || true') + '\n')
|
|
raise
|
|
|
|
def wait(operation, condition, timeout=60):
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
result = operation()
|
|
if condition(result):
|
|
return result
|
|
assert time.monotonic() < deadline, result
|
|
|
|
def bay(vm, n, state):
|
|
try:
|
|
return wait(lambda: query(vm, ['bay', str(n)])['bays'][0], lambda row: row['state'] == state)
|
|
except AssertionError:
|
|
(work / (vm.name + '-failure-state.log')).write_text(vm.capture('cat /run/fds/ejected/* /run/fds/data-sessions/* 2>/dev/null; cat /proc/self/mountinfo; fds inspect BAY02; tail -n 80 /run/log/cartridged/current; dmesg | tail -n 40') + '\n')
|
|
raise
|
|
|
|
def enter(vm, damaged=False):
|
|
vm.expect(rb'SYSTEM CANNOT BE USED' if damaged else rb'FDS_SYSTEM MEDIA NOT PRESENT')
|
|
vm.send('recovery')
|
|
vm.expect(rb'RECOVERY# ')
|
|
assert vm.capture('id -u') == '0'
|
|
assert vm.capture('cat /usr/share/fds/image-profile') == 'recovery'
|
|
wait(lambda: vm.capture('test -S /run/fds/control.sock && echo ready || echo pending'), lambda value: value == 'ready')
|
|
|
|
def finish(vm):
|
|
vm.send('fds poweroff')
|
|
vm.expect(rb'reboot: Power down', timeout=120)
|
|
assert vm.child.wait(timeout=20) == 0
|
|
|
|
with VM(work, 'without-system', base, extra=['-device', controller, '-device', 'usb-kbd,bus=xhci.0,port=4']) as vm:
|
|
enter(vm)
|
|
info = query(vm, ['info'])
|
|
assert info['pid1'].endswith('s6-svscan') and info['target'] == 'aarch64 static-musl'
|
|
vm.capture('test "$(findmnt -nr -o FSTYPE /)" = erofs && findmnt -nr -o OPTIONS / | grep -qw ro')
|
|
vm.capture('touch /etc/recovery-write-probe', ok=False)
|
|
vm.capture('pgrep -x dasungd && command -v bash ls mount e2fsck xbps-query fds-burn fds-inspect && test -s /usr/share/doc/fds/recovery.md')
|
|
report = wait(lambda: query(vm, ['topology']), lambda row: any('03' in d['interfaces'] for d in row['unmapped']))
|
|
hub = next(d['topology'] for d in report['unmapped'] if '03' in d['interfaces']).rsplit('/', 1)[0]
|
|
vm.capture('bash /usr/share/fds/capture-hardware /tmp/recovery-capture && cd /tmp/recovery-capture && sha256sum -c SHA256SUMS')
|
|
(work / 'capture-status.tsv').write_text(vm.capture('cat /tmp/recovery-capture/status.tsv') + '\n')
|
|
(work / 'boot-without-system.json').write_text(json.dumps(query(vm, ['boot-profile']), indent=2) + '\n')
|
|
finish(vm)
|
|
print('PASS: separate reproducible recovery payload; missing-SYSTEM selection, read-only root, native s6, local root console, base Dasung and diagnostic capture', flush=True)
|
|
|
|
config = ''
|
|
for name, identity in [('usb2', hub), ('usb3', hub.replace(':usb2', ':usb3'))]:
|
|
config += f'[{name}]\nhub={json.dumps(identity)}\n[{name}.ports]\n' + ''.join(f'{n}={n}\n' for n in range(1, 13))
|
|
# Only the virtual bay calibration differs from the independently packaged root.
|
|
with tarfile.open(rootfs) as source, tarfile.open(work / 'calibrated.tar', 'w', format=tarfile.PAX_FORMAT) as output:
|
|
found = False
|
|
for member in source:
|
|
if member.name == 'etc/fds/bays.toml':
|
|
found = True
|
|
data = config.encode()
|
|
member.size = len(data)
|
|
output.addfile(member, io.BytesIO(data))
|
|
else:
|
|
output.addfile(member, source.extractfile(member) if member.isfile() else None)
|
|
assert found
|
|
(work / 'calibrated').mkdir()
|
|
with (work / 'calibrated.log').open('wb') as log:
|
|
subprocess.run([str(project / 'image/build-recovery'), '--rootfs', str(work / 'calibrated.tar'), '--output-directory', str(work / 'calibrated')], check=True, stdout=log, stderr=subprocess.STDOUT)
|
|
calibrated = work / 'calibrated.img'
|
|
gpt(calibrated, [('FDS_RECOVERY', LINUX_FILESYSTEM, work / 'calibrated/recovery.erofs')])
|
|
|
|
bad = work / 'bad-system.erofs'
|
|
bad.write_bytes(bytes(4096))
|
|
bad_disk = work / 'bad-system.img'
|
|
gpt(bad_disk, [('FDS_SYSTEM', LINUX_FILESYSTEM, bad)])
|
|
with VM(work, 'damaged-system', base, extra=['-device', controller, '-drive', f'file={bad_disk},if=none,id=bad,format=raw,readonly=on', '-device', 'usb-storage,id=badusb,drive=bad,bus=xhci.0,port=2']) as vm:
|
|
enter(vm, damaged=True)
|
|
vm.capture('test "$(findmnt -nr -o FSTYPE /)" = erofs')
|
|
finish(vm)
|
|
print('PASS: unreadable SYSTEM does not prevent explicit independent recovery', flush=True)
|
|
|
|
def data_image(name, size=128, system=False, damaged=False):
|
|
root = work / (name + '-files')
|
|
(root / 'FDS').mkdir(parents=True)
|
|
(root / 'FDS/CARTRIDGE.TOML').write_text(f'format=1\n[cartridge]\nid="fds.recovery.{name}"\nname="RECOVERY {name.upper()}"\nclass="data"\nversion="1"\n[media]\nwritable=true\n')
|
|
(root / 'precious.txt').write_text('Keep this DATA payload unchanged.\n')
|
|
if system:
|
|
shutil.copyfile(source_system, root / 'replacement.img')
|
|
fs = work / (name + '.ext4')
|
|
with fs.open('xb') as stream:
|
|
stream.truncate(size * 1024 * 1024)
|
|
subprocess.run(['mke2fs', '-q', '-t', 'ext4', '-F', '-b', '4096', '-E', 'root_owner=1000:1000,lazy_itable_init=0,lazy_journal_init=0', '-d', str(root), str(fs)], check=True)
|
|
if damaged:
|
|
with (work / (name + '-corrupt.log')).open('wb') as log:
|
|
subprocess.run(['debugfs', '-w', '-R', 'set_inode_field /precious.txt links_count 3', str(fs)], check=True, stdout=log, stderr=subprocess.STDOUT)
|
|
checked = subprocess.run(['e2fsck', '-f', '-n', str(fs)], stdout=log, stderr=subprocess.STDOUT)
|
|
assert checked.returncode == 4, checked.returncode
|
|
disk = work / (name + '.img')
|
|
layout = gpt(disk, [('FDS_DATA', LINUX_FILESYSTEM, fs)])
|
|
return disk, layout
|
|
|
|
fault, fault_layout = data_image('fault', damaged=True)
|
|
source, _ = data_image('source', size=768, system=True)
|
|
clean, _ = data_image('clean')
|
|
environment_files = work / 'environment-files'
|
|
(environment_files / 'FDS').mkdir(parents=True)
|
|
(environment_files / 'FDS/CARTRIDGE.TOML').write_text('format=1\n[cartridge]\nid="fds.recovery.environment"\nname="RECOVERY ENVIRONMENT TEST"\nclass="environment"\nversion="1"\n[media]\nwritable=false\n[activation]\nprofile="windowmaker"\n')
|
|
environment_fs = work / 'environment.erofs'
|
|
subprocess.run([str(project / 'tools/in-image-tools'), 'mkfs.erofs', '--quiet', '-b', '4096', '-T', '0', str(environment_fs), str(environment_files)], check=True)
|
|
environment = work / 'environment.img'
|
|
gpt(environment, [('FDS_ENVIRONMENT', LINUX_FILESYSTEM, environment_fs)])
|
|
original = {p: digest(p) for p in [fault, source, clean, environment]}
|
|
|
|
with VM(work, 'maintenance', calibrated, extra=['-device', controller, '-netdev', 'user,id=recoverynet,restrict=on']) as vm:
|
|
enter(vm)
|
|
def attach(path, node, port, readonly=False):
|
|
vm.qmp('blockdev-add', {'driver': 'raw', 'node-name': node, 'read-only': readonly, 'file': {'driver': 'file', 'filename': str(path)}})
|
|
vm.qmp('device_add', {'driver': 'usb-storage', 'id': node + 'usb', 'drive': node, 'bus': 'xhci.0', 'port': str(port), 'serial': 'RECOVERY-' + node.upper()})
|
|
def remove(node, port):
|
|
vm.qmp('device_del', {'id': node + 'usb'})
|
|
bay(vm, port, 'empty')
|
|
attach(fault, 'fault', 2)
|
|
attach(environment, 'environment', 3, True)
|
|
vm.qmp('device_add', {'driver': 'usb-net', 'id': 'networkusb', 'netdev': 'recoverynet', 'bus': 'xhci.0', 'port': '4'})
|
|
bay(vm, 2, 'mounted_read_only')
|
|
bay(vm, 3, 'mounted_read_only')
|
|
bay(vm, 4, 'hardware')
|
|
vm.capture('test ! -e /data/FDS && ! pgrep -x Xorg && ! pgrep -x dhcpcd')
|
|
profiles = query(vm, ['profiles'])['profiles']
|
|
assert profiles['desktop'] == 'cli' and profiles['network'] == []
|
|
vm.capture('s6-setuidgid fds fds recovery check 2', ok=False)
|
|
preview = query(vm, ['recovery', 'repair', '2'])
|
|
assert preview['checked'] is False and preview['confirmation']
|
|
vm.capture('fds recovery repair 2 --confirm wrong', ok=False)
|
|
assert digest(fault) == original[fault]
|
|
vm.capture('cd /run/fds/media/02 && fds recovery check 2', ok=False)
|
|
assert bay(vm, 2, 'mounted_read_only')['mount'] == '/run/fds/media/02'
|
|
result = vm.capture('fds recovery check 2', ok=False)
|
|
assert 'e2fsck returned 4' in result, result
|
|
assert digest(fault) == original[fault], 'Read-only check changed DATA'
|
|
fault_state = bay(vm, 2, 'error')
|
|
vm.capture('s6-rc -l /run/s6-rc -d change cartridged && s6-rc -l /run/s6-rc -u change cartridged')
|
|
assert 'recovery' in bay(vm, 2, 'error')['detail'].lower()
|
|
vm.capture('fds data use 2', ok=False)
|
|
repair = query(vm, ['recovery', 'repair', '2', '--confirm', preview['confirmation']])
|
|
assert repair['checked'] and repair['repaired']
|
|
(work / 'repair-result.json').write_text(json.dumps(repair, indent=2) + '\n')
|
|
(work / 'repair.log').write_text(vm.capture('cat ' + shlex.quote(repair['log'])) + '\n')
|
|
bay(vm, 2, 'safe')
|
|
remove('fault', 2)
|
|
attach(fault, 'repaired', 2)
|
|
bay(vm, 2, 'mounted_read_only')
|
|
assert vm.capture('cat /run/fds/media/02/precious.txt') == 'Keep this DATA payload unchanged.'
|
|
vm.capture('fds recovery repair 2 --confirm ' + shlex.quote(preview['confirmation']), ok=False)
|
|
checked = query(vm, ['recovery', 'check', '2'])
|
|
assert checked['checked'] and not checked['repaired']
|
|
remove('repaired', 2)
|
|
remove('environment', 3)
|
|
vm.qmp('device_del', {'id': 'networkusb'})
|
|
bay(vm, 4, 'empty')
|
|
print('PASS: no automatic DATA/GUI/network activation; root-only recovery, busy and stale-confirmation rejection, unchanged read-only check, actual ext4 repair, restart quarantine and payload preservation', flush=True)
|
|
|
|
attach(clean, 'clean', 2)
|
|
bay(vm, 2, 'mounted_read_only')
|
|
checked = query(vm, ['recovery', 'check', '2'])
|
|
assert checked['checked'] and digest(clean) == original[clean]
|
|
remove('clean', 2)
|
|
attach(source, 'source', 1, True)
|
|
bay(vm, 1, 'mounted_read_only')
|
|
target = work / 'replacement-target.img'
|
|
with target.open('xb') as stream:
|
|
stream.truncate(640 * 1024 * 1024)
|
|
attach(target, 'target', 4)
|
|
bay(vm, 4, 'unrecognized_storage')
|
|
wait(lambda: vm.capture('fds --json inspect BAY04 2>/dev/null || printf pending'), lambda value: value.startswith('{'))
|
|
job = query(vm, ['burn', 'system', '/run/fds/media/01/replacement.img', 'BAY04'])
|
|
completed = query(vm, ['burn', 'confirm', job['id'], job['confirmation']])
|
|
assert completed['phase'] == 'complete'
|
|
(work / 'burn-result.json').write_text(json.dumps(completed, indent=2) + '\n')
|
|
remove('target', 4)
|
|
# Attach an actual SYSTEM only after recovery has booted, then inspect it.
|
|
attach(target, 'new-system', 4, True)
|
|
inspected = bay(vm, 4, 'mounted_read_only')
|
|
assert inspected['manifest']['cartridge']['class'] == 'system'
|
|
vm.capture('fds recovery repair 4', ok=False)
|
|
vm.capture('fds eject 4')
|
|
vm.capture('test -z \"$(losetup -l -n -O NAME)\"')
|
|
finish(vm)
|
|
print('PASS: clean DATA check leaves bytes unchanged; recovery writes and verifies replacement SYSTEM, inspects it, excludes it from DATA repair, and shuts down through native s6', flush=True)
|
|
|
|
# Kill the real supervised checker while storage reads are throttled. The
|
|
# throttle is a fault-observation fixture, never a production readiness delay.
|
|
interrupted, _ = data_image('interrupted')
|
|
interrupted_before = digest(interrupted)
|
|
with VM(work, 'checker-parent-death', calibrated, extra=['-device', controller]) as vm:
|
|
enter(vm)
|
|
vm.qmp('blockdev-add', {'driver': 'raw', 'node-name': 'interruptdisk', 'file': {'driver': 'file', 'filename': str(interrupted)}})
|
|
vm.qmp('object-add', {'qom-type': 'throttle-group', 'id': 'recoverylimit', 'limits': {}})
|
|
vm.qmp('blockdev-add', {'driver': 'throttle', 'node-name': 'interruptlimited', 'throttle-group': 'recoverylimit', 'file': 'interruptdisk'})
|
|
vm.qmp('device_add', {'driver': 'usb-storage', 'id': 'interruptusb', 'drive': 'interruptlimited', 'bus': 'xhci.0', 'port': '2', 'serial': 'RECOVERY-INTERRUPT'})
|
|
bay(vm, 2, 'mounted_read_only')
|
|
vm.capture('echo 3 >/proc/sys/vm/drop_caches')
|
|
vm.qmp('qom-set', {'path': '/objects/recoverylimit', 'property': 'limits', 'value': {'bps-read': 16384}})
|
|
vm.capture('fds recovery check 2 >/tmp/interrupted-check.log 2>&1 & echo $! >/tmp/check-client.pid')
|
|
checker = wait(lambda: vm.capture('pgrep -x e2fsck || true'), lambda value: value.isdigit(), timeout=180)
|
|
daemon = vm.capture('pgrep -x fds-cartridged')
|
|
vm.capture('kill -KILL ' + daemon)
|
|
vm.qmp('qom-set', {'path': '/objects/recoverylimit', 'property': 'limits', 'value': {'bps-read': 0}})
|
|
wait(lambda: vm.capture('pgrep -x e2fsck || true'), lambda value: not value)
|
|
wait(lambda: vm.capture('test -S /run/fds/control.sock && pgrep -x fds-cartridged || true'), lambda value: value.isdigit() and value != daemon)
|
|
wait(lambda: vm.capture('fds --json bay 2 2>/dev/null || printf pending'), lambda value: value.startswith('{'))
|
|
state = bay(vm, 2, 'error')
|
|
assert 'recovery' in state['detail'].lower(), state
|
|
vm.capture('fds data use 2', ok=False)
|
|
assert digest(interrupted) == interrupted_before
|
|
# A successful explicit retry clears only this insertion's incomplete check.
|
|
checked = query(vm, ['recovery', 'check', '2'])
|
|
assert checked['checked'] and not checked['repaired']
|
|
bay(vm, 2, 'safe')
|
|
wait(lambda: vm.capture('losetup -l -n -O NAME'), lambda value: not value)
|
|
disk = vm.capture("""lsblk -dnpo NAME,SERIAL | awk '$2=="RECOVERY-INTERRUPT" {print $1}'""")
|
|
assert disk.startswith('/dev/') and len(disk.splitlines()) == 1
|
|
# eudev's short-lived blkid probes can legitimately make BLKRRPART busy.
|
|
# Retry that observed condition, never wait for a guessed time interval.
|
|
def reread():
|
|
result = vm.capture('blockdev --rereadpt ' + shlex.quote(disk) + ' >/tmp/reread-result 2>&1; status=$?; cat /tmp/reread-result; printf "STATUS:%s" "$status"')
|
|
assert result == 'STATUS:0' or 'Device or resource busy' in result, result
|
|
return result == 'STATUS:0'
|
|
for iteration in range(20):
|
|
wait(reread, bool, timeout=20)
|
|
query(vm, ['rescan'])
|
|
bay(vm, 2, 'safe')
|
|
bay(vm, 2, 'safe')
|
|
vm.capture('s6-rc -l /run/s6-rc -d change cartridged && s6-rc -l /run/s6-rc -u change cartridged')
|
|
bay(vm, 2, 'safe')
|
|
(work / 'checker-interruption.json').write_text(json.dumps({'checker_pid': checker, 'daemon_pid': daemon, 'quarantine': state, 'retry': checked}, indent=2) + '\n')
|
|
vm.capture('test -z \"$(losetup -l -n -O NAME)\"')
|
|
finish(vm)
|
|
print('PASS: real checker interruption kills the child, retains quarantine across daemon restart, preserves DATA bytes and allows an explicit verified retry', flush=True)
|
|
|
|
# Filesystem validation outside the guest and a separate boot of the written OS.
|
|
part = fault_layout['partitions'][0]
|
|
verified = work / 'repaired.ext4'
|
|
with fault.open('rb') as src, verified.open('wb') as dst:
|
|
src.seek(part['start'] * 512)
|
|
remaining = part['payload_bytes']
|
|
while remaining:
|
|
chunk = src.read(min(remaining, 1024 * 1024))
|
|
assert chunk
|
|
dst.write(chunk)
|
|
remaining -= len(chunk)
|
|
with (work / 'independent-fsck.log').open('wb') as log:
|
|
subprocess.run(['e2fsck', '-f', '-n', str(verified)], check=True, stdout=log, stderr=subprocess.STDOUT)
|
|
for path in [source, clean, environment]:
|
|
assert digest(path) == original[path], f'Unexpected media change: {path}'
|
|
with VM(work, 'boot-replacement', target, system_usb=True) as vm:
|
|
vm.expect(rb'FDS> ')
|
|
assert vm.capture('id -u') == '1000'
|
|
assert query(vm, ['info'])['pid1'].endswith('s6-svscan')
|
|
finish(vm)
|
|
link = project / 'out/m12-recovery-latest.next'
|
|
link.symlink_to(work.name)
|
|
link.replace(project / 'out/m12-recovery-latest')
|
|
print(f'PASS: independent fsck and ordinary-user boot of the SYSTEM written by recovery: {work}', flush=True)
|
|
print('SKIP: physical Pi boot, display, power-loss and media-controller durability')
|