update fds-flash tool

This commit is contained in:
2026-09-24 14:51:56 +08:00
parent 685b4135df
commit a14ea77145
18 changed files with 1761 additions and 10 deletions
+7 -3
View File
@@ -24,7 +24,7 @@ result=subprocess.run(command,capture_output=True,text=True)
(work/'install.log').write_text(result.stdout+result.stderr)
assert result.returncode==0,(result.stdout,result.stderr)
version=subprocess.check_output([str(project/'tools/version')],text=True).strip()
for tool in ['fds-cartridge','fds-emulator']:
for tool in ['fds-cartridge','fds-emulator','fds-flash']:
output=subprocess.check_output([str(root/'usr/bin'/tool),'--version'],text=True).strip()
assert output==f'{tool} {version}',output
subprocess.run([str(root/'usr/bin'/tool),'--help'],check=True,stdout=subprocess.DEVNULL)
@@ -37,6 +37,10 @@ assert not (manual/'.git').exists()
assert (root/'usr/share/licenses/fds-tools/RUST-NOTICES.txt').stat().st_size>0
installed=subprocess.check_output(['pacman','--config',str(config),'--root',str(root),'--dbpath',str(root/'var/lib/pacman'),'-Q','fds-tools'],text=True).strip()
assert installed==f'fds-tools {version}-1',installed
(work/'acceptance.json').write_text(json.dumps(dict(status='passed',package=str(package),installed=installed,installer='tools/install-workstation',host_install=False),indent=2)+'\n')
# makepkg may set different compiler flags from a direct Cargo build. Exercise
# the installed flasher itself against disposable files, not just its version.
subprocess.run(['python3',str(project/'tests/integration/workstation-flash.py'),
'--cli',str(root/'usr/bin/fds-flash')],check=True)
(work/'acceptance.json').write_text(json.dumps(dict(status='passed',package=str(package),installed=installed,installer='tools/install-workstation',installed_flash_verified=True,host_install=False),indent=2)+'\n')
(project/'out/arch-package-current.txt').write_text(str(work)+'\n')
print('PASS: real pacman installation and both installed commands:',work)
print('PASS: real pacman installation and all three installed commands:',work)
+8 -1
View File
@@ -88,6 +88,12 @@ class CleanupTests(unittest.TestCase):
self.file('out/manifests/dasung-s6-database.txt', str(self.root / 'out/dasung-s6.new000/compiled'))
self.file('out/workstation-images.new000/acceptance.json')
self.file('out/workstation-images-current.txt', 'out/workstation-images.new000\n')
self.file('out/workstation-flash.old000/disk.img')
self.file('out/workstation-flash.new000/acceptance.json')
self.file('out/workstation-flash-current.txt', 'out/workstation-flash.new000\n')
self.file('out/workstation-flash-vm.old000/fixture.tar')
self.file('out/workstation-flash-vm.new000/acceptance.json')
self.file('out/workstation-flash-vm-current.txt', 'out/workstation-flash-vm.new000\n')
self.file('out/m9-images.old000/root/program.img')
self.file('out/m9-images.new000/root/program.img')
os.utime(self.root / 'out/m9-images.old000', ns=(1, 1))
@@ -95,7 +101,8 @@ class CleanupTests(unittest.TestCase):
self.file('out/m8-vm.other0/disk.img')
(self.root / 'out/m8-vm.kept00/dependency').symlink_to('../m8-vm.other0')
self.assertEqual(self.selected(), {
'out/emu-test.old000', 'out/dasung-s6.old000', 'out/m9-images.old000'})
'out/emu-test.old000', 'out/dasung-s6.old000', 'out/m9-images.old000',
'out/workstation-flash.old000', 'out/workstation-flash-vm.old000'})
def test_symlinks_do_not_delete_external_content(self):
outside = Path(self.temporary.name) / 'outside'
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""Run the public flasher against disposable NVMe and USB disks in an ARM VM."""
import io
import json
from pathlib import Path
import shlex
import subprocess
import sys
import tarfile
import tempfile
import time
project = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(project/'tools'))
from image_formats import digest, gpt, LINUX_FILESYSTEM
from vm_test import VM
work = Path(tempfile.mkdtemp(prefix='workstation-flash-vm.', dir=project/'out'))
fixtures = Path((project/'out/workstation-flash-current.txt').read_text().strip())
binary = project/'target/aarch64-unknown-linux-musl/release/fds-flash'
archive = project/'out/rootfs-cli.tar'
for path in (binary, archive, fixtures/'internal.img', fixtures/'system.img'):
assert path.is_file(), path
inputs = [binary, archive, project/'out/kernel/boot/kernel_2712.img', project/'out/fds-initramfs.img']
(work/'inputs.sha256').write_text(''.join(f'{digest(p)} {p}\n' for p in inputs))
# This is an isolated fixture with a root serial console and the new workstation
# binary. It does not alter production login, rootfs archives or attached disks.
additions = {
'usr/libexec/fds/console-session': (b"#!/bin/bash\nexec env HOME=/root PS1='FLASH# ' bash --noprofile --norc\n", 0o755),
'usr/bin/fds-flash': (binary.read_bytes(), 0o755),
'usr/share/fds/flash-internal.img': ((fixtures/'internal.img').read_bytes(), 0o644),
'usr/share/fds/flash-system.img': ((fixtures/'system.img').read_bytes(), 0o644),
}
with tarfile.open(archive) as source, tarfile.open(work/'fixture.tar','w',format=tarfile.PAX_FORMAT) as target:
for member in source:
if member.name not in additions:
target.addfile(member, source.extractfile(member) if member.isfile() else None)
for name,(data,mode) in additions.items():
member=tarfile.TarInfo(name);member.size=len(data);member.mode=mode
target.addfile(member,io.BytesIO(data))
(work/'system').mkdir()
with (work/'build.log').open('w') as log:
subprocess.run([str(project/'image/build-system-cartridge'),'--rootfs',str(work/'fixture.tar'),
'--output-directory',str(work/'system')],check=True,stdout=log,stderr=subprocess.STDOUT)
filesystem=work/'mountable.ext4'
with filesystem.open('xb') as stream:stream.truncate(32*1024*1024)
subprocess.run([str(project/'tools/in-image-tools'),'mke2fs','-q','-F','-t','ext4',str(filesystem)],check=True)
disk=work/'nvme.img'
gpt(disk,[('FDS_DATA',LINUX_FILESYSTEM,filesystem)])
with (work/'4k.img').open('xb') as stream:stream.truncate(32*1024*1024)
extra=['-device','qemu-xhci,id=xhci,addr=05.0',
'-drive',f'file={disk},if=none,id=nvme,format=raw',
'-device','nvme,drive=nvme,serial=FDS-FLASH',
'-drive',f'file={work/"4k.img"},if=none,id=fourk,format=raw',
'-device','nvme,drive=fourk,serial=FDS-4K,logical_block_size=4096,physical_block_size=4096']
def wait(read, predicate, timeout=60):
deadline=time.monotonic()+timeout
while True:
result=read()
if predicate(result):return result
assert time.monotonic()<deadline,result
def cmd(vm, arguments, ok=True):
try:
return vm.capture(shlex.join(['fds-flash',*map(str,arguments)])+' 2>/tmp/flash-error',ok=ok)
except AssertionError as error:
raise AssertionError((arguments, vm.capture('cat /tmp/flash-error'))) from error
def preview(vm,image,path,ok=True):
text=cmd(vm,['--image',image,'--device',path,'--dry-run','--json'],ok)
return json.loads(text)['plan'] if ok else text
def write(vm,image,path,plan,ok=True):
return cmd(vm,['--image',image,'--device',path,'--unattended',
'--expect-target',plan['target_id'],'--sha256',plan['sha256'],'--json'],ok)
record = {}
with VM(work,'flash',work/'system/system.img',extra=extra) as vm:
vm.expect(rb'FLASH# ')
vm.capture('s6-rc -b -l /run/s6-rc -d change cartridged')
rows=json.loads(cmd(vm,['list','--json']))
nvme=next(row for row in rows if row['target'].get('serial')=='FDS-FLASH')['target']['path']
fourk=next(row for row in rows if row['target'].get('serial')=='FDS-4K')['target']['path']
source='/usr/share/fds/flash-internal.img'
before=digest(disk)
preview(vm,source,'/dev/vda',False)
assert 'read-only' in vm.capture('cat /tmp/flash-error')
preview(vm,source,fourk,False)
assert '512-byte' in vm.capture('cat /tmp/flash-error')
preview(vm,source,nvme+'p1',False)
assert 'whole disk' in vm.capture('cat /tmp/flash-error')
vm.capture('mkdir /tmp/flash-mounted; mount -o ro,noload '+shlex.quote(nvme+'p1')+' /tmp/flash-mounted')
preview(vm,source,nvme,False)
assert 'mounted' in vm.capture('cat /tmp/flash-error')
vm.capture('umount /tmp/flash-mounted')
assert digest(disk)==before
plan=preview(vm,source,nvme)
assert json.loads(write(vm,source,nvme,plan))['status']=='verified'
# Kernel partition names are independent of the asynchronous udev/blkid
# cache (the deliberately inert payloads are not mountable filesystems).
def labels():
return vm.capture('cat /sys/class/block/'+Path(nvme).name+"p*/uevent | sed -n 's/^PARTNAME=//p'").split()
labels=wait(labels,lambda names:names==['FDS_BOOT','FDS_RECOVERY','FDS_INTERNAL'])
assert labels==['FDS_BOOT','FDS_RECOVERY','FDS_INTERNAL'],labels
record['nvme_internal_write_and_kernel_partition_reread']=True
# Reinstalling the same offline FDS disk is an intended operation.
source='/usr/share/fds/flash-system.img'
plan=preview(vm,source,nvme)
assert json.loads(write(vm,source,nvme,plan))['status']=='verified'
assert wait(lambda:vm.capture('cat /sys/class/block/'+Path(nvme).name+"p*/uevent | sed -n 's/^PARTNAME=//p'").split(),
lambda names:names==['FDS_SYSTEM'])==['FDS_SYSTEM']
record['reflash_internal_disk_with_system']=True
record['mounted_partition_readonly_and_4k_refused']=True
# Hotplug only generated files into this VM. No host block paths are used.
def attach(name, rule=None):
path=work/(name+'.img')
with path.open('xb') as stream:stream.truncate(32*1024*1024)
vm.qmp('blockdev-add',{'driver':'raw','node-name':name,'file':{'driver':'file','filename':str(path)}})
top=name
if rule:
top=name+'-fault'
vm.qmp('blockdev-add',{'driver':'blkdebug','node-name':top,'image':name,
'inject-error':[{'event':'none','errno':5,'once':False,**rule}]})
vm.qmp('device_add',{'driver':'usb-storage','id':name,'drive':top,'bus':'xhci.0','port':'2','serial':name})
def rows():return json.loads(cmd(vm,['list','--json']))
entries=wait(rows,lambda entries:any(row['target'].get('serial')==name for row in entries))
return path,next(row['target']['path'] for row in entries if row['target'].get('serial')==name)
def detach(name):
vm.qmp('device_del',{'id':name})
wait(lambda:json.loads(cmd(vm,['list','--json'])),lambda entries:all(row['target'].get('serial')!=name for row in entries))
usb,path=attach('usb-first')
plan=preview(vm,source,path)
detach('usb-first')
usb2,path2=attach('usb-replacement')
write(vm,source,path2,plan,False)
assert 'Target identity' in vm.capture('cat /tmp/flash-error')
fresh=preview(vm,source,path2)
assert json.loads(write(vm,source,path2,fresh))['status']=='verified'
record['usb_replacement_refused_and_fresh_write_verified']=True
detach('usb-replacement')
for name,rule in [('write-error',{'iotype':'write','sector':2056}),
('flush-error',{'iotype':'flush'}),
('readback-error',{'iotype':'read','sector':2056})]:
target,path=attach(name,rule)
plan=preview(vm,source,path)
output=write(vm,source,path,plan,False)
error=vm.capture('cat /tmp/flash-error')
assert 'os error 5' in error,(name,error)
assert 'verified' not in output
record[name]=True
detach(name)
# The complete wizard must select the intended disk without CLI paths.
rows=json.loads(cmd(vm,['list','--json']))
selection=next(n+1 for n,row in enumerate(rows) if row['target']['path']==nvme)
vm.send('fds-flash; printf "\\nFLASH_WIZARD_STATUS:%s\\n" "$?"')
vm.expect(rb'Image path: ');vm.send(source)
vm.expect(rb'Disk number \(no default\): ');vm.send(str(selection))
vm.expect(rb' to proceed: ');vm.send('ERASE '+nvme)
vm.expect(rb'VERIFIED: image written')
vm.expect(rb'FLASH_WIZARD_STATUS:0\r?\n')
record['interactive_image_disk_selection_and_confirmation']=True
vm.capture('s6-rc -b -l /run/s6-rc -u change cartridged')
vm.send('fds poweroff');vm.expect(rb'reboot: Power down');assert vm.child.wait(timeout=20)==0
# The final successful NVMe write has a valid GPT on the actual backing file.
subprocess.run(['sfdisk','--verify',str(disk)],check=True)
record.update(status='passed',physical_hardware='not tested',source_binary_sha256=digest(binary))
(work/'acceptance.json').write_text(json.dumps(record,indent=2)+'\n')
(project/'out/workstation-flash-vm-current.txt').write_text(str(work)+'\n')
print('PASS: real virtual NVMe/USB flashing, protected disks, replacement and I/O failures:',work)
print('SKIP: physical Pi, drive/controller flush behavior and power-loss recovery')
+207
View File
@@ -0,0 +1,207 @@
#!/usr/bin/env python3
"""Exercise the actual interactive and unattended flasher using disposable files."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import pty
import selectors
import shutil
import struct
import subprocess
import sys
import tempfile
import time
project = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(project / 'tools'))
from image_formats import gpt, LINUX_FILESYSTEM, EFI_SYSTEM, digest
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--cli', type=Path, required=True)
args = parser.parse_args()
cli = args.cli.resolve()
work = Path(tempfile.mkdtemp(prefix='workstation-flash.', dir=project / 'out'))
log = (work / 'commands.log').open('w')
# Deliberately inert filesystem signatures in small independent GPT fixtures.
# These test the writer and table validation, not filesystem usability or boot.
parts = []
for name, kind, signature, offset in [
('FDS_BOOT', EFI_SYSTEM, b'FAT32 ', 82),
('FDS_RECOVERY', LINUX_FILESYSTEM, bytes.fromhex('e2e1f5e0'), 1024),
('FDS_INTERNAL', LINUX_FILESYSTEM, bytes.fromhex('53ef'), 1080),
]:
path = work / (name + '.bin')
with path.open('xb') as stream:
stream.truncate(1024 * 1024)
stream.seek(offset); stream.write(signature)
if name == 'FDS_BOOT':
stream.seek(11); stream.write(b'\0\x02')
stream.seek(510); stream.write(b'\x55\xaa')
stream.seek(8192); stream.write(b'known payload, not a bootable filesystem')
parts.append((name, kind, path))
internal = work / 'internal.img'
gpt(internal, parts)
system = work / 'system.img'
gpt(system, [('FDS_SYSTEM', LINUX_FILESYSTEM, parts[1][2])])
def invoke(arguments, ok=True):
result = subprocess.run([str(cli), *map(str, arguments)], capture_output=True, text=True, timeout=60)
log.write(repr(arguments) + '\n' + result.stdout + result.stderr); log.flush()
assert (result.returncode == 0) == ok, (arguments, result.returncode, result.stdout, result.stderr)
return result
def target(name, size):
path = work / name
with path.open('xb') as stream: stream.truncate(size)
return path
def preview(image, disk):
return json.loads(invoke(['--image', image, '--device', disk, '--file-target', '--dry-run', '--json']).stdout)['plan']
def unattended(image, disk, plan, **changes):
return ['--image', image, '--device', disk, '--file-target', '--unattended',
'--expect-target', changes.get('target_id', plan['target_id']),
'--sha256', changes.get('sha256', plan['sha256']), '--json']
def verify(image, disk):
subprocess.run(['sfdisk', '--verify', str(disk)], check=True, stdout=log, stderr=log)
original = json.loads(subprocess.check_output(['sfdisk', '--json', str(image)]))['partitiontable']
written = json.loads(subprocess.check_output(['sfdisk', '--json', str(disk)]))['partitiontable']
assert original['id'] == written['id']
assert written['lastlba'] == disk.stat().st_size // 512 - 34
with image.open('rb') as source, disk.open('rb') as output:
for a, b in zip(original['partitions'], written['partitions'], strict=True):
for key in ('start', 'size', 'type', 'uuid', 'name'): assert a[key] == b[key], (a,b)
source.seek(a['start']*512); output.seek(b['start']*512)
assert source.read(a['size']*512) == output.read(b['size']*512)
if disk.stat().st_size == image.stat().st_size: assert digest(image) == digest(disk)
for image in [internal, system]:
source_hash = digest(image)
for extra in [0, 512, 4*1024*1024]:
disk = target(f'{image.stem}-{extra}.target', image.stat().st_size + extra)
before = digest(disk)
plan = preview(image, disk)
assert digest(disk) == before
result = invoke(unattended(image, disk, plan, target_id='0'*64), False)
assert 'Target identity' in result.stderr and digest(disk) == before
result = invoke(unattended(image, disk, plan, sha256='0'*64), False)
assert 'SHA-256' in result.stderr and digest(disk) == before
assert json.loads(invoke(unattended(image, disk, plan)).stdout)['status'] == 'verified'
verify(image, disk)
invoke(unattended(image, disk, plan), False) # A file target's contents changed.
assert digest(image) == source_hash
for name, partitions in [
('data', [('FDS_DATA', LINUX_FILESYSTEM, parts[2][2])]),
('environment', [('FDS_ENVIRONMENT', LINUX_FILESYSTEM, parts[1][2])]),
('program', [('FDS_PROGRAM', LINUX_FILESYSTEM, parts[1][2])]),
('software', [(label, LINUX_FILESYSTEM, parts[1][2]) for label in ('FDS_METADATA','FDS_PAYLOAD02','FDS_PAYLOAD03')]),
]:
image=work/(name+'.img');gpt(image,partitions)
disk=target(name+'.target',image.stat().st_size+1024*1024)
plan=preview(image,disk)
assert json.loads(invoke(unattended(image,disk,plan)).stdout)['status']=='verified'
verify(image,disk)
# Invalid source geometry and destinations must fail without writing.
disk = target('protected.target', internal.stat().st_size)
before = digest(disk)
invoke(['--image', internal, '--device', disk, '--unattended'], False)
invoke(['--image', internal, '--device', disk, '--file-target'], False) # No terminal.
invoke(['--image', internal, '--device', disk, '--dry-run'], False) # Not a block device.
small = target('small.target', 1024)
invoke(['--image', internal, '--device', small, '--file-target', '--dry-run'], False)
invoke(['--image', internal, '--device', internal, '--file-target', '--dry-run'], False)
alias = work/'hardlink.img'; os.link(internal, alias)
invoke(['--image', internal, '--device', alias, '--file-target', '--dry-run'], False)
for name, change in [('bad-primary', 512+16), ('bad-backup', internal.stat().st_size-512+16),
('bad-table', 1024+56), ('bad-filesystem', 2048*512+82)]:
image = work/(name+'.img'); shutil.copyfile(internal,image)
with image.open('r+b') as stream:
stream.seek(change); byte=stream.read(1); stream.seek(change); stream.write(bytes([byte[0]^1]))
invoke(['--image', image, '--device', disk, '--file-target', '--dry-run'], False)
invoke(['--image', parts[0][2], '--device', disk, '--file-target', '--dry-run'], False)
assert digest(disk) == before
# Drive the real terminal workflow, including cancellation and changed inputs
# after the review is displayed. No shell evaluates the user's paths or input.
def interactive(name, reply, mutate=None, ok=True, include_image=True):
disk = target(name+'.target', system.stat().st_size + 1024*1024)
image = work/(name+'.img'); shutil.copyfile(system,image)
before = digest(disk)
master, slave = pty.openpty()
command = [str(cli), '--device', str(disk), '--file-target', '--json']
if include_image: command += ['--image', str(image)]
child = subprocess.Popen(command, stdin=slave, stderr=slave, stdout=subprocess.PIPE, text=True)
os.close(slave)
selector=selectors.DefaultSelector();selector.register(master,selectors.EVENT_READ)
output=bytearray()
def until(marker):
deadline=time.monotonic()+20
while marker not in output:
assert time.monotonic()<deadline, output
for _,_ in selector.select(max(0,deadline-time.monotonic())):
chunk=os.read(master,65536); assert chunk; output.extend(chunk)
try:
if not include_image:
until(b'Image path: ');os.write(master,(str(image)+'\n').encode())
until(b' to proceed: ')
if mutate: mutate(image,disk)
expected = 'ERASE '+str(disk)
os.write(master,((expected if reply else 'cancel')+'\n').encode())
deadline=time.monotonic()+30
while child.poll() is None:
assert time.monotonic()<deadline, output
for _,_ in selector.select(1):
try: output.extend(os.read(master,65536))
except OSError: pass
stdout=child.stdout.read()
assert (child.returncode==0)==ok, (stdout,output)
if ok:
assert json.loads(stdout)['status']=='verified'
verify(image,disk)
elif mutate is None or name=='changed-source': assert digest(disk)==before
(work/(name+'.terminal.log')).write_bytes(output+stdout.encode())
finally:
if child.poll() is None: child.kill();child.wait()
child.stdout.close();selector.close();os.close(master)
interactive('interactive-cancel',False,ok=False)
interactive('interactive-write',True,include_image=False)
def change_source(image,disk):
with image.open('r+b') as stream:stream.seek(1024*1024+8192);stream.write(b'CHANGED')
interactive('changed-source',True,change_source,ok=False)
def change_target(image,disk):
disk.unlink()
with disk.open('xb') as stream:stream.truncate(image.stat().st_size+1024*1024)
interactive('replaced-target',True,change_target,ok=False)
actual_images = []
for name in ('fds-internal.img', 'fds-system-cli.img'):
image=project/'out'/name
if image.is_file():
disk=target('actual-'+name+'.target',image.stat().st_size)
plan=preview(image,disk)
assert plan['image']['bytes']==image.stat().st_size
actual_images.append(dict(path=str(image.resolve()),sha256=plan['sha256']))
else:
print('SKIP: build image not available for read-only inspection:',image)
record=dict(status='passed',cli=str(cli),cli_sha256=digest(cli),internal_and_all_cartridge_classes=True,interactive_terminal=True,
unattended_bindings=True,exact_larger_and_overlapping_gpt=True,independent_sfdisk=True,
partition_payloads_unchanged=True,invalid_sources_and_targets_rejected=True,
changed_source_and_target_rejected=True,actual_build_images_inspected=actual_images,physical_disks_written=False)
(work/'acceptance.json').write_text(json.dumps(record,indent=2)+'\n')
(project/'out/workstation-flash-current.txt').write_text(str(work)+'\n')
print('PASS: interactive/unattended flash, rejection paths, payload readback and relocated GPT:',work)
print('SKIP: physical devices; fixtures are disposable files with inert filesystem signatures')