Support native SD storage and consolidate fds-flash tooling

Accept native SD cards for internal settings alongside legacy NVMe,
reject USB ancestry and ambiguous disks, and preserve read-only boot
loading with explicit recovery writes. Require built-in MMC drivers,
validate cached kernel configuration, and provide SD-only, SD-first and
USB-first EEPROM profiles.

Expose typed create and inspect commands through fds-flash, reuse the
existing cartridge creation code, and document the optional e2fsprogs
package dependency. Extend storage, EEPROM, flashing and hardware-test
coverage and advance the wiki reference to the published SD guide.

Validation: rootfs checks for CLI/development, boot matrix, internal
storage, EEPROM, flash, workstation, emulator, make check and wiki checks
passed. The full internal suite passed on an unchanged rerun after one
unexplained VM shutdown stall. Standalone init-test was blocked by its
unavailable pinned upstream kernel. Physical Pi checks remain pending.

tests/hardware/session-template.json

@@ -17,7 +17,12 @@
     "power_source": null,
     "battery_state": null,
     "hub_models": [],
-    "display_model": "Dasung Paperlike 13K"
+    "display_model": "Dasung Paperlike 13K",
+    "internal_medium": null,
+    "sd_card_model": null,
+    "sd_card_capacity_bytes": null,
+    "maintenance_usb_model": null,
+    "boot_order": null
   },
   "bay_map": [
     {
@@ -197,6 +202,38 @@
       "samples": [],
       "errors": [],
       "notes": null
+    },
+    {
+      "name": "sd_usb_system_boot",
+      "status": "not_run",
+      "captures": [],
+      "samples": [],
+      "errors": [],
+      "notes": null
+    },
+    {
+      "name": "sd_recovery_persistence",
+      "status": "not_run",
+      "captures": [],
+      "samples": [],
+      "errors": [],
+      "notes": null
+    },
+    {
+      "name": "maintenance_usb_override",
+      "status": "not_run",
+      "captures": [],
+      "samples": [],
+      "errors": [],
+      "notes": null
+    },
+    {
+      "name": "return_to_fds_sd",
+      "status": "not_run",
+      "captures": [],
+      "samples": [],
+      "errors": [],
+      "notes": null
     }
   ],
   "timing": {

tests/integration/m12-eeprom.py

@@ -27,11 +27,12 @@ def run(arguments,ok=True):
     assert (result.returncode==0)==ok,(arguments,result.stdout,result.stderr)
     return result
 
-for profile in ['production','development']:
+for profile, boot_order in [('production','0xf1'),('development','0xf41'),('maintenance','0xf14')]:
     output=work/profile
     run(['--profile',profile,'--output-directory',output])
     manifest=json.loads((output/'manifest.json').read_text())
     assert manifest['hardware_modified'] is False and manifest['custom_inputs_provided'] is False
+    assert manifest['settings']['BOOT_ORDER'] == boot_order
     for name,digest in manifest['files'].items():assert sha256(output/name)==digest
     image=upstream.BootloaderImage(str(output/'configured.bin'))
     for name,data in protected.items():assert image.get_file(name)==data,(profile,name)
@@ -47,7 +48,7 @@ saved.write_text('[all]\nBOOT_ORDER=0xf461\nCUSTOM_BOARD_SETTING=retained\n[gpio
 output=work/'custom'
 run(['--current-config',saved,'--output-directory',output])
 changed=(output/'configured.conf').read_text()
-assert changed.count('BOOT_ORDER=')==1 and 'BOOT_ORDER=0xf6' in changed
+assert changed.count('BOOT_ORDER=')==1 and 'BOOT_ORDER=0xf1' in changed
 assert 'CUSTOM_BOARD_SETTING=retained' in changed and '[gpio8=0]\nOTHER_SETTING=unchanged' in changed
 rollback=upstream.BootloaderImage(str(output/'rollback.bin')).get_file('bootconf.txt').decode()
 assert rollback==saved.read_text(),'Rollback lost the original conditional settings'

tests/integration/m12-internal.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Exercise the packaged machine settings service against disposable virtual NVMe."""
+"""Exercise packaged machine settings against disposable native SD and NVMe."""
 import json
 from pathlib import Path
 import shlex
@@ -41,6 +41,22 @@ def extra(path, node='internal', readonly=False):
     return ['-drive', f'file={path},if=none,id={node},format=raw' + (',readonly=on' if readonly else ''),
             '-device', f'nvme,drive={node},serial=FDS-{node.upper()}']
 
+def sd_extra(path):
+    return ['-drive', f'file={path},if=none,id=internal_sd,format=raw',
+            '-device', 'sdhci-pci,id=sdhci',
+            '-device', 'sd-card,drive=internal_sd']
+
+def sd_capacity(path):
+    # QEMU SD cards require a power-of-two capacity. Sparse extension and GPT
+    # relocation model flashing the small image onto a larger native card.
+    with path.open('r+b') as output:
+        output.truncate(2 * 1024**3)
+    subprocess.run(['sfdisk', '--relocate', 'gpt-bak-std', str(path)], check=True,
+                   stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    subprocess.run(['sfdisk', '--verify', str(path)], check=True,
+                   stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    return path
+
 def wait(operation, condition, timeout=60):
     deadline = time.monotonic() + timeout
     while True:
@@ -128,27 +144,62 @@ with VM(work, 'persistent-settings', empty, extra=[*extra(machine), '-device', c
     finish(vm)
 print('PASS: independent reboot loads saved bay/catalog/name settings and retrieves the previous boot record', flush=True)
 
+sd = sd_capacity(copy_image('native-sd'))
+original_sd = digest(sd)
+with VM(work, 'sd-install-settings', empty, extra=sd_extra(sd)) as vm:
+    status = enter(vm)
+    assert status['source'] == 'internal_sd' and status['name'] == 'FP-85', status
+    assert digest(sd) == original_sd, 'Loading SD settings changed the card'
+    assert vm.capture('cat /sys/class/block/mmcblk0/device/type') == 'SD'
+    vm.capture('fds machine export /tmp/sd-machine')
+    vm.capture("printf 'format=1\\nname=\"FP-85 SD\"\\n' >/tmp/sd-machine/machine.toml")
+    vm.capture('fds machine validate /tmp/sd-machine && fds machine install /tmp/sd-machine')
+    assert query(vm, ['machine', 'status'])['name'] == 'FP-85'
+    vm.capture('fds --json boot-profile >/tmp/boot.json && fds machine store sd-boot.json /tmp/boot.json')
+    vm.capture('test -z "$(findmnt -nr -o TARGET | grep /run/fds/machine/internal)"')
+    finish(vm)
+unchanged_payloads(sd)
+with VM(work, 'sd-persistent-settings', empty, extra=sd_extra(sd)) as vm:
+    status = enter(vm)
+    assert status['source'] == 'internal_sd' and status['name'] == 'FP-85 SD', status
+    vm.capture('fds machine fetch sd-boot.json /tmp/saved.json')
+    assert json.loads(vm.capture('cat /tmp/saved.json'))['boot_id'] != query(vm, ['boot-profile'])['boot_id']
+    (work / 'sd-persistent-status.json').write_text(json.dumps(status, indent=2) + '\n')
+    finish(vm)
+
+# SD supplies machine storage while the actual normal SYSTEM remains on USB.
+with VM(work, 'sd-usb-system', project / 'out/fds-system-cli.img', system_usb=True, extra=sd_extra(sd)) as vm:
+    vm.expect(rb'FDS> ')
+    wait(lambda: vm.capture('test -S /run/fds/control.sock && echo ready || echo pending'), lambda text: text == 'ready')
+    status = query(vm, ['machine', 'status'])
+    assert status['source'] == 'internal_sd' and status['name'] == 'FP-85 SD', status
+    assert vm.capture('findmnt -no FSTYPE /') == 'erofs'
+    finish(vm)
+print('PASS: native MMC SD recovery, read-only settings load, explicit writes, reboot persistence and normal USB SYSTEM handoff', flush=True)
+
 # Independent filesystem inspection after real guest writes.
 part = layout['partitions'][2]
-settings = work / 'after-guest.ext4'
-with machine.open('rb') as source, settings.open('wb') as output:
-    source.seek(part['start'] * 512)
-    remaining = part['payload_bytes']
-    while remaining:
-        chunk = source.read(min(remaining, 1024 * 1024))
-        assert chunk
-        output.write(chunk)
-        remaining -= len(chunk)
-with (work / 'after-guest-fsck.log').open('wb') as log:
-    subprocess.run([runner, 'e2fsck', '-f', '-n', str(settings)], check=True, stdout=log, stderr=subprocess.STDOUT)
-previous = subprocess.check_output([runner, 'debugfs', '-R', 'cat /config/previous.json', str(settings)], stderr=subprocess.DEVNULL)
-assert json.loads(previous)['name'] == 'FP-85'
+for source_disk, name in [(machine, 'nvme'), (sd, 'sd')]:
+    settings = work / (name + '-after-guest.ext4')
+    with source_disk.open('rb') as source, settings.open('wb') as output:
+        source.seek(part['start'] * 512)
+        remaining = part['payload_bytes']
+        while remaining:
+            chunk = source.read(min(remaining, 1024 * 1024))
+            assert chunk
+            output.write(chunk)
+            remaining -= len(chunk)
+    with (work / (name + '-after-guest-fsck.log')).open('wb') as log:
+        subprocess.run([runner, 'e2fsck', '-f', '-n', str(settings)], check=True, stdout=log, stderr=subprocess.STDOUT)
+    previous = subprocess.check_output([runner, 'debugfs', '-R', 'cat /config/previous.json', str(settings)], stderr=subprocess.DEVNULL)
+    assert json.loads(previous)['name'] == 'FP-85'
+print('PASS: independent ext4 checks after guest SD/NVMe writes and previous-settings retention', flush=True)
 
 # Both recovery and settings are present on USB, but only recovery may be selected
 # by its GPT label. USB cannot supply persistent machine settings.
 with VM(work, 'usb-lookalike', image, system_usb=True) as vm:
     status = enter(vm)
-    assert status['source'] == 'image_defaults' and 'No complete internal NVMe' in status['error'], status
+    assert status['source'] == 'image_defaults' and 'No complete internal SD/NVMe' in status['error'], status
     finish(vm)
 
 # Boot the ordinary CLI SYSTEM with two eligible NVMes already present. Stage0
@@ -159,10 +210,19 @@ with VM(work, 'ambiguous-internal', project / 'out/fds-system-cli.img', extra=[*
     assert vm.capture('stat -c %u:%g:%a /') == '0:0:755'
     wait(lambda: vm.capture('test -S /run/fds/control.sock && echo ready || echo pending'), lambda text: text == 'ready')
     status = query(vm, ['machine', 'status'])
-    assert status['source'] == 'image_defaults' and 'Multiple internal NVMe' in status['error'], status
+    assert status['source'] == 'image_defaults' and 'Multiple internal SD/NVMe' in status['error'], status
     finish(vm)
 print('PASS: USB settings spoof and two eligible NVMes fall back with explicit diagnostics', flush=True)
 
+with VM(work, 'ambiguous-sd-nvme', project / 'out/fds-system-cli.img',
+        extra=[*sd_extra(sd), *extra(image, 'old_nvme', True)]) as vm:
+    vm.expect(rb'FDS> ')
+    wait(lambda: vm.capture('test -S /run/fds/control.sock && echo ready || echo pending'), lambda text: text == 'ready')
+    status = query(vm, ['machine', 'status'])
+    assert status['source'] == 'image_defaults' and 'Multiple internal SD/NVMe' in status['error'], status
+    finish(vm)
+print('PASS: concurrent eligible SD and NVMe are rejected instead of silently selecting settings', flush=True)
+
 # Corrupt configuration and an unclean ext4 are distinct failure cases. Build
 # fixtures by changing only the settings partition, retaining the packaged OS.
 def altered(name, commands):
@@ -191,8 +251,16 @@ for name, disk, expected in fixtures:
         assert digest(disk) == before, 'Failure path wrote internal storage'
         finish(vm)
 print('PASS: invalid JSON, writable/symlink settings and unclean ext4 preserve bytes and leave the recovery console usable', flush=True)
+dirty_sd = sd_capacity(altered('unclean-sd', ['set_super_value state 0']))
+before = digest(dirty_sd)
+with VM(work, 'unclean-sd', empty, extra=sd_extra(dirty_sd)) as vm:
+    status = enter(vm)
+    assert status['source'] == 'image_defaults' and 'unclean' in status['error'], status
+    assert digest(dirty_sd) == before, 'Unclean SD was modified during fallback'
+    finish(vm)
+print('PASS: unclean SD falls back without journal replay or persistent writes', flush=True)
 link = project / 'out/m12-internal-latest.next'
 link.symlink_to(work.name)
 link.replace(project / 'out/m12-internal-latest')
 print(f'PASS: internal storage software acceptance: {work}', flush=True)
-print('SKIP: Pi EEPROM/NVMe boot, real bay calibration, power-loss and flash durability')
+print('SKIP: physical Pi EEPROM/SD/NVMe boot, maintenance USB boot, real bay calibration, power-loss and flash durability')

tests/integration/workstation-flash.py

@@ -87,6 +87,11 @@ def verify(image, disk):
 
 for image in [internal, system]:
     source_hash = digest(image)
+    inspection = json.loads(invoke(['inspect', image, '--json']).stdout)
+    assert inspection['status'] == 'inspected'
+    assert inspection['inspection']['sha256'] == source_hash
+    assert inspection['inspection']['image']['kind'] == image.stem
+    assert source_hash in invoke(['inspect', image]).stdout
     for extra in [0, 512, 4*1024*1024]:
         disk = target(f'{image.stem}-{extra}.target', image.stat().st_size + extra)
         before = digest(disk)
@@ -108,11 +113,48 @@ for name, partitions in [
     ('software', [(label, LINUX_FILESYSTEM, parts[1][2]) for label in ('FDS_METADATA','FDS_PAYLOAD02','FDS_PAYLOAD03')]),
 ]:
     image=work/(name+'.img');gpt(image,partitions)
+    inspection=json.loads(invoke(['inspect',image,'--json']).stdout)['inspection']
+    assert inspection['sha256']==digest(image)
+    assert [p['name'] for p in inspection['image']['partitions']]==[p[0] for p in partitions]
     disk=target(name+'.target',image.stat().st_size+1024*1024)
     plan=preview(image,disk)
     assert json.loads(invoke(unattended(image,disk,plan)).stdout)['status']=='verified'
     verify(image,disk)
 
+# The consolidated command creates a real DATA filesystem, inspects it, and
+# flashes it through the same unattended writer used for existing images.
+# e2fsprogs is required for this acceptance check, as for DATA creation itself.
+tree=work/'data-tree';(tree/'FDS').mkdir(parents=True)
+(tree/'FDS/CARTRIDGE.TOML').write_text('format=1\n[cartridge]\nid="flash.data"\nname="Flash test"\nclass="data"\nversion="1"\n[media]\nwritable=true\n')
+(tree/'sample.txt').write_text('Created and flashed with fds-flash\n')
+created=work/'created-data.img'
+result=json.loads(invoke(['create','data',tree,created,'--size-mib','32','--json']).stdout)
+assert result['status']=='created'
+inspection=json.loads(invoke(['inspect',created,'--json']).stdout)['inspection']
+assert result['inspection']==inspection and inspection['sha256']==digest(created)
+disk=target('created-data.target',created.stat().st_size+1024*1024)
+assert json.loads(invoke(unattended(created,disk,preview(created,disk))).stdout)['status']=='verified'
+verify(created,disk)
+partition=inspection['image']['partitions'][0]
+payload=work/'created-data.ext4'
+with disk.open('rb') as stream:
+    stream.seek(partition['start']);payload.write_bytes(stream.read(partition['bytes']))
+readback=subprocess.run(['debugfs','-R','cat /sample.txt',str(payload)],capture_output=True,text=True,check=True)
+assert readback.stdout==(tree/'sample.txt').read_text(),readback
+before=digest(created)
+invoke(['create','data',tree,created,'--size-mib','32'],False)
+assert digest(created)==before
+for arguments in [
+    ['create','data',tree,tree/'inside.img','--size-mib','32'],
+    ['create','environment',tree,work/'wrong-class.img'],
+    ['create','system',tree,work/'wrong-system.img'],
+    ['create','program',tree,work/'legacy-program.img'],
+    ['create','data',tree,work/'bad-size.img','--size-mib','31'],
+]:
+    invoke(arguments,False)
+assert not (tree/'inside.img').exists()
+assert not any((work/name).exists() for name in ['wrong-class.img','wrong-system.img','legacy-program.img','bad-size.img'])
+
 # Invalid source geometry and destinations must fail without writing.
 disk = target('protected.target', internal.stat().st_size)
 before = digest(disk)
@@ -130,7 +172,9 @@ for name, change in [('bad-primary', 512+16), ('bad-backup', internal.stat().st_
     with image.open('r+b') as stream:
         stream.seek(change); byte=stream.read(1); stream.seek(change); stream.write(bytes([byte[0]^1]))
     invoke(['--image', image, '--device', disk, '--file-target', '--dry-run'], False)
+    invoke(['inspect', image, '--json'], False)
 invoke(['--image', parts[0][2], '--device', disk, '--file-target', '--dry-run'], False)
+invoke(['inspect', parts[0][2]], False)
 assert digest(disk) == before
 
 # Drive the real terminal workflow, including cancellation and changed inputs
@@ -198,10 +242,11 @@ for name in ('fds-internal.img', 'fds-system-cli.img'):
         print('SKIP: build image not available for read-only inspection:',image)
 
 record=dict(status='passed',cli=str(cli),cli_sha256=digest(cli),internal_and_all_cartridge_classes=True,interactive_terminal=True,
+            image_inspection_without_target=True,data_create_inspect_flash_and_filesystem_readback=True,
             unattended_bindings=True,exact_larger_and_overlapping_gpt=True,independent_sfdisk=True,
             partition_payloads_unchanged=True,invalid_sources_and_targets_rejected=True,
             changed_source_and_target_rejected=True,actual_build_images_inspected=actual_images,physical_disks_written=False)
 (work/'acceptance.json').write_text(json.dumps(record,indent=2)+'\n')
 (project/'out/workstation-flash-current.txt').write_text(str(work)+'\n')
-print('PASS: interactive/unattended flash, rejection paths, payload readback and relocated GPT:',work)
-print('SKIP: physical devices; fixtures are disposable files with inert filesystem signatures')
+print('PASS: create/inspect, interactive/unattended flash, rejection paths, payload readback and relocated GPT:',work)
+print('SKIP: physical devices; disposable-file fixtures include real DATA and inert signatures for other layouts')