Meshtastic build manifest

Author: vidplace7

.github/workflows/build_firmware.yml

@@ -62,6 +62,7 @@ jobs:
           name: firmware-${{ inputs.platform }}-${{ inputs.pio_env }}-${{ inputs.version }}.zip
           overwrite: true
           path: |
+            release/*.mt.json
             release/*.bin
             release/*.elf
             release/*.uf2

.github/workflows/main_matrix.yml

@@ -27,7 +27,6 @@ on:
 
 jobs:
   setup:
-    if: github.repository == 'meshtastic/firmware'
     strategy:
       fail-fast: true
       matrix:
@@ -42,10 +41,6 @@ jobs:
           python-version: 3.x
           cache: pip
       - run: pip install -U platformio
-      - name: Uncomment build epoch
-        shell: bash
-        run: |
-          sed -i 's/#-DBUILD_EPOCH=$UNIX_TIME/-DBUILD_EPOCH=$UNIX_TIME/' platformio.ini
       - name: Generate matrix
         id: jsonStep
         run: |
@@ -62,7 +57,6 @@ jobs:
       check: ${{ steps.jsonStep.outputs.check }}
 
   version:
-    if: github.repository == 'meshtastic/firmware'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
@@ -226,10 +220,11 @@ jobs:
           name: firmware-${{matrix.arch}}-${{ needs.version.outputs.long }}
           overwrite: true
           path: |
+            ./firmware-*.mt.json
             ./firmware-*.bin
             ./firmware-*.uf2
             ./firmware-*.hex
-            ./firmware-*-ota.zip
+            ./firmware-*.zip
             ./device-*.sh
             ./device-*.bat
             ./littlefs-*.bin

bin/build-esp32.sh

@@ -5,7 +5,8 @@ set -e
 VERSION=`bin/buildinfo.py long`
 SHORT_VERSION=`bin/buildinfo.py short`
 
-OUTDIR=release/
+BUILDDIR=.pio/build/$1
+OUTDIR=release
 
 rm -f $OUTDIR/firmware*
 rm -r $OUTDIR/* || true
@@ -14,24 +15,22 @@ rm -r $OUTDIR/* || true
 platformio pkg install -e $1
 
 echo "Building for $1 with $PLATFORMIO_BUILD_FLAGS"
-rm -f .pio/build/$1/firmware.*
+rm -f $BUILDDIR/firmware*
 
 # The shell vars the build tool expects to find
 export APP_VERSION=$VERSION
 
 basename=firmware-$1-$VERSION
 
 pio run --environment $1 # -v
-SRCELF=.pio/build/$1/firmware.elf
-cp $SRCELF $OUTDIR/$basename.elf
+
+cp $BUILDDIR/$basename.elf $OUTDIR/$basename.elf
 
 echo "Copying ESP32 bin file"
-SRCBIN=.pio/build/$1/firmware.factory.bin
-cp $SRCBIN $OUTDIR/$basename.bin
+cp $BUILDDIR/$basename.factory.bin $OUTDIR/$basename.factory.bin
 
 echo "Copying ESP32 update bin file"
-SRCBIN=.pio/build/$1/firmware.bin
-cp $SRCBIN $OUTDIR/$basename-update.bin
+cp $BUILDDIR/$basename.bin $OUTDIR/$basename.bin
 
 echo "Building Filesystem for ESP32 targets"
 # If you want to build the webui, uncomment the following lines
@@ -41,6 +40,10 @@ echo "Building Filesystem for ESP32 targets"
 # ls -l data/static # Diagnostic list of files
 # rm -rf data/static
 pio run --environment $1 -t buildfs
-cp .pio/build/$1/littlefs.bin $OUTDIR/littlefs-$1-$VERSION.bin
-cp bin/device-install.* $OUTDIR
-cp bin/device-update.* $OUTDIR
+cp $BUILDDIR/littlefs-$1-$VERSION.bin $OUTDIR/littlefs-$1-$VERSION.bin
+cp bin/device-install.* $OUTDIR/
+cp bin/device-update.* $OUTDIR/
+
+# Generate the manifest file
+pio run --environment $1 -t mtjson
+cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json

bin/build-native.sh

@@ -17,15 +17,19 @@ VERSION=$(bin/buildinfo.py long)
 SHORT_VERSION=$(bin/buildinfo.py short)
 PIO_ENV=${1:-native}
 
-OUTDIR=release/
+BUILDDIR=.pio/build/$PIO_ENV
+OUTDIR=release
 
-rm -f $OUTDIR/firmware*
+rm -f $OUTDIR/meshtasticd*
 
 mkdir -p $OUTDIR/
 rm -r $OUTDIR/* || true
 
+basename=meshtasticd-$1-$VERSION
+
 # Important to pull latest version of libs into all device flavors, otherwise some devices might be stale
 pio pkg install --environment "$PIO_ENV" || platformioFailed
 pio run --environment "$PIO_ENV" || platformioFailed
-cp ".pio/build/$PIO_ENV/program" "$OUTDIR/meshtasticd_linux_$(uname -m)"
-cp bin/native-install.* $OUTDIR
+
+cp "$BUILDDIR/$basename" "$OUTDIR/meshtasticd_linux_$(uname -m)"
+cp bin/native-install.* $OUTDIR/

bin/build-nrf52.sh

@@ -5,7 +5,8 @@ set -e
 VERSION=$(bin/buildinfo.py long)
 SHORT_VERSION=$(bin/buildinfo.py short)
 
-OUTDIR=release/
+BUILDDIR=.pio/build/$1
+OUTDIR=release
 
 rm -f $OUTDIR/firmware*
 rm -r $OUTDIR/* || true
@@ -14,40 +15,44 @@ rm -r $OUTDIR/* || true
 platformio pkg install -e $1
 
 echo "Building for $1 with $PLATFORMIO_BUILD_FLAGS"
-rm -f .pio/build/$1/firmware.*
+rm -f $BUILDDIR/firmware.*
 
 # The shell vars the build tool expects to find
 export APP_VERSION=$VERSION
 
 basename=firmware-$1-$VERSION
 
 pio run --environment $1 # -v
-SRCELF=.pio/build/$1/firmware.elf
-cp $SRCELF $OUTDIR/$basename.elf
 
-echo "Generating NRF52 dfu file"
-DFUPKG=.pio/build/$1/firmware.zip
-cp $DFUPKG $OUTDIR/$basename-ota.zip
+cp $BUILDDIR/$basename.elf $OUTDIR/$basename.elf
 
-echo "Generating NRF52 uf2 file"
-SRCHEX=.pio/build/$1/firmware.hex
+echo "Copying NRF52 dfu (OTA) file"
+cp $BUILDDIR/$basename.zip $OUTDIR/$basename.zip
+
+SRCHEX=$BUILDDIR/$basename.hex
 
 # if WM1110 target, merge hex with softdevice 7.3.0
+# TODO move this logic into platformio-custom.py
 if (echo $1 | grep -q "wio-sdk-wm1110"); then
 	echo "Merging with softdevice"
-	bin/mergehex -m bin/s140_nrf52_7.3.0_softdevice.hex $SRCHEX -o .pio/build/$1/$basename.hex
-	SRCHEX=.pio/build/$1/$basename.hex
+	bin/mergehex -m bin/s140_nrf52_7.3.0_softdevice.hex $SRCHEX -o $BUILDDIR/$basename.merged.hex
+	SRCHEX=$BUILDDIR/$basename.merged.hex
 	bin/uf2conv.py $SRCHEX -c -o $OUTDIR/$basename.uf2 -f 0xADA52840
-	cp $SRCHEX $OUTDIR
-	cp bin/*.uf2 $OUTDIR
+	cp $SRCHEX $OUTDIR/
+	cp bin/*.uf2 $OUTDIR/
 else
-	bin/uf2conv.py $SRCHEX -c -o $OUTDIR/$basename.uf2 -f 0xADA52840
-	cp bin/device-install.* $OUTDIR
-	cp bin/device-update.* $OUTDIR
-	cp bin/*.uf2 $OUTDIR
+	# UF2 generated by default as part of platformio-custom.py
+	cp $BUILDDIR/$basename.uf2 $OUTDIR/$basename.uf2
+	cp bin/device-install.* $OUTDIR/
+	cp bin/device-update.* $OUTDIR/
+	cp bin/*.uf2 $OUTDIR/
 fi
 
 if (echo $1 | grep -q "rak4631"); then
 	echo "Copying hex file"
-	cp .pio/build/$1/firmware.hex $OUTDIR/$basename.hex
-fi
+	cp $SRCHEX $OUTDIR/$basename.hex
+fi
+
+# Generate the manifest file
+pio run --environment $1 -t mtjson
+cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json

bin/build-rp2xx0.sh

@@ -5,7 +5,8 @@ set -e
 VERSION=`bin/buildinfo.py long`
 SHORT_VERSION=`bin/buildinfo.py short`
 
-OUTDIR=release/
+BUILDDIR=.pio/build/$1
+OUTDIR=release
 
 rm -f $OUTDIR/firmware*
 rm -r $OUTDIR/* || true
@@ -14,20 +15,23 @@ rm -r $OUTDIR/* || true
 platformio pkg install -e $1
 
 echo "Building for $1 with $PLATFORMIO_BUILD_FLAGS"
-rm -f .pio/build/$1/firmware.*
+rm -f $BUILDDIR/firmware*
 
 # The shell vars the build tool expects to find
 export APP_VERSION=$VERSION
 
 basename=firmware-$1-$VERSION
 
 pio run --environment $1 # -v
-SRCELF=.pio/build/$1/firmware.elf
-cp $SRCELF $OUTDIR/$basename.elf
+
+cp $BUILDDIR/$basename.elf $OUTDIR/$basename.elf
 
 echo "Copying uf2 file"
-SRCBIN=.pio/build/$1/firmware.uf2
-cp $SRCBIN $OUTDIR/$basename.uf2
+cp $BUILDDIR/$basename.uf2 $OUTDIR/$basename.uf2
+
+cp bin/device-install.* $OUTDIR/
+cp bin/device-update.* $OUTDIR/
 
-cp bin/device-install.* $OUTDIR
-cp bin/device-update.* $OUTDIR
+# Generate the manifest file
+pio run --environment $1 -t mtjson
+cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json

bin/build-stm32wl.sh

@@ -5,7 +5,8 @@ set -e
 VERSION=$(bin/buildinfo.py long)
 SHORT_VERSION=$(bin/buildinfo.py short)
 
-OUTDIR=release/
+BUILDDIR=.pio/build/$1
+OUTDIR=release
 
 rm -f $OUTDIR/firmware*
 rm -r $OUTDIR/* || true
@@ -14,16 +15,20 @@ rm -r $OUTDIR/* || true
 platformio pkg install -e $1
 
 echo "Building for $1 with $PLATFORMIO_BUILD_FLAGS"
-rm -f .pio/build/$1/firmware.*
+rm -f $BUILDDIR/firmware*
 
 # The shell vars the build tool expects to find
 export APP_VERSION=$VERSION
 
 basename=firmware-$1-$VERSION
 
 pio run --environment $1 # -v
-SRCELF=.pio/build/$1/firmware.elf
-cp $SRCELF $OUTDIR/$basename.elf
 
-SRCBIN=.pio/build/$1/firmware.bin
-cp $SRCBIN $OUTDIR/$basename.bin
+cp $BUILDDIR/$basename.elf $OUTDIR/$basename.elf
+
+echo "Copying STM32 bin file"
+cp $BUILDDIR/$basename.bin $OUTDIR/$basename.bin
+
+# Generate the manifest file
+pio run --environment $1 -t mtjson
+cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json

bin/device-install.bat

@@ -93,6 +93,27 @@ IF NOT EXIST !FILENAME! (
     GOTO eof
 )
 
+CALL :LOG_MESSAGE DEBUG "Checking for metadata..."
+@REM Derive metadata filename from firmware filename.
+SET "METAFILE=!FILENAME:.bin=!.mt.json"
+IF EXIST !METAFILE! (
+    @REM Print parsed json with powershell
+    CALL :LOG_MESSAGE INFO "Firmware metadata: !METAFILE!"
+    powershell -NoProfile -Command "(Get-Content '!METAFILE!' | ConvertFrom-Json | Out-String).Trim()"
+
+    @REM Save metadata values to variables for later use.
+    FOR /f "usebackq" %%A IN (`powershell -NoProfile -Command ^
+        "(Get-Content '!METAFILE!' | ConvertFrom-Json).mcu"`) DO SET "MCU=%%A"
+    FOR /f "usebackq" %%A IN (`powershell -NoProfile -Command ^
+        "(Get-Content '!METAFILE!' | ConvertFrom-Json).part | Where-Object { $_.subtype -eq 'ota_1' } | Select-Object -ExpandProperty offset"`
+    ) DO SET "OTA_OFFSET=%%A"
+    FOR /f "usebackq" %%A IN (`powershell -NoProfile -Command ^
+        "(Get-Content '!METAFILE!' | ConvertFrom-Json).part | Where-Object { $_.subtype -eq 'spiffs' } | Select-Object -ExpandProperty offset"`
+    ) DO SET "SPIFFS_OFFSET=%%A"
+) ELSE (
+    CALL :LOG_MESSAGE WARN "No metadata file found: !METAFILE!"
+)
+
 IF NOT "!FILENAME:update=!"=="!FILENAME!" (
     CALL :LOG_MESSAGE DEBUG "We are working with a *update* file. !FILENAME!"
     CALL :LOG_MESSAGE INFO "Use script device-update.bat to flash update !FILENAME!."

bin/device-install.sh

@@ -76,6 +76,14 @@ else
     exit 1
 fi
 
+# Check for jq
+if ! command -v jq >/dev/null 2>&1; then
+    echo "Error: jq not found" >&2
+    echo "Install jq with your package manager." >&2
+    echo "e.g. 'apt install jq', 'dnf install jq', 'brew install jq', etc." >&2
+    exit 1
+fi
+
 set -e
 
 # Usage info
@@ -141,14 +149,31 @@ if [[ "$FILENAME" != firmware-* ]]; then
   exit 1
 fi
 
+# Extract BASENAME from %FILENAME% for later use.
+BASENAME="${FILENAME/firmware-/}"
+
+# Derive metadata filename from firmware filename.
+METAFILE="${FILENAME%.bin}.mt.json"
+
+# Display metadata if it exists
+if [[ -f "$METAFILE" ]]; then
+    echo "Firmware metadata: ${METAFILE}"
+    jq . "$METAFILE"
+    # Extract relevant fields from metadata
+    if [[ $(jq -r '.part' "$METAFILE") != "null" ]]; then
+        OTA_OFFSET=$(jq -r '.part[] | select(.subtype == "ota_1") | .offset' "$METAFILE")
+        SPIFFS_OFFSET=$(jq -r '.part[] | select(.subtype == "spiffs") | .offset' "$METAFILE")
+    fi
+    MCU=$(jq -r '.mcu' "$METAFILE")
+else
+    echo "WARNING: No metadata file found: ${METAFILE}"
+fi
+
 # Check if FILENAME contains "-tft-" and set target partitionScheme accordingly.
 if [[ "${FILENAME//-tft-/}" != "$FILENAME" ]]; then
     TFT_BUILD=true
 fi
 
-# Extract BASENAME from %FILENAME% for later use.
-BASENAME="${FILENAME/firmware-/}"
-
 if [ -f "${FILENAME}" ] && [ -n "${FILENAME##*"update"*}" ]; then
     # Default littlefs* offset.
     OFFSET=0x300000