Skip to content

Build main for dev #186

Build main for dev

Build main for dev #186

name: 🛠️ Build Firmware
run-name: "Build ${{ github.ref_name }} for ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && 'production' || inputs.target_env }}"
on:
push:
tags:
- '[0-9][0-9][0-9][0-9].[0-9][0-9].[0-9][0-9]'
workflow_dispatch:
inputs:
target_env:
type: choice
description: "Select destination environment"
options:
- dev
- production
env:
ARTIFACT_ROOT: /home/runner/artifacts
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
jobs:
load-matrix:
if: github.event_name == 'workflow_dispatch' || github.event_name == 'push'
uses: ./.github/workflows/generate-matrix.yaml
ConfigureAndBuild:
needs: load-matrix
environment: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && 'production' || inputs.target_env }}
name: Build ${{ matrix.application }} (${{ matrix.product_id }})
runs-on: ubuntu-24.04
permissions:
id-token: write
contents: read
strategy:
matrix: ${{ fromJson(needs.load-matrix.outputs.matrix) }}
steps:
- name: Set and Adjust Build Variables
run: |
COMMIT_HASH=${GITHUB_SHA::7}
echo "COMMIT_HASH=$COMMIT_HASH" >> $GITHUB_ENV
BRANCH=${GITHUB_REF_NAME//\//-}
echo "BRANCH=$BRANCH" >> $GITHUB_ENV
- name: Echo Tag or Branch Name and Commit Hash
run: |
echo "BRANCH: ${{ env.BRANCH }}"
echo "COMMIT_HASH: ${{ env.COMMIT_HASH }}"
echo "ARTIFACT_ROOT: ${{ env.ARTIFACT_ROOT }}"
- name: Set VERSION based on tag or branch
run: |
if [ "$GITHUB_REF_TYPE" = "tag" ]; then
TAG="${GITHUB_REF#refs/tags/}"
echo "VERSION=$TAG" >> $GITHUB_ENV
echo "Release tag: $TAG"
if [[ ! "$TAG" =~ ^[0-9]{4}\.(0[1-9]|1[0-2])\.[0-9]{2}$ ]]; then
echo "❌ Invalid tag format. Expected YYYY.MM.bb"
exit 1
fi
else
echo "VERSION=DEBUG" >> $GITHUB_ENV
fi
- name: Check out repository
uses: actions/checkout@v6
- name: Update Swagger version
run: |
BRANCH="${{ env.BRANCH }}"
COMMIT_HASH="${{ env.COMMIT_HASH }}"
VERSION="${{ env.VERSION }}"
if [ "${VERSION}" != "DEBUG" ]; then
sed -i "s/^ version: .*/ version: \"${VERSION}\"/" ./${{ matrix.application }}/openapi.yaml
fi
- name: Generate controller_products in useDatabase.js from devices.yaml
if: matrix.application == 'Controller'
run: |
PRODUCTS_JS=$(yq -o=json ${{ github.workspace }}/devices.yaml \
| jq -r '[.devices[] | select(.status != "RETIRED") | " { pid: \(.product_id | @json), inputs: { count: \(.inputs_count) }, outputs: { count: \(.outputs_count) } }"] | join(",\n")')
DB_FILE="${{ github.workspace }}/Controller/ui/src/composables/useDatabase.js"
PRODUCTS_JS="$PRODUCTS_JS" DB_FILE="$DB_FILE" python3 - <<'PYEOF'
import re, os
products_js = os.environ["PRODUCTS_JS"]
db_file = os.environ["DB_FILE"]
with open(db_file, "r") as f:
content = f.read()
new_block = "const defaultControllerProducts = [\n" + products_js + "\n]"
content = re.sub(r"const defaultControllerProducts = \[.*?\]", new_block, content, flags=re.DOTALL)
with open(db_file, "w") as f:
f.write(content)
PYEOF
echo "Updated defaultControllerProducts:"
grep -A6 "const defaultControllerProducts" "$DB_FILE"
- name: Generate partitions.csv from devices.yaml
run: |
set -e
echo "# Name, Type, SubType, Offset, Size, Flags" \
> ${{ github.workspace }}/${{ matrix.application }}/partitions.csv
yq -o=json ${{ github.workspace }}/devices.yaml | jq -r \
--arg hex "${{ matrix.product_hex }}" '
def type_name: if . == 0 then "app" elif . == 1 then "data" else tostring end;
def subtype_name:
if . == "nvs" then "nvs"
elif . == "otadata" then "ota"
elif . == "app0" then "ota_0"
elif . == "app1" then "ota_1"
elif . == "coredump" then "coredump"
else "spiffs" end;
.devices[]
| select(.product_hex == $hex)
| .partition_scheme[]
| "\(.label),\t\(.type | type_name),\t\(.label | subtype_name),\t\(.address),\t\(.size),"
' >> ${{ github.workspace }}/${{ matrix.application }}/partitions.csv
echo "Generated partitions.csv for ${{ matrix.product_id }}:"
cat ${{ github.workspace }}/${{ matrix.application }}/partitions.csv
- name: Generate boards.local.txt from template
run: |
APP_SIZE_DEC=$(( ${{ matrix.partition_size_app }} ))
BOOT_MODE="qio"
FLASH_MODE="dio"
FLASH_FREQ="80m"
sed \
-e "s|{{BOOTLOADER_ADDR}}|${{ matrix.bootloader_addr }}|g" \
-e "s|{{APP_PARTITION_MAX_SIZE}}|${APP_SIZE_DEC}|g" \
-e "s|{{FLASH_MODE}}|${FLASH_MODE}|g" \
-e "s|{{BOOT_MODE}}|${BOOT_MODE}|g" \
-e "s|{{FLASH_FREQ}}|${FLASH_FREQ}|g" \
${{ github.workspace }}/boards.local.txt.template \
> ${{ github.workspace }}/boards.local.txt
- name: Install Arduino CLI
if: ${{ !env.ACT }}
uses: arduino/setup-arduino-cli@v2
with:
version: 1.4.1
- name: Read ESP32 core version
run: echo "VERSION_ESP_Core=$(yq -r '.cores[] | select(.name == "esp32:esp32") | .version' ${{ github.workspace }}/libraries.yaml)" >> $GITHUB_ENV
- name: Initialize the Arduino CLI Installation
if: ${{ !env.ACT }}
run: arduino-cli config init
- name: Add the ESP32 Board Manager Packages to Arduino CLI
if: ${{ !env.ACT }}
run: arduino-cli config set board_manager.additional_urls https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_index.json
- name: Update CLI the Core Index
if: ${{ !env.ACT }}
run: arduino-cli core update-index
- name: Cache ESP32 core
id: cache-esp32-core
if: ${{ !env.ACT }}
uses: actions/cache@v5
with:
path: ~/.arduino15/packages/esp32
key: esp32-core-${{ env.VERSION_ESP_Core }}
- name: Install ESP32 Core
if: ${{ !env.ACT && steps.cache-esp32-core.outputs.cache-hit != 'true' }}
run: arduino-cli core install esp32:esp32@$VERSION_ESP_Core
- name: Create symlink for boards.local.txt
run: |
ARDUINO_DIR=$(arduino-cli config get directories.data)
ln -sf ${{ github.workspace }}/boards.local.txt ${ARDUINO_DIR}/packages/esp32/hardware/esp32/$VERSION_ESP_Core/boards.local.txt
- name: Enable Unsafe Installation Flag to Enable Git Installs
run: arduino-cli config set library.enable_unsafe_install true
- name: Cache Arduino libraries
id: cache-arduino-libs
if: ${{ !env.ACT }}
uses: actions/cache@v5
with:
path: ~/Arduino/libraries
key: arduino-libs-${{ hashFiles('libraries.yaml') }}
- name: Install Required Libraries at Specified Revisions
if: ${{ !env.ACT && steps.cache-arduino-libs.outputs.cache-hit != 'true' }}
run: |
yq -r '.libraries[] | .url + ".git#" + .version' ${{ github.workspace }}/libraries.yaml | \
xargs -I{} arduino-cli lib install --git-url {}
- name: Create Output Directory
run: mkdir -p "${{ env.ARTIFACT_ROOT }}/${{ matrix.product_id }}/${{ matrix.application }}/${{ env.BRANCH }}-${{ env.COMMIT_HASH }}"
- name: Compile ${{ matrix.application }} (${{ matrix.product_id }})
run: |
set -e
BRANCH="${{ env.BRANCH }}"
COMMIT_HASH="${{ env.COMMIT_HASH }}"
APPLICATION="${{ matrix.application }}"
OUTDIR="${{ env.ARTIFACT_ROOT }}/${{ matrix.product_id }}/${APPLICATION}/${BRANCH}-${COMMIT_HASH}"
mkdir -p "$OUTDIR"
EFUSE_FLAG=""
if [ "$APPLICATION" = "Hardware-Registration-and-Configuration" ] && [ "${{ matrix.burn_vdd_sdio_efuse }}" = "true" ]; then
EFUSE_FLAG=" -DBURN_VDD_SDIO_EFUSE=1"
fi
PSRAM_FLAGS="-DBOARD_HAS_PSRAM -mfix-esp32-psram-cache-issue -mfix-esp32-psram-cache-strategy=memw"
if [ "$APPLICATION" = "Controller" ]; then
PROJECT_NAME="FireFly Controller"
else
PROJECT_NAME="HW Reg and Config"
fi
PROJECT_NAME_ESCAPED="${PROJECT_NAME// /\\x20}"
if [ "$VERSION" = "DEBUG" ]; then
echo "Building DEBUG firmware ($BRANCH)"
arduino-cli compile \
-b esp32:esp32:firefly_controller ./${APPLICATION} \
--warnings more --clean \
--build-property "build.extra_flags=-DASYNCWEBSERVER_REGEX -DPRODUCT_HEX=${{ matrix.product_hex }} -DESP32 -DDISABLE_ALL_LIBRARY_WARNINGS -DCORE_DEBUG_LEVEL=4 -DPROJECT_VER=\"9999.99.99\" -DPROJECT_NAME=\"${PROJECT_NAME_ESCAPED}\" -DCOMMIT_HASH=\"${COMMIT_HASH}\" -DFIREFLY_CLOUD_API_ROOT=\"${{ vars.FIREFLY_CLOUD_API_ROOT }}\" -I${{ github.workspace }}${EFUSE_FLAG} ${PSRAM_FLAGS}" \
--output-dir "$OUTDIR"
else
echo "Building RELEASE firmware"
arduino-cli compile \
-b esp32:esp32:firefly_controller ./${APPLICATION} \
--warnings more --clean \
--build-property "build.extra_flags=-DASYNCWEBSERVER_REGEX -DPRODUCT_HEX=${{ matrix.product_hex }} -DESP32 -DDISABLE_ALL_LIBRARY_WARNINGS -DCORE_DEBUG_LEVEL=0 -DPROJECT_VER=\"${VERSION}\" -DPROJECT_NAME=\"${PROJECT_NAME_ESCAPED}\" -DCOMMIT_HASH=\"${COMMIT_HASH}\" -DFIREFLY_CLOUD_API_ROOT=\"${{ vars.FIREFLY_CLOUD_API_ROOT }}\" -I${{ github.workspace }}${EFUSE_FLAG} ${PSRAM_FLAGS}" \
--output-dir "$OUTDIR"
fi
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: '${{ matrix.application }}/ui/package-lock.json'
- name: Build Vue UI
if: ${{ matrix.application == 'Hardware-Registration-and-Configuration' }}
run: |
cd $GITHUB_WORKSPACE/Hardware-Registration-and-Configuration/ui
npm ci
npm run build
cp $GITHUB_WORKSPACE/Hardware-Registration-and-Configuration/openapi.yaml \
$GITHUB_WORKSPACE/Hardware-Registration-and-Configuration/ui-dist/openapi.yaml
gzip -r -9 $GITHUB_WORKSPACE/Hardware-Registration-and-Configuration/ui-dist
TOTAL=$(du -sb $GITHUB_WORKSPACE/Hardware-Registration-and-Configuration/ui-dist | cut -f1)
if [ "$TOTAL" -gt 2097152 ]; then
echo "ERROR: ui-dist/ exceeds 2MB (${TOTAL} bytes)"
exit 1
fi
echo '{"application":"Hardware-Registration-and-Configuration","version":"${{ env.VERSION }}","commit":"${{ env.COMMIT_HASH }}"}' > $GITHUB_WORKSPACE/Hardware-Registration-and-Configuration/ui-dist/version.json
- name: Build Vue UI (Controller)
if: ${{ matrix.application == 'Controller' }}
run: |
cd $GITHUB_WORKSPACE/Controller/ui
npm ci --legacy-peer-deps
npm run build
cp $GITHUB_WORKSPACE/Controller/openapi.yaml \
$GITHUB_WORKSPACE/Controller/ui-dist/openapi.yaml
gzip -r -9 $GITHUB_WORKSPACE/Controller/ui-dist
TOTAL=$(du -sb $GITHUB_WORKSPACE/Controller/ui-dist | cut -f1)
if [ "$TOTAL" -gt 3014656 ]; then
echo "ERROR: ui-dist/ exceeds partition size of 0x2e0000 (${TOTAL} bytes)"
exit 1
fi
echo '{"application":"Controller","version":"${{ env.VERSION }}","commit":"${{ env.COMMIT_HASH }}"}' > $GITHUB_WORKSPACE/Controller/ui-dist/version.json
- name: Create Empty Directory for Use by LittleFS
run: mkdir -p "${{ env.ARTIFACT_ROOT }}/LittleFS"
- name: Build LittleFS Images
run: |
set -e
ARDUINO_DIR=$(arduino-cli config get directories.data)
OUTDIR="${{ env.ARTIFACT_ROOT }}/${{ matrix.product_id }}/${{ matrix.application }}/${{ env.BRANCH }}-${{ env.COMMIT_HASH }}"
mkdir -p "$OUTDIR"
${ARDUINO_DIR}/packages/esp32/tools/mklittlefs/**/mklittlefs \
-s ${{ matrix.partition_size_ui }} \
-c ${{ github.workspace }}/${{ matrix.application }}/ui-dist \
"$OUTDIR/ui.bin"
${ARDUINO_DIR}/packages/esp32/tools/mklittlefs/**/mklittlefs \
-s ${{ matrix.partition_size_config }} \
-c "${{ env.ARTIFACT_ROOT }}/LittleFS" \
"$OUTDIR/config.bin"
- name: Generate manifest JSON
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
ROOT="${{ env.ARTIFACT_ROOT }}"
NAME="${{ matrix.product_id }}"
APPLICATION="${{ matrix.application }}"
BRANCH="${{ env.BRANCH }}"
VERSION="${{ env.VERSION }}"
COMMIT_HASH="${{ env.COMMIT_HASH }}"
JSON_FILE="${ROOT}/${NAME}/${APPLICATION}/${BRANCH}-${COMMIT_HASH}/manifest.json"
# ISO 8601 timestamp (UTC)
CREATED_ISO=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
# Fetch the GitHub Release HTML URL for tag builds only
RELEASE_URL=""
if [ "$GITHUB_REF_TYPE" = "tag" ]; then
RELEASE_URL=$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${VERSION}" \
--jq '.html_url // empty' 2>/dev/null || true)
fi
# Start with an empty JSON array
FILES_JSON=$(jq -nc '[]')
# Recursively find all files
while IFS= read -r f; do
NAME=$(basename "$f")
SHA256=$(sha256sum "$f" | awk '{print $1}')
FILES_JSON=$(echo "$FILES_JSON" \
| jq -c --arg name "$NAME" --arg sha256 "$SHA256" \
'. += [{name: $name, sha256: $sha256}]')
done < <(find "$ROOT" -type f)
# Map application to firmware_type
case "$APPLICATION" in
Controller) FIRMWARE_TYPE="FireFly Controller" ;;
Hardware-Registration-and-Configuration) FIRMWARE_TYPE="FireFly Hardware Registration and Configuration" ;;
*) FIRMWARE_TYPE="$APPLICATION" ;;
esac
# Build the base JSON object
JSON=$(jq -nc \
--arg class "controller" \
--arg product_id "${{ matrix.product_id }}" \
--arg product_hex "${{ matrix.product_hex }}" \
--arg firmware_type "$FIRMWARE_TYPE" \
--arg application "${{ matrix.application }}" \
--arg branch "${{ env.BRANCH }}" \
--arg version "${{ env.VERSION }}" \
--arg commit "${{ env.COMMIT_HASH }}" \
--arg created "$CREATED_ISO" \
--arg bootloader_addr "${{ matrix.bootloader_addr }}" \
--argjson files "$FILES_JSON" \
'{
class: $class,
product_id: $product_id,
product_hex: $product_hex,
firmware_type: $firmware_type,
application: $application,
branch: $branch,
version: $version,
commit: $commit,
created: $created,
bootloader_addr: $bootloader_addr,
files: $files
}')
# Append release_url for tag builds (omitted entirely for DEBUG)
if [ -n "$RELEASE_URL" ]; then
JSON=$(echo "$JSON" | jq --arg url "$RELEASE_URL" '. + {release_url: $url}')
fi
echo "$JSON" > "$JSON_FILE"
- uses: actions/upload-artifact@v7
name: Upload Artifacts
with:
if-no-files-found: error
name: ${{ matrix.product_id }}-${{ matrix.application }}-${{ env.BRANCH }}-${{ env.COMMIT_HASH }}
path: ${{ env.ARTIFACT_ROOT }}/${{ matrix.product_id }}/${{ matrix.application }}/${{ env.BRANCH }}-${{ env.COMMIT_HASH }}/*
- name: Configure AWS Credentials
if: ${{ !env.ACT }}
uses: aws-actions/configure-aws-credentials@v6
with:
role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
aws-region: ${{ secrets.AWS_REGION }}
- name: Upload ZIP to S3
if: ${{ !env.ACT }}
run: |
OUTDIR="${{ env.ARTIFACT_ROOT }}/${{ matrix.product_id }}/${{ matrix.application }}/${{ env.BRANCH }}-${{ env.COMMIT_HASH }}"
DEST_KEY="incoming/${{ matrix.application }}-${{ matrix.product_hex }}-${{ env.BRANCH }}-${{ env.COMMIT_HASH }}.zip"
cd "$OUTDIR"
zip -r /tmp/firmware-upload.zip .
aws s3 cp /tmp/firmware-upload.zip \
"s3://${{ secrets.S3_FIRMWARE_BUCKET_NAME }}/${DEST_KEY}"
trigger-configurator-ui:
needs: ConfigureAndBuild
if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) }}
uses: ./.github/workflows/trigger-configurator-ui.yaml
with:
target_env: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && 'production' || inputs.target_env }}
ref: ${{ github.ref_name }}
secrets:
FIREFLY_CLOUD_TOKEN: ${{ secrets.FIREFLY_CLOUD_TOKEN }}
sync-library-docs:
needs: ConfigureAndBuild
if: ${{ github.event.inputs.target_env == 'production' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) }}
runs-on: ubuntu-24.04
environment: production
permissions:
contents: read
steps:
- name: Checkout FireFly-Controller
uses: actions/checkout@v6
- name: Set Build Variables
run: |
echo "BRANCH=$GITHUB_REF_NAME" >> $GITHUB_ENV
- name: Generate core table
id: core_table
run: |
TABLE=$(yq -r '
.cores
| sort_by(.name)
| .[]
| "| [" + .name + "](" + .url + ") | " + .version + " |"
' libraries.yaml)
echo "table<<EOF" >> $GITHUB_OUTPUT
echo "$TABLE" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
- name: Generate library table
id: table
run: |
TABLE=$(yq -r '
.libraries
| sort_by(.name)
| .[]
| "| [" + .name + "](" + .url + ") | " + .version + " |"
' libraries.yaml)
echo "table<<EOF" >> $GITHUB_OUTPUT
echo "$TABLE" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
- name: Checkout FireFly-Docs
uses: actions/checkout@v6
with:
repository: BrentIO/FireFly-Docs
token: ${{ secrets.FIREFLY_DOCS_TOKEN }}
path: FireFly-Docs
- name: Update core and library tables and open PR
env:
GH_TOKEN: ${{ secrets.FIREFLY_DOCS_TOKEN }}
CORE_TABLE: ${{ steps.core_table.outputs.table }}
LIBRARY_TABLE: ${{ steps.table.outputs.table }}
run: |
cd FireFly-Docs
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
SYNC_BRANCH="sync/library-list-${{ env.BRANCH }}"
git checkout -b "$SYNC_BRANCH"
python3 - <<'PYEOF'
import re, os
path = "controller/development_environment/index.md"
content = open(path).read()
core_table = os.environ["CORE_TABLE"]
new_core_block = (
"<!-- CORE-TABLE-START -->\n"
"| Core | Version |\n"
"| ---- | ------- |\n"
+ core_table + "\n"
"<!-- CORE-TABLE-END -->"
)
content = re.sub(
r"<!-- CORE-TABLE-START -->.*?<!-- CORE-TABLE-END -->",
new_core_block,
content,
flags=re.DOTALL
)
library_table = os.environ["LIBRARY_TABLE"]
new_library_block = (
"<!-- LIBRARY-TABLE-START -->\n"
"| Library | Version |\n"
"| ------- | ------- |\n"
+ library_table + "\n"
"<!-- LIBRARY-TABLE-END -->"
)
content = re.sub(
r"<!-- LIBRARY-TABLE-START -->.*?<!-- LIBRARY-TABLE-END -->",
new_library_block,
content,
flags=re.DOTALL
)
open(path, "w").write(content)
PYEOF
git add controller/development_environment/index.md
if git diff --cached --quiet; then
echo "Core and library tables unchanged, skipping PR"
exit 0
fi
git commit -m "Update core and library versions from FireFly-Controller ${{ env.BRANCH }}"
git push origin "$SYNC_BRANCH"
gh pr create \
--repo BrentIO/FireFly-Docs \
--title "Update core and library versions from FireFly-Controller ${{ env.BRANCH }}" \
--body "Automated core and library version sync triggered by production build of \`${{ env.BRANCH }}\`." \
--base main \
--head "$SYNC_BRANCH"