Skip to content
Merged
18 changes: 18 additions & 0 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ jobs:
docker-cache:
name: Build Cache Docker Images
strategy:
# Independent legs: a failing rdk-focal build must not cancel the others.
fail-fast: false
matrix:
include:
# rdk-devenv
Expand Down Expand Up @@ -51,6 +53,22 @@ jobs:
image: antique2
file: etc/Dockerfile.antique-cache
tag: armhf
# rdk-focal (self-contained Focal image; MAIN_TAG/BASE_TAG build-args are ignored)
- arch: ubuntu-latest
tag: amd64
platform: linux/amd64
image: rdk-focal
file: etc/Dockerfile.focal
- arch: ubuntu-small-arm
tag: arm64
platform: linux/arm64
image: rdk-focal
file: etc/Dockerfile.focal
- arch: ubuntu-small-arm
tag: armhf
platform: linux/arm/v7
image: rdk-focal
file: etc/Dockerfile.focal
Comment thread
acmorrow marked this conversation as resolved.
runs-on: ${{ matrix.arch }}
timeout-minutes: 30
steps:
Expand Down
141 changes: 141 additions & 0 deletions .github/workflows/focal-build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
name: Focal Static Build

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when we're directionally in agreement, i'd like to merge this PR so i can follow up with improvements on a new branch and run the workflow from the branch

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm happy for you to merge it, for my part.


# Builds, boots, and (per release_type) publishes a viam-server-focal channel
# on the rdk-focal image. Called by pullrequest-trusted.yml for labeled PRs
# (release_type: pr -> focal-pr-<number>); schedule and default dispatch validate only.
# No concurrency block: github.ref is the base ref for every PR here, so a
# shared group would cancel unrelated builds. The parent handles PR concurrency.

on:
workflow_dispatch:
inputs:
release_type:
description: "'focal' publishes the shared channel; 'validate' builds and boots only"
required: true
default: validate
type: choice
options: [validate, focal]
schedule:
- cron: '0 8 * * *' # daily, after docker.yml (6AM UTC) republishes the image
workflow_call:
inputs:
release_type:
required: true
type: string
secrets:
GCP_CREDENTIALS:
required: true

permissions:
contents: read

jobs:
focal-static:
name: Focal Static Build and Boot
strategy:
fail-fast: false
matrix:
include:
- arch: ubuntu-large
image: ghcr.io/viamrobotics/rdk-focal:amd64-cache
platform: linux/amd64
- arch: ubuntu-large-arm
image: ghcr.io/viamrobotics/rdk-focal:arm64-cache
platform: linux/arm64
runs-on: ${{ matrix.arch }}
container:
image: ${{ matrix.image }}
options: --platform ${{ matrix.platform }}
timeout-minutes: 30

steps:
- name: Check out code
if: github.event_name != 'pull_request_target'
# actions/checkout@v4.3.1
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with:
fetch-depth: 0 # needed for dev-version.sh to work

- name: Check out PR branch code
if: github.event_name == 'pull_request_target'
# actions/checkout@v4.3.1
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
with:
ref: ${{ github.event.pull_request.head.sha }}

- name: Fix permissions
run: chown -R testbot:testbot .

- name: Install upx

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there is a github actions marketplace implementation of this that might be worth investigating if you are willing to accept the external dep.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i'll follow up with another pr. i can't really test the flow until i merge so it would be nice to see it work

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good, I'm just happy to see it out of the container!

env:
UPX_VERSION: 5.2.0
run: |
set -euo pipefail
case "$(uname -m)" in
x86_64) upx_arch=amd64 ;;
aarch64) upx_arch=arm64 ;;
*) echo "unsupported arch" >&2; exit 1 ;;
esac
curl -fsSL "https://github.com/upx/upx/releases/download/v${UPX_VERSION}/upx-${UPX_VERSION}-${upx_arch}_linux.tar.xz" \
| tar -C /usr/local/bin --strip-components=1 --wildcards -xJ '*/upx'
upx --version

- name: Resolve channel and arch
id: meta
run: |
case "${{ inputs.release_type }}" in
pr)
echo "channel=focal-pr-${{ github.event.pull_request.number }}" >> "$GITHUB_OUTPUT"
echo "publish=true" >> "$GITHUB_OUTPUT" ;;
focal)
echo "channel=focal" >> "$GITHUB_OUTPUT"
echo "publish=true" >> "$GITHUB_OUTPUT" ;;
*)
echo "channel=focal" >> "$GITHUB_OUTPUT"
echo "publish=false" >> "$GITHUB_OUTPUT" ;;
esac
echo "arch=$(uname -m)" >> "$GITHUB_OUTPUT"

- name: Build viam-server-${{ steps.meta.outputs.channel }}
run: |
sudo -Hu testbot bash -lc 'make BUILD_CHANNEL=${{ steps.meta.outputs.channel }} static-release'

- name: Boot smoke test
run: |
sudo -Hu testbot bash -lc '
set -euo pipefail
bin=$(find etc/packaging/static/deploy -type f -name "viam-server-*" | head -1)
port=$((30000 + RANDOM))
echo "{\"network\":{\"bind_address\":\"localhost:${port}\"}}" > /tmp/smoke.json
"$bin" -config /tmp/smoke.json &
srv=$!
curl --retry 8 --retry-delay 2 --retry-connrefused -s "localhost:${port}" >/dev/null
echo "boot OK"
kill $srv 2>/dev/null || true
wait $srv 2>/dev/null || true
'

- name: Authorize GCP upload
if: steps.meta.outputs.publish == 'true'
# google-github-actions/auth@v2.1.13
uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed
with:
credentials_json: ${{ secrets.GCP_CREDENTIALS }}

- name: Publish ${{ steps.meta.outputs.channel }} channel
if: steps.meta.outputs.publish == 'true'
# google-github-actions/upload-cloud-storage@v2.2.4
uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23
with:
headers: "cache-control: no-cache"
path: etc/packaging/static/deploy/
destination: packages.viam.com/apps/viam-server/
glob: 'viam-server-${{ steps.meta.outputs.channel }}-*'
parent: false

- name: Summary
if: steps.meta.outputs.publish == 'true'
run: |
url="https://storage.googleapis.com/packages.viam.com/apps/viam-server/viam-server-${{ steps.meta.outputs.channel }}-${{ steps.meta.outputs.arch }}"
echo "### viam-server-${{ steps.meta.outputs.channel }}-${{ steps.meta.outputs.arch }}" >> "$GITHUB_STEP_SUMMARY"
echo "$url" >> "$GITHUB_STEP_SUMMARY"
12 changes: 12 additions & 0 deletions .github/workflows/pullrequest-trusted.yml
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,18 @@ jobs:
- windows x86_64: https://storage.googleapis.com/packages.viam.com/apps/viam-server/viam-server-pr-${{ github.event.pull_request.number }}-windows-x86_64
- macOS arm64: https://storage.googleapis.com/packages.viam.com/apps/viam-server/viam-server-pr-${{ github.event.pull_request.number }}-darwin-aarch64
# This lets people add a "focal-build" tag to publish a focal-pr-<number> channel
focal:
needs: [test]
if: |
always() && !cancelled() && contains(github.event.pull_request.labels.*.name, 'safe to test') &&
contains(github.event.pull_request.labels.*.name, 'focal-build') && needs.test.result == 'success'
uses: viamrobotics/rdk/.github/workflows/focal-build.yml@main
with:
release_type: 'pr'
secrets:
GCP_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS }}

droid:
needs: test
uses: viamrobotics/rdk/.github/workflows/droid.yml@main
Expand Down
79 changes: 79 additions & 0 deletions etc/Dockerfile.focal
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
# Minimal Focal toolchain image for building RDK's semi-static viam-server.
FROM ubuntu:focal

# GO_VERSION tracks the `go` directive in go.mod.
ARG GO_VERSION=1.25.9
ARG NLOPT_VERSION=2.11.0
ARG CMAKE_VERSION=4.3.4
# x264 stable branch. Built from source: focal's prebuilt libx264.a references
# __*_finite glibc symbols that fail to resolve when statically linked.
ARG X264_COMMIT=b35605ace3ddf7c1a5d67a2eb553f034aef41d55

ENV DEBIAN_FRONTEND=noninteractive
ENV PATH=/usr/local/go/bin:$PATH

# Build/link deps. nasm builds x264.
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
ca-certificates curl file git make nasm pkg-config sudo xz-utils \
libjpeg-turbo8-dev \
&& rm -rf /var/lib/apt/lists/*

# cmake >= 3.18 is required by nlopt >= 2.11; focal apt ships 3.16. Kitware
# prebuilt where available, pip wheel (manylinux_2_31 matches focal) on armhf.
RUN set -eux; \
case "$(dpkg --print-architecture)" in \
amd64|arm64) \
curl -fsSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" \
| tar -xz --strip-components=1 -C /usr/local ;; \
armhf) \
apt-get update && apt-get install -y --no-install-recommends python3-pip \
&& pip3 install "cmake==${CMAKE_VERSION}" \
&& rm -rf /var/lib/apt/lists/* ;; \
*) echo "unsupported arch" >&2; exit 1 ;; \
esac; \
cmake --version

# nlopt into /usr/local: headers + PIC static lib + pkg-config. NLOPT_CXX=OFF
# skips the C++-only algorithms so the library stays plain C.
RUN set -eux; \

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it make sense to extract all of these run commands to a single script? Seems like its mostly a single concern: ensuring that the appropriate versions of the packages are installed, something like the ensure-dependencies scripts we use elsewhere.

curl -fsSL "https://github.com/stevengj/nlopt/archive/refs/tags/v${NLOPT_VERSION}.tar.gz" \
| tar -C /tmp -xz; \
cmake -S "/tmp/nlopt-${NLOPT_VERSION}" -B /tmp/nlopt-build \
Comment thread
acmorrow marked this conversation as resolved.
-DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DNLOPT_CXX=OFF \
-DNLOPT_PYTHON=OFF -DNLOPT_OCTAVE=OFF -DNLOPT_MATLAB=OFF \
-DNLOPT_GUILE=OFF -DNLOPT_SWIG=OFF -DNLOPT_TESTS=OFF; \
cmake --build /tmp/nlopt-build -j "$(nproc)"; \
cmake --install /tmp/nlopt-build; \
rm -rf "/tmp/nlopt-${NLOPT_VERSION}" /tmp/nlopt-build

# x264 into /usr/local: headers + shared + PIC static + pkg-config.
Comment thread
acmorrow marked this conversation as resolved.
RUN set -eux; \
curl -fsSL "https://code.videolan.org/videolan/x264/-/archive/${X264_COMMIT}/x264-${X264_COMMIT}.tar.gz" \
| tar -C /tmp -xz; \
cd "/tmp/x264-${X264_COMMIT}"; \
./configure --prefix=/usr/local --enable-shared --enable-static --enable-pic --disable-cli --disable-opencl; \
make -j "$(nproc)"; \
make install; \
ldconfig; \
cd /; rm -rf "/tmp/x264-${X264_COMMIT}"

# Go toolchain.
RUN set -eux; \
case "$(dpkg --print-architecture)" in \
amd64) go_arch=amd64 ;; \
arm64) go_arch=arm64 ;; \
armhf) go_arch=armv6l ;; \
*) echo "unsupported arch" >&2; exit 1 ;; \
esac; \
curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${go_arch}.tar.gz" \
| tar -C /usr/local -xz; \
ln -s /usr/local/go/bin/go /usr/local/bin/go; \
go version

# Unprivileged build user (uid/gid 1000) with passwordless sudo.
RUN groupadd -g 1000 testbot \
&& useradd -u 1000 -g 1000 -m -s /bin/bash testbot \
&& echo 'testbot ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/testbot \
&& chmod 0440 /etc/sudoers.d/testbot
2 changes: 1 addition & 1 deletion etc/setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ do_linux(){
# Debian/Ubuntu
INSTALL_CMD="apt-get install --assume-yes build-essential procps curl file git debianutils"
if [ $(source /etc/os-release && echo $VERSION_CODENAME) = focal ]; then
echo -e ${COLOR_RED}WARNING:${COLOR_NULL} Ubuntu focal has known issues. Your build may fail.
echo -e ${COLOR_RED}NOTE:${COLOR_NULL} Focal is RDK\'s minimum baseline, but native brew-based dev on Focal is not fully validated yet. Your build may fail.
read -p "Continue? (y/n) " yesno
if [ $yesno != y ]; then
echo Okay, quitting
Expand Down
Loading