diff --git a/.github/workflows/imagebuild.yml b/.github/workflows/imagebuild.yml index 48f2a4329..03a028eb4 100644 --- a/.github/workflows/imagebuild.yml +++ b/.github/workflows/imagebuild.yml @@ -66,7 +66,7 @@ jobs: --build-arg COMMIT_SHA=v$tag \ --build-arg BUILD_ARGS='-DBUILD_TESTS=OFF -DBUILD_GUI=OFF -DBUILD_BENCH=OFF -DBUILD_UTIL=ON -DBUILD_FUZZ_BINARY=OFF -DWITH_ZMQ=ON' \ -t bitcoindevproject/bitcoin:$tag \ - --file resources/images/bitcoin/Dockerfile.dev \ + --file resources/images/bitcoin/Dockerfile.cmake \ resources/images/bitcoin/ \ --push done diff --git a/docker-bake.hcl b/docker-bake.hcl deleted file mode 100644 index 93a51e33d..000000000 --- a/docker-bake.hcl +++ /dev/null @@ -1,230 +0,0 @@ -group "all" { - targets = [ - "bitcoin-28-1", - "bitcoin-27", - "bitcoin-26", - "v0-21-1", - "v0-20-0", - "v0-19-2", - "v0-17-0", - "v0-16-1", - "bitcoin-unknown-message", - "bitcoin-invalid-blocks", - "bitcoin-50-orphans", - "bitcoin-no-mp-trim", - "bitcoin-disabled-opcodes", - "bitcoin-5k-inv" - ] -} - -group "maintained" { - targets = [ - "bitcoin-28-1", - "bitcoin-27", - "bitcoin-26" - ] -} - -group "practice" { - targets = [ - "bitcoin-unknown-message", - "bitcoin-invalid-blocks", - "bitcoin-50-orphans", - "bitcoin-no-mp-trim", - "bitcoin-disabled-opcodes", - "bitcoin-5k-inv" - ] -} - -group "vulnerable" { - targets = [ - "v0-21-1", - "v0-20-0", - "v0-19-2", - "v0-17-0", - "v0-16-1", - ] -} - -target "maintained-base" { - context = "./resources/images/bitcoin" - args = { - REPO = "bitcoin/bitcoin" - BUILD_ARGS = "--disable-tests --without-gui --disable-bench --disable-fuzz-binary --enable-suppress-external-warnings" - } - platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7"] -} - -target "cmake-base" { - inherits = ["maintained-base"] - dockerfile = "./Dockerfile.dev" - args = { - BUILD_ARGS = "-DBUILD_TESTS=OFF -DBUILD_GUI=OFF -DBUILD_BENCH=OFF -DBUILD_UTIL=ON -DBUILD_FUZZ_BINARY=OFF -DWITH_ZMQ=ON" - } -} - -target "autogen-base" { - inherits = ["maintained-base"] - dockerfile = "./Dockerfile" -} - -target "bitcoin-28-1" { - inherits = ["autogen-base"] - tags = ["bitcoindevproject/bitcoin:28.1"] - args = { - COMMIT_SHA = "32efe850438ef22e2de39e562af557872a402c31" - } -} - -target "bitcoin-27" { - inherits = ["autogen-base"] - tags = ["bitcoindevproject/bitcoin:27.2"] - args = { - COMMIT_SHA = "bf03c458e994abab9be85486ed8a6d8813313579" - } -} - -target "bitcoin-26" { - inherits = ["autogen-base"] - tags = ["bitcoindevproject/bitcoin:26.2"] - args = { - COMMIT_SHA = "7b7041019ba5e7df7bde1416aa6916414a04f3db" - } -} - -target "practice-base" { - dockerfile = "./Dockerfile" - context = "./resources/images/bitcoin/insecure" - contexts = { - bitcoin-src = "." - } - args = { - ALPINE_VERSION = "3.20" - BITCOIN_VERSION = "28.1.1" - EXTRA_PACKAGES = "sqlite-dev" - EXTRA_RUNTIME_PACKAGES = "" - REPO = "willcl-ark/bitcoin" - } - platforms = ["linux/amd64", "linux/armhf"] -} - -target "bitcoin-unknown-message" { - inherits = ["practice-base"] - tags = ["bitcoindevproject/bitcoin:99.0.0-unknown-message"] - args = { - COMMIT_SHA = "ae999611026e941eca5c0b61f22012c3b3f3d8dc" - } -} - -target "bitcoin-invalid-blocks" { - inherits = ["practice-base"] - tags = ["bitcoindevproject/bitcoin:98.0.0-invalid-blocks"] - args = { - COMMIT_SHA = "9713324368e5a966ec330389a533ae8ad7a0ea8f" - } -} - -target "bitcoin-50-orphans" { - inherits = ["practice-base"] - tags = ["bitcoindevproject/bitcoin:97.0.0-50-orphans"] - args = { - COMMIT_SHA = "cbcb308eb29621c0db3a105e1a1c1788fb0dab6b" - } -} - -target "bitcoin-no-mp-trim" { - inherits = ["practice-base"] - tags = ["bitcoindevproject/bitcoin:96.0.0-no-mp-trim"] - args = { - COMMIT_SHA = "a3a15a9a06dd541d1dafba068c00eedf07e1d5f8" - } -} - -target "bitcoin-disabled-opcodes" { - inherits = ["practice-base"] - tags = ["bitcoindevproject/bitcoin:95.0.0-disabled-opcodes"] - args = { - COMMIT_SHA = "5bdb8c52a8612cac9aa928c84a499dd701542b2a" - } -} - -target "bitcoin-5k-inv" { - inherits = ["practice-base"] - tags = ["bitcoindevproject/bitcoin:94.0.0-5k-inv"] - args = { - COMMIT_SHA = "e70e610e07eea3aeb0c49ae0bd9f4049ffc1b88c" - } -} - -target "CVE-base" { - dockerfile = "./Dockerfile" - context = "./resources/images/bitcoin/insecure" - contexts = { - bitcoin-src = "." - } - platforms = ["linux/amd64", "linux/armhf"] - args = { - REPO = "josibake/bitcoin" - } -} - -target "v0-16-1" { - inherits = ["CVE-base"] - tags = ["bitcoindevproject/bitcoin:0.16.1"] - args = { - ALPINE_VERSION = "3.7" - BITCOIN_VERSION = "0.16.1" - COMMIT_SHA = "dc94c00e58c60412a4e1a540abdf0b56093179e8" - EXTRA_PACKAGES = "protobuf-dev libressl-dev" - EXTRA_RUNTIME_PACKAGES = "boost boost-program_options libressl" - PRE_CONFIGURE_COMMANDS = "sed -i '/AC_PREREQ/a\\AR_FLAGS=cr' src/univalue/configure.ac && sed -i '/AX_PROG_CC_FOR_BUILD/a\\AR_FLAGS=cr' src/secp256k1/configure.ac && sed -i 's:sys/fcntl.h:fcntl.h:' src/compat.h" - } -} - -target "v0-17-0" { - inherits = ["CVE-base"] - tags = ["bitcoindevproject/bitcoin:0.17.0"] - args = { - ALPINE_VERSION = "3.9" - BITCOIN_VERSION = "0.17.0" - COMMIT_SHA = "f6b2db49a707e7ad433d958aee25ce561c66521a" - EXTRA_PACKAGES = "protobuf-dev libressl-dev" - EXTRA_RUNTIME_PACKAGES = "boost boost-program_options libressl sqlite-dev" - } -} - -target "v0-19-2" { - inherits = ["CVE-base"] - tags = ["bitcoindevproject/bitcoin:0.19.2"] - args = { - ALPINE_VERSION = "3.12.12" - BITCOIN_VERSION = "0.19.2" - COMMIT_SHA = "e20f83eb5466a7d68227af14a9d0cf66fb520ffc" - EXTRA_PACKAGES = "sqlite-dev libressl-dev" - EXTRA_RUNTIME_PACKAGES = "boost boost-program_options libressl sqlite-dev" - } -} - -target "v0-20-0" { - inherits = ["CVE-base"] - tags = ["bitcoindevproject/bitcoin:0.20.0"] - args = { - ALPINE_VERSION = "3.12.12" - BITCOIN_VERSION = "0.20.0" - COMMIT_SHA = "0bbff8feff0acf1693dfe41184d9a4fd52001d3f" - EXTRA_PACKAGES = "sqlite-dev miniupnpc-dev" - EXTRA_RUNTIME_PACKAGES = "boost-filesystem miniupnpc-dev sqlite-dev" - } -} - -target "v0-21-1" { - inherits = ["CVE-base"] - tags = ["bitcoindevproject/bitcoin:0.21.1"] - args = { - ALPINE_VERSION = "3.17" - BITCOIN_VERSION = "0.21.1" - COMMIT_SHA = "e0a22f14c15b4877ef6221f9ee2dfe510092d734" - EXTRA_PACKAGES = "sqlite-dev" - EXTRA_RUNTIME_PACKAGES = "boost-filesystem sqlite-dev" - } -} diff --git a/docs/developer-notes.md b/docs/developer-notes.md index 3952a7a9c..3dec95b31 100644 --- a/docs/developer-notes.md +++ b/docs/developer-notes.md @@ -75,31 +75,5 @@ python3 -m twine upload dist/* ## Building docker images -The Bitcoin Core docker images used by warnet are specified in the *docker-bake.hcl* file. -This uses the (experimental) `bake` build functionality of docker buildx. -We use [HCL language](https://github.com/hashicorp/hcl) in the declaration file itself. -See the `bake` [documentation](https://docs.docker.com/build/bake/) for more information on specifications, and how to e.g. override arguments. +See [docs for `warnet image build` command](./warnet.md#warnet-image-build) -In order to build (or "bake") a certain image, find the image's target (name) in the *docker-bake.hcl* file, and then run `docker buildx bake `. - -```bash -# build the dummy image that will crash on 5k invs -docker buildx bake bitcoin-5k-inv - -# build the same image, but set platform to only linux/amd64 -docker buildx bake bitcoin-5k-inv --set bitcoin-5k-inv.platform=linux/amd64 -``` - -To load the single-platform build result to `docker images`, run: - -```bash -docker buildx bake --load bitcoin-5k-inv -``` - -Push the build result to a registry by running: - -```bash -docker buildx bake --push bitcoin-5k-inv -``` - -It will automatically push the build result to registry. diff --git a/resources/images/bitcoin/Dockerfile b/resources/images/bitcoin/Dockerfile.autotools similarity index 57% rename from resources/images/bitcoin/Dockerfile rename to resources/images/bitcoin/Dockerfile.autotools index 6efba5335..7209f03ea 100644 --- a/resources/images/bitcoin/Dockerfile +++ b/resources/images/bitcoin/Dockerfile.autotools @@ -1,5 +1,5 @@ # Setup deps stage -FROM alpine AS deps +FROM alpine:3.20 AS deps ARG REPO ARG COMMIT_SHA ARG BUILD_ARGS @@ -20,7 +20,8 @@ RUN --mount=type=cache,target=/var/cache/apk \ libtool \ linux-headers \ sqlite-dev \ - zeromq-dev + zeromq-dev \ + db-dev COPY isroutable.patch /tmp/ COPY addrman.patch /tmp/ @@ -31,19 +32,37 @@ FROM deps AS build ENV BITCOIN_PREFIX=/opt/bitcoin WORKDIR /build +# Assume $COMMIT_SHA is a git commit but if that fails, then +# assume it is a release tag that can be checked out RUN set -ex \ && cd /build \ - && git clone --depth 1 "https://github.com/${REPO}" \ - && cd bitcoin \ - && git fetch --depth 1 origin "$COMMIT_SHA" \ - && git checkout "$COMMIT_SHA" \ - && git apply /tmp/isroutable.patch \ - && git apply /tmp/addrman.patch \ - && sed -i s:sys/fcntl.h:fcntl.h: src/compat/compat.h \ + && git init \ + && git remote add origin "https://github.com/${REPO}" \ + # Try commit/branch first + && if git fetch --depth 1 origin "$COMMIT_SHA:refs/temp/ref" 2>/dev/null; then \ + REF=refs/temp/ref; \ + else \ + # Fallback: fetch tag explicitly into local refs + git fetch --depth 1 origin "refs/tags/$COMMIT_SHA:refs/tags/$COMMIT_SHA"; \ + REF="refs/tags/$COMMIT_SHA"; \ + fi \ + # Resolve tag -> commit if needed + && resolved=$(git rev-parse --verify "$REF^{commit}") \ + && git checkout "$resolved" \ + # Build + && git apply /tmp/isroutable.patch || true \ + && git apply /tmp/addrman.patch || true \ + && sed -i s:sys/fcntl.h:fcntl.h: src/compat/compat.h || true \ && ./autogen.sh \ + && DB_PREFIX="$(ls -d /opt/db-* 2>/dev/null | head -n1 || true)" \ + && if [ -n "$DB_PREFIX" ] && [ -d "$DB_PREFIX/lib" ]; then \ + export LDFLAGS="-L${DB_PREFIX}/lib"; \ + export CPPFLAGS="-g0 -I${DB_PREFIX}/include --param ggc-min-expand=1 --param ggc-min-heapsize=32768"; \ + else \ + export LDFLAGS=""; \ + export CPPFLAGS="-g0 --param ggc-min-expand=1 --param ggc-min-heapsize=32768"; \ + fi \ && ./configure \ - LDFLAGS=-L`ls -d /opt/db*`/lib/ \ - CPPFLAGS="-g0 -I`ls -d /opt/db*`/include/ --param ggc-min-expand=1 --param ggc-min-heapsize=32768" \ --prefix=${BITCOIN_PREFIX} \ ${BUILD_ARGS} \ && make -j$(nproc) \ @@ -56,8 +75,7 @@ RUN set -ex \ && rm ${BITCOIN_PREFIX}/bin/bitcoin-wallet \ && rm ${BITCOIN_PREFIX}/bin/bitcoin-util -# Final clean stage -FROM alpine +FROM alpine:3.20 ARG UID=100 ARG GID=101 ENV BITCOIN_DATA=/root/.bitcoin @@ -84,4 +102,3 @@ EXPOSE 8332 8333 18332 18333 18443 18444 38333 38332 ENTRYPOINT ["/entrypoint.sh"] CMD ["bitcoind"] - diff --git a/resources/images/bitcoin/Dockerfile.dev b/resources/images/bitcoin/Dockerfile.cmake similarity index 99% rename from resources/images/bitcoin/Dockerfile.dev rename to resources/images/bitcoin/Dockerfile.cmake index 97ee6eb2f..acb14160f 100644 --- a/resources/images/bitcoin/Dockerfile.dev +++ b/resources/images/bitcoin/Dockerfile.cmake @@ -38,21 +38,18 @@ RUN set -ex \ && cd /build \ && git init \ && git remote add origin "https://github.com/${REPO}" \ - # Try commit/branch first && if git fetch --depth 1 origin "$COMMIT_SHA:refs/temp/ref" 2>/dev/null; then \ REF=refs/temp/ref; \ else \ - # Fallback: fetch tag explicitly into local refs git fetch --depth 1 origin "refs/tags/$COMMIT_SHA:refs/tags/$COMMIT_SHA"; \ REF="refs/tags/$COMMIT_SHA"; \ fi \ - # Resolve tag -> commit if needed && resolved=$(git rev-parse --verify "$REF^{commit}") \ && git checkout "$resolved" \ - + # Build && git apply /tmp/isroutable.patch \ && git apply /tmp/addrman.patch \ && sed -i s:sys/fcntl.h:fcntl.h: src/compat/compat.h \ @@ -66,7 +63,6 @@ RUN set -ex \ && rm -f ${BITCOIN_PREFIX}/lib/libbitcoinconsensus.a \ && rm -f ${BITCOIN_PREFIX}/lib/libbitcoinconsensus.so.0.0.0 -# Final clean stage FROM alpine:3.20 ARG UID=100 ARG GID=101 diff --git a/src/warnet/image_build.py b/src/warnet/image_build.py index 042315bd9..a51310b50 100644 --- a/src/warnet/image_build.py +++ b/src/warnet/image_build.py @@ -1,9 +1,70 @@ import subprocess +import urllib.error +import urllib.request from importlib.resources import files ARCHES = ["amd64", "arm64"] -dockerfile_path = files("resources.images.bitcoin").joinpath("Dockerfile.dev") +CMAKE_BUILD_ARGS = ( + '"-DBUILD_TESTS=OFF -DBUILD_GUI=OFF -DBUILD_BENCH=OFF' + " -DBUILD_UTIL=ON -DBUILD_FUZZ_BINARY=OFF -DWITH_ZMQ=ON" + ' "' +) +AUTOTOOLS_BUILD_ARGS = ( + '"--disable-tests --without-gui --disable-bench' + " --disable-fuzz-binary --enable-suppress-external-warnings" + ' --with-incompatible-bdb"' +) + +dockerfile_cmake = files("resources.images.bitcoin").joinpath("Dockerfile.cmake") +dockerfile_autotools = files("resources.images.bitcoin").joinpath("Dockerfile.autotools") + + +def detect_build_system(repo: str, commit_sha: str) -> str: + """Detect whether a Bitcoin Core ref uses CMake or autotools. + + Probes the GitHub raw content URL for CMakeLists.txt at the given ref + (branch, tag, or commit SHA). A 200 means cmake, a 404 means autotools. + Any other error is logged and falls back to cmake so we don't silently + pick the wrong build system when the probe itself misbehaves. + """ + url = f"https://raw.githubusercontent.com/{repo}/{commit_sha}/CMakeLists.txt" + print(f"Detecting build system: GET {url}") + try: + with urllib.request.urlopen(url, timeout=15) as resp: + print(f" HTTP {resp.status} -> cmake") + return "cmake" + except urllib.error.HTTPError as e: + if e.code == 404: + api_url = ( + f"https://api.github.com/repos/{repo}/contents/CMakeLists.txt?ref={commit_sha}" + ) + req = urllib.request.Request(api_url, headers={"User-Agent": "warnet-image-build"}) + try: + with urllib.request.urlopen(req, timeout=15) as api_resp: + print(f" HTTP {api_resp.status} -> cmake") + return "cmake" + except urllib.error.HTTPError as api_error: + body = api_error.read().decode() + if api_error.code == 404: + if "No commit found for the ref" in body: + raise ValueError(f"Ref not found in {repo}: {commit_sha}") from api_error + print(f" HTTP 404 -> autotools") + return "autotools" + print( + f"Warning: HTTP {api_error.code} {api_error.reason} probing {api_url}; defaulting to cmake" + ) + return "cmake" + except urllib.error.URLError as api_error: + print( + f"Warning: could not verify 404 from GitHub API: {api_error}; defaulting to cmake" + ) + return "cmake" + print(f"Warning: HTTP {e.code} {e.reason} probing {url}; defaulting to cmake") + return "cmake" + except urllib.error.URLError as e: + print(f"Warning: could not reach GitHub to detect build system: {e}; defaulting to cmake") + return "cmake" def run_command(command): @@ -22,10 +83,21 @@ def build_image( arches: str, action: str, ): - if not build_args: - build_args = '"-DBUILD_TESTS=OFF -DBUILD_GUI=OFF -DBUILD_BENCH=OFF -DBUILD_UTIL=ON -DBUILD_FUZZ_BINARY=OFF -DWITH_ZMQ=ON "' + try: + build_system = detect_build_system(repo, commit_sha) + except ValueError as e: + print(f"Error: {e}") + return False + print(f"Detected build system: {build_system}") + + if build_system == "cmake": + dockerfile_path = dockerfile_cmake + default_build_args = CMAKE_BUILD_ARGS else: - build_args = f'"{build_args}"' + dockerfile_path = dockerfile_autotools + default_build_args = AUTOTOOLS_BUILD_ARGS + + build_args = default_build_args if not build_args else f'"{build_args}"' build_arches = [] if not arches: @@ -38,8 +110,8 @@ def build_image( print(f"{tags=:}") print(f"{build_args=:}") print(f"{build_arches=:}") + print(f"Using Dockerfile: {dockerfile_path}") - # Setup buildkit builder_name = "bitcoind-builder" create_builder_cmd = f"docker buildx create --name {builder_name} --use" use_builder_cmd = f"docker buildx use --builder {builder_name}"