diff --git a/src/bci_build/package/__init__.py b/src/bci_build/package/__init__.py index 79bbc69f2..1235f9018 100644 --- a/src/bci_build/package/__init__.py +++ b/src/bci_build/package/__init__.py @@ -44,16 +44,6 @@ #: a sed statement to avoid using `EVALUATE=udev` in `/etc/blkid.conf` which doesn't work inside containers SET_BLKID_SCAN: str = f"# Avoid blkid waiting on udev (bsc#1247914)\n{DOCKERFILE_RUN} sed -i -e 's/^EVALUATE=.*/EVALUATE=scan/g' /etc/blkid.conf" -#: Remove various log files and temporary files. While it is possible to just ``rm -rf /var/log/*``, -#: that would also remove some package owned directories (not %ghost) -LOG_CLEAN: str = textwrap.dedent("""rm -rf {/target,}/var/log/{alternatives.log,lastlog,tallylog,zypper.log,zypp/history,YaST2}; \\ - rm -rf {/target,}/run/*; \\ - rm -f {/target,}/etc/{shadow-,group-,passwd-,.pwd.lock}; \\ - rm -f {/target,}/usr/lib/sysimage/rpm/.rpm.lock; \\ - rm -f {/target,}/var/cache/ldconfig/aux-cache; \\ - command -v zypper >/dev/null 2>&1 || rm -f /var/lib/zypp/AutoInstalled -""") - #: The string to use as a placeholder for the build source services to put in the release number _RELEASE_PLACEHOLDER = "%RELEASE%" @@ -198,16 +188,23 @@ class BaseContainerImage(abc.ABC): #: Packages to be installed inside the container image package_list: list[str] | list[Package] = field(default_factory=list) - #: This string is appended to the automatically generated dockerfile and can - #: contain arbitrary instructions valid for a :file:`Dockerfile`. + #: This string is appended to the end of the automatically generated dockerfile + #: and can contain arbitrary instructions valid for a :file:`Dockerfile`. #: #: .. note:: #: Setting both this property and :py:attr:`~BaseContainerImage.config_sh_script` #: is not possible and will result in an error. + #: + #: Installing packages or calling Zypper is forbidden, as it will polute the image + #: after cleanup. Use :py:attr:`~BaseContainerImage.build_stage_custom_end` instead. custom_end: str = "" - #: This string is appended to the the build stage in a multistage build and can - #: contain arbitrary instructions valid for a :file:`Dockerfile`. + #: This string is appended to the build area of the automatically generated dockerfile + #: and can contain arbitrary instructions valid for a :file:`Dockerfile`. + #: + #: .. note:: + #: Use this if you need to customize package installation, call Zypper, + #: or perform critical changes to the image. build_stage_custom_end: str | None = None #: This string defines which build counter identifier should be used for this @@ -571,20 +568,130 @@ def config_sh(self) -> str: {self.config_sh_script} -#======================================= -# Clean up after zypper if it is present -#--------------------------------------- +exit 0 +""" + + @property + def images_sh(self) -> str: + """The full :file:`images.sh` script used to cleanup kiwi builds.""" + return f"""#!{self.config_sh_interpreter} +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: (c) 2022-{datetime.datetime.now().date().strftime("%Y")} SUSE LLC + +{_BASH_SET} + +{self.file_cleanup_script} + +exit 0 +""" + + @property + def file_cleanup_script(self) -> str: + """ + The container cleanup script appended to `Dockerfiles` or used by `images.sh`. + + This will purge logs, temporary files, locks and everything else that should + not be shipped in containers, specially for reproducible builds. + """ + if self.custom_end and ( + "zypper -n" in self.custom_end + or "zypper --non-interactive" in self.custom_end + ): + raise ValueError( + "Zypper calls in `custom_end` are forbidden. Use `build_stage_custom_end` instead" + ) + + file_list = [ + # remove zypper-related logs + "/var/log/alternatives.log", + "/var/log/lastlog", + "/var/log/tallylog", + "/var/log/zypper.log", + "/var/log/zypp/history", + "/var/log/YaST2", + # remove zypp uuid (bsc#1098535) + "/var/lib/zypp/AnonymousUniqueId", + # remove the entire zypper cache content + "/var/cache/zypp/*", + # remove tmpfs data because it is ephemeral + "/run/*", + # remove user/group backups + "/etc/shadow-", + "/etc/group-", + "/etc/passwd-", + "/etc/.pwd.lock", + # remove rpm lock + "/usr/lib/sysimage/rpm/.rpm.lock", + # drop useless device/inode specific cache file + # see https://github.com/docker-library/official-images/issues/16044 + "/var/cache/ldconfig/aux-cache", + ] + + zypp_free_list = [ + # does not make sense in zypper-free images + "/var/lib/zypp/AutoInstalled", + # recreated by rpm on the next run + "/usr/lib/sysimage/rpm/Index.db", + ] + + if self.from_target_image: + target = "/target" + zypp_clean = f"zypper -n --installroot {target} clean -a" + else: + target = "" + zypp_clean = "zypper -n clean -a" + + if self.build_recipe_type == BuildType.KIWI: + rm_files = "\n".join(f"rm -vrf {target}{f}" for f in file_list) + rm_zypp_files = "\n".join( + f" rm -vrf {target}{f}" for f in zypp_free_list + ) + # this needs to be portable because it executes on all Kiwi images + return rf"""#====================================== +# Image Cleanup +#-------------------------------------- if command -v zypper > /dev/null; then - zypper -n clean -a + {zypp_clean} + # drop timestamp + tail -n +2 /var/lib/zypp/AutoInstalled > /var/lib/zypp/AutoInstalled.new && mv /var/lib/zypp/AutoInstalled.new /var/lib/zypp/AutoInstalled +else + # it does not make sense in a zypper-free image +{rm_zypp_files} fi -#============================================= -# Clean up logs and temporary files if present -#--------------------------------------------- -{LOG_CLEAN} +# set the day of last password change to empty +# prefer sed if available +if command -v sed > /dev/null; then + sed -i 's/^\([^:]*:[^:]*:\)[^:]*\(:.*\)$/\1\2/' {target}/etc/shadow +else + while IFS=: read -r username password last_change min_age max_age warn inactive expire reserved; do + echo "$username:$password::$min_age:$max_age:$warn:$inactive:$expire:$reserved" >> /etc/shadow.new + done < /etc/shadow + mv /etc/shadow.new /etc/shadow + chmod 640 /etc/shadow +fi -exit 0 +# remove logs and temporary files +{rm_files} """ + elif self.build_recipe_type == BuildType.DOCKER: + if self.from_target_image: + # Multi-stage images have zypper on the first stage + # but don't have a package manager on the final stage. + file_list = file_list + zypp_free_list + + rm_files = "\n".join(f" rm -vrf {target}{f}; \\" for f in file_list) + + return rf"""# image cleanup +{DOCKERFILE_RUN} {zypp_clean}; \ +{rm_files} + [ -f /var/lib/zypp/AutoInstalled ] && sed -i '1d' /var/lib/zypp/AutoInstalled; \ + sed -i 's/^\([^:]*:[^:]*:\)[^:]*\(:.*\)$/\1\2/' {target}/etc/shadow +""" + else: + raise ValueError( + f"Cleanup is tot supported for type '{self.build_recipe_type}'" + ) @property def _from_image(self) -> str | None: @@ -1083,7 +1190,7 @@ async def write_file_to_dest(fname: str, contents: str | bytes) -> None: image=self, INFOHEADER=infoheader, DOCKERFILE_RUN=DOCKERFILE_RUN, - LOG_CLEAN=LOG_CLEAN, + CLEAN_SCRIPT=self.file_cleanup_script, BUILD_FLAVOR=self.build_flavor, ) if dockerfile[-1] != "\n": @@ -1106,6 +1213,10 @@ async def write_file_to_dest(fname: str, contents: str | bytes) -> None: tasks.append(write_file_to_dest("config.sh", self.config_sh)) files.append("config.sh") + if self.images_sh: + tasks.append(write_file_to_dest("images.sh", self.images_sh)) + files.append("images.sh") + else: assert False, ( f"got an unexpected build_recipe_type: '{self.build_recipe_type}'" diff --git a/src/bci_build/package/base.py b/src/bci_build/package/base.py index a31f00f16..023e9beda 100644 --- a/src/bci_build/package/base.py +++ b/src/bci_build/package/base.py @@ -73,30 +73,6 @@ def _get_base_config_sh_script(os_version: OsVersion) -> str: zypper -n ar --refresh --gpgcheck --priority 100 --disable 'https://public-dl.suse.com/SUSE/Products/SLE-BCI/$releasever_major.$releasever_minor/$basearch/product_source/' SLE_BCI_source {%- endif %} -#====================================== -# Remove zypp uuid (bsc#1098535) -#-------------------------------------- -rm -f /var/lib/zypp/AnonymousUniqueId - -# Remove the entire zypper cache content (not the dir itself, owned by libzypp) -rm -rf /var/cache/zypp/* - -# drop timestamp -tail -n +2 /var/lib/zypp/AutoInstalled > /var/lib/zypp/AutoInstalled.new && mv /var/lib/zypp/AutoInstalled.new /var/lib/zypp/AutoInstalled - -# drop useless device/inode specific cache file (see https://github.com/docker-library/official-images/issues/16044) -rm -vf /var/cache/ldconfig/aux-cache - -# remove backup of /etc/{shadow,group,passwd} and lock file -rm -vf /etc/{shadow-,group-,passwd-,.pwd.lock} - -# drop pid and lock files -rm -vrf /run/* -rm -vf /usr/lib/sysimage/rpm/.rpm.lock - -# set the day of last password change to empty -sed -i 's/^\([^:]*:[^:]*:\)[^:]*\(:.*\)$/\1\2/' /etc/shadow - {% if os_version.is_tumbleweed -%} # Assign a fixed architecture in zypp.conf, to use the container's arch even if # the host arch differs (e.g. docker with --platform doesn't affect uname) @@ -108,14 +84,6 @@ def _get_base_config_sh_script(os_version: OsVersion) -> str: fi {%- endif -%} -#========================================== -# Hack! The go container management tools can't handle sparse files: -# https://github.com/golang/go/issues/13548 -# If lastlog doesn't exist, useradd doesn't attempt to reserve space, -# also in derived containers. -#------------------------------------------ -rm -f /var/log/lastlog - {% if os_version.is_sle15 and not os_version.is_ltss -%} #====================================== # Avoid blkid waiting on udev (bsc#1247914) diff --git a/src/bci_build/package/basecontainers.py b/src/bci_build/package/basecontainers.py index e63468bd2..d82e9cf24 100644 --- a/src/bci_build/package/basecontainers.py +++ b/src/bci_build/package/basecontainers.py @@ -17,7 +17,6 @@ from bci_build.os_version import _SUPPORTED_UNTIL_SLE from bci_build.os_version import OsVersion from bci_build.package import DOCKERFILE_RUN -from bci_build.package import LOG_CLEAN from bci_build.package import OsContainer from bci_build.package import Package from bci_build.package import generate_disk_size_constraints @@ -70,12 +69,6 @@ def _get_micro_package_list(os_version: OsVersion) -> list[Package]: {DOCKERFILE_RUN} zypper -n install jdupes \\ && jdupes -1 -L -r /target/usr/""") ), - custom_end=textwrap.dedent(f""" - # not making sense in a zypper-free image - {DOCKERFILE_RUN} rm -vf /var/lib/zypp/AutoInstalled - # includes device and inode numbers that change on deploy - {DOCKERFILE_RUN} rm -vf /var/cache/ldconfig/aux-cache - """), post_build_checks_containers=os_version in CAN_BE_SAC_VERSION, ) for os_version in ALL_BASE_OS_VERSIONS @@ -328,7 +321,6 @@ def _get_minimal_kwargs(os_version: OsVersion): package_list.extend(_get_micro_package_list(os_version)) package_list.append(Package("jdupes", pkg_type=PackageType.BOOTSTRAP)) - package_list.append(Package("sed", pkg_type=PackageType.BOOTSTRAP)) if os_version.is_sle15: # in SLE15, rpm still depends on Perl. package_list.extend( @@ -368,19 +360,6 @@ def _get_minimal_kwargs(os_version: OsVersion): # don't have duplicate licenses of the same type jdupes -1 -L -r /usr/share/licenses rpm -e jdupes - - # set the day of last password change to empty - sed -i 's/^\\([^:]*:[^:]*:\\)[^:]*\\(:.*\\)$/\\1\\2/' /etc/shadow - rpm -e sed - - # not making sense in a zypper-free image - rm -vf /var/lib/zypp/AutoInstalled - - # includes device and inode numbers that change on deploy - rm -vf /var/cache/ldconfig/aux-cache - - # Will be recreated by the next rpm(1) run as root user - rm -vf /usr/lib/sysimage/rpm/Index.db """ ), ) @@ -414,18 +393,6 @@ def _get_minimal_kwargs(os_version: OsVersion): config_sh_script=textwrap.dedent( """ sed -i 's|/bin/bash|/bin/sh|' /etc/passwd - - # not making sense in a zypper-free image - rm -vf /var/lib/zypp/AutoInstalled - - # includes device and inode numbers that change on deploy - rm -vf /var/cache/ldconfig/aux-cache - - # Will be recreated by the next rpm(1) run as root user - rm -vf /usr/lib/sysimage/rpm/Index.db - - # set the day of last password change to empty - sed -i 's/^\\([^:]*:[^:]*:\\)[^:]*\\(:.*\\)$/\\1\\2/' /etc/shadow """ ), config_sh_interpreter="/bin/sh", @@ -475,18 +442,17 @@ def _get_minimal_kwargs(os_version: OsVersion): + (["kernel-syms"] if os_version.is_sle15 else []) + (["suse-module-tools-scriptlets"] if os_version.is_sl16 else []) ), - custom_end=textwrap.dedent( + build_stage_custom_end=textwrap.dedent( f""" #!ArchExclusiveLine: aarch64 - {DOCKERFILE_RUN} if [ "$(uname -m)" = "aarch64" ] && zypper -n install kernel-64kb-devel; then zypper -n clean -a; fi + {DOCKERFILE_RUN} if [ "$(uname -m)" = "aarch64" ]; then zypper -n install kernel-64kb-devel; fi """ if os_version.is_sl16 else f""" #!ArchExclusiveLine: x86_64 aarch64 - {DOCKERFILE_RUN} if zypper -n install mokutil; then zypper -n clean -a; fi + {DOCKERFILE_RUN} zypper -n install mokutil """ - ) - + f"{DOCKERFILE_RUN} {LOG_CLEAN}", + ), extra_files={"_constraints": generate_disk_size_constraints(8)}, ) ) diff --git a/src/bci_build/package/golang.py b/src/bci_build/package/golang.py index f1af25f9f..531559257 100644 --- a/src/bci_build/package/golang.py +++ b/src/bci_build/package/golang.py @@ -8,7 +8,6 @@ from bci_build.os_version import CAN_BE_LATEST_OS_VERSION from bci_build.os_version import OsVersion from bci_build.package import DOCKERFILE_RUN -from bci_build.package import LOG_CLEAN from bci_build.package import DevelopmentContainer from bci_build.package import generate_disk_size_constraints from bci_build.replacement import Replacement @@ -66,17 +65,16 @@ def _get_golang_kwargs( parse_version=ParseVersion.PATCH, ) ], - "custom_end": ( + "build_stage_custom_end": ( textwrap.dedent( f""" # only available on go's tsan_arch architectures #!ArchExclusiveLine: x86_64 aarch64 s390x ppc64le - {DOCKERFILE_RUN} if zypper -n install {go}-race; then zypper -n clean -a; fi + {DOCKERFILE_RUN} zypper -n install {go}-race WORKDIR /go {DOCKERFILE_RUN} install -m 755 -d /go/bin /go/src """ ) - + f"{DOCKERFILE_RUN} {LOG_CLEAN}" ), "package_list": [*go_packages, "make"] + os_version.common_devel_packages diff --git a/src/bci_build/templates.py b/src/bci_build/templates.py index cda701d95..c7e35534f 100644 --- a/src/bci_build/templates.py +++ b/src/bci_build/templates.py @@ -65,14 +65,9 @@ {%- if image.build_stage_custom_end %} {{ image.build_stage_custom_end }} {%- endif %} -{% if image.packages %} -# cleanup logs and temporary files -{{ DOCKERFILE_RUN }} zypper -n {%- if image.from_target_image %} --installroot /target {%- endif %} clean -a; \\ - {{ LOG_CLEAN }} -{%- endif %} -# set the day of last password change to empty -{{ DOCKERFILE_RUN }} sed -i 's/^\\([^:]*:[^:]*:\\)[^:]*\\(:.*\\)$/\\1\\2/' {% if image.from_target_image %}/target{% endif %}/etc/shadow -{% if image.from_target_image %}FROM {{ image.dockerfile_from_target_ref }} +{{ CLEAN_SCRIPT }} +{%- if image.from_target_image %} +FROM {{ image.dockerfile_from_target_ref }} COPY --from=builder /target /{% endif %} # Define labels according to https://en.opensuse.org/Building_derived_containers # labelprefix={{ image.labelprefix }} diff --git a/src/dotnet/updater.py b/src/dotnet/updater.py index 0f7d97b4b..acd974056 100644 --- a/src/dotnet/updater.py +++ b/src/dotnet/updater.py @@ -22,7 +22,6 @@ from bci_build.logger import LOGGER from bci_build.os_version import CAN_BE_LATEST_OS_VERSION from bci_build.os_version import OsVersion -from bci_build.package import LOG_CLEAN from bci_build.package import DevelopmentContainer from bci_build.package import generate_disk_size_constraints from staging.build_result import Arch @@ -103,10 +102,6 @@ COPY prod.repo /etc/zypp/repos.d/microsoft-dotnet-prod.repo COPY dotnet-host.check /etc/zypp/systemCheck.d/dotnet-host.check -RUN rm -rf /tmp/* && zypper clean -a && """ - + LOG_CLEAN - + """ - {% if not image.is_sdk and image.use_nonprivileged_user %} ENV APP_UID=1654 ASPNETCORE_HTTP_PORTS=8080 DOTNET_RUNNING_IN_CONTAINER=true ENV DOTNET_VERSION={{ dotnet_version }} @@ -317,7 +312,7 @@ def prepare_template(self) -> None: f"additional_versions property must be unset, but got {self.additional_versions}" ) - self.custom_end = CUSTOM_END_TEMPLATE.render( + self.build_stage_custom_end = CUSTOM_END_TEMPLATE.render( image=self, dotnet_packages=pkgs, dotnet_version=self.version, diff --git a/tests/test_build_recipe.py b/tests/test_build_recipe.py index 77a4a3ced..e894de737 100644 --- a/tests/test_build_recipe.py +++ b/tests/test_build_recipe.py @@ -36,13 +36,7 @@ RUN \\ zypper -n install --no-recommends gcc emacs - -# cleanup logs and temporary files -RUN zypper -n clean -a; \\ - ##LOGCLEAN## -# set the day of last password change to empty -RUN sed -i 's/^\\([^:]*:[^:]*:\\)[^:]*\\(:.*\\)$/\\1\\2/' /etc/shadow - +##LOGCLEAN## # Define labels according to https://en.opensuse.org/Building_derived_containers # labelprefix=com.suse.bci.test LABEL org.opencontainers.image.authors="https://github.com/SUSE/bci/discussions" @@ -157,13 +151,7 @@ RUN \\ zypper -n install --no-recommends gcc emacs - -# cleanup logs and temporary files -RUN zypper -n clean -a; \\ - ##LOGCLEAN## -# set the day of last password change to empty -RUN sed -i 's/^\\([^:]*:[^:]*:\\)[^:]*\\(:.*\\)$/\\1\\2/' /etc/shadow - +##LOGCLEAN## # Define labels according to https://en.opensuse.org/Building_derived_containers # labelprefix=com.suse.bci.test LABEL org.opencontainers.image.authors="https://github.com/SUSE/bci/discussions" @@ -265,13 +253,7 @@ RUN \\ zypper -n install --no-recommends gcc emacs - -# cleanup logs and temporary files -RUN zypper -n clean -a; \\ - ##LOGCLEAN## -# set the day of last password change to empty -RUN sed -i 's/^\\([^:]*:[^:]*:\\)[^:]*\\(:.*\\)$/\\1\\2/' /etc/shadow - +##LOGCLEAN## # Define labels according to https://en.opensuse.org/Building_derived_containers # labelprefix=com.suse.bci.test LABEL org.opencontainers.image.authors="https://github.com/SUSE/bci/discussions" @@ -383,13 +365,7 @@ RUN \\ zypper -n install --no-recommends gcc emacs - -# cleanup logs and temporary files -RUN zypper -n clean -a; \\ - ##LOGCLEAN## -# set the day of last password change to empty -RUN sed -i 's/^\\([^:]*:[^:]*:\\)[^:]*\\(:.*\\)$/\\1\\2/' /etc/shadow - +##LOGCLEAN## # Define labels according to https://en.opensuse.org/Building_derived_containers # labelprefix=org.opensuse.bci.test LABEL org.opencontainers.image.authors="invalid@suse.com" @@ -533,7 +509,7 @@ def test_build_recipe_templates( DOCKERFILE_RUN="RUN", image=image, INFOHEADER="# Copyright header", - LOG_CLEAN="##LOGCLEAN##", + CLEAN_SCRIPT="##LOGCLEAN##", ) == dockerfile ) @@ -658,12 +634,7 @@ def test_os_build_recipe_templates(kiwi_xml: str, image: OsContainer) -> None: export CHKSTAT_ALLOW_INSECURE_MODE_IF_NO_PROC=1; \\ zypper -n --installroot /target --gpg-auto-import-keys install --no-recommends emacs; \\ zypper -n --installroot /target remove util-linux - -# cleanup logs and temporary files -RUN zypper -n --installroot /target clean -a; \\ - ##LOGCLEAN## -# set the day of last password change to empty -RUN sed -i 's/^\\([^:]*:[^:]*:\\)[^:]*\\(:.*\\)$/\\1\\2/' /target/etc/shadow +##LOGCLEAN## FROM registry.suse.com/bci/bci-micro:15.7 COPY --from=builder /target / # Define labels according to https://en.opensuse.org/Building_derived_containers @@ -725,12 +696,7 @@ def test_os_build_recipe_templates(kiwi_xml: str, image: OsContainer) -> None: export CHKSTAT_ALLOW_INSECURE_MODE_IF_NO_PROC=1; \\ zypper -n --installroot /target --gpg-auto-import-keys install --no-recommends git-core; \\ zypper -n --installroot /target remove util-linux - -# cleanup logs and temporary files -RUN zypper -n --installroot /target clean -a; \\ - ##LOGCLEAN## -# set the day of last password change to empty -RUN sed -i 's/^\\([^:]*:[^:]*:\\)[^:]*\\(:.*\\)$/\\1\\2/' /target/etc/shadow +##LOGCLEAN## FROM registry.suse.com/bci/bci-micro:15.7 COPY --from=builder /target / # Define labels according to https://en.opensuse.org/Building_derived_containers @@ -779,7 +745,7 @@ def test_appcollection_app_templates( DOCKERFILE_RUN="RUN", image=image, INFOHEADER="# Copyright header", - LOG_CLEAN="##LOGCLEAN##", + CLEAN_SCRIPT="##LOGCLEAN##", ) == dockerfile )