Docker implementation #1219
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -245,7 +245,8 @@ | |
| "all", | ||
| "exe", | ||
| "pkg", | ||
| "sh", | ||
| "docker" | ||
| ], | ||
| "title": "InstallerTypes", | ||
| "type": "string" | ||
|
|
@@ -607,7 +608,7 @@ | |
| } | ||
| ], | ||
| "default": null, | ||
| "description": "Set default install prefix. On Linux, if not provided, the default prefix is `${HOME}/<NAME>` (or, if `HOME` is not set, `/opt/<NAME>`). On Windows, this is used only for \"Just Me\" installations; for \"All Users\" installations, use the `default_prefix_all_users` key. If not provided, the default prefix is `%USERPROFILE%\\<NAME>`. Environment variables will be expanded at install time. If creating a Docker output, the default is `/opt/<NAME>` and can be overridden during the Docker build process.", | ||
| "title": "Default Prefix" | ||
| }, | ||
| "default_prefix_all_users": { | ||
|
|
@@ -638,6 +639,65 @@ | |
| "description": "Set default installation prefix for domain users. If not provided, the installation prefix for domain users will be `%LOCALAPPDATA%\\<NAME>`. By default, it is different from the `default_prefix` value to avoid installing the distribution into the roaming profile. Environment variables will be expanded at install time. Windows only.", | ||
| "title": "Default Prefix Domain User" | ||
| }, | ||
| "docker_base_image": { | ||
| "anyOf": [ | ||
| { | ||
| "minLength": 1, | ||
| "type": "string" | ||
| }, | ||
| { | ||
| "type": "null" | ||
| } | ||
| ], | ||
| "default": null, | ||
| "description": "Required to use docker-related features. Base image to use for docker builds and Dockerfiles. This can be any valid docker image reference, including a tag and/or digest. For example: `debian:13.4-slim@sha256:abc123...`.", | ||
| "title": "Docker Base Image" | ||
| }, | ||
| "docker_image": { | ||
| "anyOf": [ | ||
| { | ||
| "enum": [ | ||
| "tar", | ||
| "gz", | ||
| "zst" | ||
| ], | ||
| "type": "string" | ||
| }, | ||
| { | ||
| "type": "null" | ||
| } | ||
| ], | ||
| "default": null, | ||
| "description": "If set, builds a docker image using the Dockerfile generated by constructor and saves it as a portable tarball either uncompressed or compressed. ``<name>-<version>-<platform>-<arch>-docker.tar`` will be created in the output docker directory.", | ||
|
Contributor
There was a problem hiding this comment. Is the format |
||
| "title": "Docker Image" | ||
| }, | ||
| "docker_labels": { | ||
| "additionalProperties": { | ||
| "minLength": 1, | ||
| "type": "string" | ||
| }, | ||
| "default": {}, | ||
| "description": "Additional labels to add to the built docker image. The labels `org.opencontainers.image.title` and `org.opencontainers.image.version` are set automatically from `name` and `version`.", | ||
| "propertyNames": { | ||
| "minLength": 1 | ||
| }, | ||
| "title": "Docker Labels", | ||
| "type": "object" | ||
| }, | ||
| "docker_tag": { | ||
| "anyOf": [ | ||
| { | ||
| "minLength": 1, | ||
| "type": "string" | ||
| }, | ||
| { | ||
| "type": "null" | ||
| } | ||
| ], | ||
| "default": null, | ||
| "description": "Tag to use for the docker image. If not provided, it will default to `<name>:<version>`. Has no effect if not using the `docker_image` feature.", | ||
| "title": "Docker Tag" | ||
| }, | ||
| "environment": { | ||
| "anyOf": [ | ||
| { | ||
|
|
@@ -864,7 +924,7 @@ | |
| } | ||
| ], | ||
| "default": null, | ||
| "description": "The type of the installer being created. Possible values are:\n- `sh`: shell-based installer for Linux or macOS\n- `pkg`: macOS GUI installer built with Apple's `pkgbuild`\n- `exe`: Windows GUI installer built with NSIS\n- `docker`: generates a `Dockerfile` or image of the installation environment.\nThe default type is `sh` on Linux and macOS, and `exe` on Windows. A special value of `all` builds _both_ `sh` and `pkg` installers on macOS, as well as `sh` on Linux and `exe` on Windows.", | ||
| "title": "Installer Type" | ||
| }, | ||
| "keep_pkgs": { | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,166 @@ | ||||||||
| """Logic for creating a Dockerfile and/or building portable Docker images from Constructor installers.""" | ||||||||
|
|
||||||||
| import logging | ||||||||
| import shutil | ||||||||
| import subprocess | ||||||||
| import tempfile | ||||||||
| from pathlib import Path | ||||||||
|
|
||||||||
| from jinja2 import Template | ||||||||
|
|
||||||||
| from . import __version__ | ||||||||
|
|
||||||||
| logger = logging.getLogger(__name__) | ||||||||
|
|
||||||||
| TEMPLATE_PATH = Path(__file__).parent / "dockerfile_template.tmpl" | ||||||||
|
|
||||||||
| DOCKER_PLATFORM_MAP = { | ||||||||
| "linux-64": "linux/amd64", | ||||||||
| "linux-aarch64": "linux/arm64", | ||||||||
| "linux-armv7l": "linux/arm/v7", | ||||||||
| "linux-32": "linux/386", | ||||||||
| "linux-ppc64le": "linux/ppc64le", | ||||||||
| "linux-s390x": "linux/s390x", | ||||||||
| } | ||||||||
|
|
||||||||
|
|
||||||||
| def generate_dockerfile(info: dict, docker_dir: Path) -> Path: | ||||||||
| """ | ||||||||
| Render the Dockerfile template and write it to the Docker build directory. | ||||||||
|
|
||||||||
| Parameters | ||||||||
| ---------- | ||||||||
| info: dict | ||||||||
| Constructor installer info dict. | ||||||||
| docker_dir: Path | ||||||||
| Path to the Docker build directory returned by prepare_docker_context(). | ||||||||
|
|
||||||||
| Returns | ||||||||
| ------- | ||||||||
| Path | ||||||||
| Path to the generated Dockerfile. | ||||||||
| """ | ||||||||
| from .conda_interface import MatchSpec | ||||||||
|
|
||||||||
| specs = {MatchSpec(spec).name for spec in info.get("specs", ())} | ||||||||
|
Contributor
There was a problem hiding this comment.
Suggested change
That will make things more clear later. Should probably also be done with |
||||||||
| has_mamba = "mamba" in specs | ||||||||
|
|
||||||||
| docker_template = Template(TEMPLATE_PATH.read_text()) | ||||||||
|
|
||||||||
| rendered_dockerfile = docker_template.render( | ||||||||
| constructor_version=__version__, | ||||||||
| base_image=info.get("docker_base_image"), | ||||||||
| default_prefix=info.get("default_prefix", f"/opt/{info['name'].lower()}"), | ||||||||
|
Jrice1317 marked this conversation as resolved.
|
||||||||
| installer_filename=Path(info["_outpath"]).name, | ||||||||
| name=info["name"], | ||||||||
| version=info["version"], | ||||||||
| labels=info.get("docker_labels", {}), | ||||||||
| init_cmd="$PREFIX/bin/mamba shell" if has_mamba else "$PREFIX/bin/python -m conda", | ||||||||
|
Contributor
There was a problem hiding this comment. This is still assuming that either
Contributor
Author
There was a problem hiding this comment. Good catch. Moving this to the template and adding a v1 fallback |
||||||||
| register_envs=info.get("register_envs"), | ||||||||
| keep_pkgs=info.get("keep_pkgs"), | ||||||||
| ) | ||||||||
|
|
||||||||
| logger.info("Writing Dockerfile...") | ||||||||
| dockerfile_path = docker_dir / "Dockerfile" | ||||||||
| dockerfile_path.write_text(rendered_dockerfile) | ||||||||
| return dockerfile_path | ||||||||
|
|
||||||||
|
|
||||||||
| def build_image(info: dict, docker_dir: Path) -> Path: | ||||||||
| """Optionally build the docker image from the generated Dockerfile. | ||||||||
| Currently supported on linux and macOS platforms. | ||||||||
|
|
||||||||
| Parameters | ||||||||
| ---------- | ||||||||
| info: dict | ||||||||
| Constructor installer info dict. | ||||||||
| docker_dir: Path | ||||||||
| Path to the Docker directory containing the Docker outputs. | ||||||||
|
|
||||||||
| Returns | ||||||||
| ------- | ||||||||
| Path | ||||||||
| Path to the saved Docker image tarball. | ||||||||
|
|
||||||||
| """ | ||||||||
| if not (docker_platform := DOCKER_PLATFORM_MAP.get(info["_platform"])): | ||||||||
| raise RuntimeError( | ||||||||
| f"Unsupported platform for Docker build: {info['_platform']}. " | ||||||||
| f"Supported platforms are: {', '.join(DOCKER_PLATFORM_MAP.keys())}." | ||||||||
| ) | ||||||||
|
|
||||||||
| tag = info.get("docker_tag", f"{info['name'].lower()}:{info['version']}") | ||||||||
| tarball_dest = docker_dir / f"{Path(info['_outpath']).stem}-docker.tar" | ||||||||
|
|
||||||||
| cmd = [ | ||||||||
| "docker", | ||||||||
| "buildx", | ||||||||
| "build", | ||||||||
| str(docker_dir), | ||||||||
| "--platform", | ||||||||
| docker_platform, | ||||||||
| "-t", | ||||||||
| tag, | ||||||||
|
Jrice1317 marked this conversation as resolved.
|
||||||||
| "--load", | ||||||||
| ] | ||||||||
|
|
||||||||
| logger.info("Building Docker image: '%s'", tag) | ||||||||
| try: | ||||||||
| subprocess.run(cmd, check=True) | ||||||||
| except subprocess.CalledProcessError as e: | ||||||||
| # Gather diagnostics on failure | ||||||||
| docker_version = subprocess.run(["docker", "--version"], capture_output=True, text=True) | ||||||||
| buildx_version = subprocess.run( | ||||||||
| ["docker", "buildx", "version"], capture_output=True, text=True | ||||||||
| ) | ||||||||
| buildx_ls = subprocess.run(["docker", "buildx", "ls"], capture_output=True, text=True) | ||||||||
| raise RuntimeError( | ||||||||
|
Contributor
There was a problem hiding this comment. Why do we have such detailed error reporting for
Contributor
There was a problem hiding this comment. I can provide some context here, I suggested to add that to debug I'm OK removing/keeping it - hopefully we dont need to debug issues like that again in the future.
Contributor
Author
There was a problem hiding this comment.
|
||||||||
| f"Docker build failed.\n" | ||||||||
| f"Command: {cmd}\n" | ||||||||
| f"Docker version: {docker_version.stdout.strip()}\n" | ||||||||
| f"Buildx version: {buildx_version.stdout.strip() or buildx_version.stderr.strip()}\n" | ||||||||
| f"Buildx builders: {buildx_ls.stdout.strip()}" | ||||||||
| ) from e | ||||||||
|
|
||||||||
| logger.info("Saving Docker image to tarball: '%s'", tarball_dest) | ||||||||
| subprocess.run(["docker", "save", tag, "-o", str(tarball_dest)], check=True) | ||||||||
| subprocess.run(["docker", "rmi", tag], check=False) | ||||||||
| return tarball_dest | ||||||||
|
|
||||||||
|
|
||||||||
| def create(info: dict, verbose: bool = False) -> None: | ||||||||
| """Build a Docker output | ||||||||
|
Contributor
There was a problem hiding this comment. Can we add something to the doc string describing where the SH installer is coming from? |
||||||||
|
|
||||||||
| Parameters | ||||||||
| ---------- | ||||||||
| info: dict | ||||||||
| Constructor installer info dict. | ||||||||
| verbose: bool, optional | ||||||||
| If ``True``, enables verbose logging. | ||||||||
| Defaults to ``False``. | ||||||||
|
|
||||||||
| """ | ||||||||
| with tempfile.TemporaryDirectory() as temp_dir: | ||||||||
| docker_tmp_dir = Path(temp_dir) | ||||||||
|
|
||||||||
| installer_path = Path(info["_outpath"]) | ||||||||
| if not installer_path.exists(): | ||||||||
| raise RuntimeError(f"Expected .sh installer not found: {installer_path}") | ||||||||
| shutil.copy(installer_path, docker_tmp_dir / installer_path.name) | ||||||||
| logger.info("Copied installer to build directory.") | ||||||||
|
|
||||||||
| generate_dockerfile(info, docker_tmp_dir) | ||||||||
|
|
||||||||
| if info.get("docker_image") == "tar": | ||||||||
| tarball = build_image(info, docker_tmp_dir) | ||||||||
| shutil.copy(tarball, Path(info["_output_dir"]) / tarball.name) | ||||||||
| else: | ||||||||
| output_dir = Path(info["_output_dir"]) / installer_path.stem | ||||||||
| output_dir.mkdir(parents=True, exist_ok=True) | ||||||||
| shutil.copy(docker_tmp_dir / "Dockerfile", output_dir / "Dockerfile") | ||||||||
| shutil.copy( | ||||||||
| docker_tmp_dir / Path(info["_outpath"]).name, | ||||||||
| output_dir / Path(info["_outpath"]).name, | ||||||||
| ) | ||||||||
|
|
||||||||
| logger.info("Docker output complete. Docker directory: '%s'", info["_output_dir"]) | ||||||||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This should mention which variable to use to override it.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ok