From adcd6517dba3f9c799ca36bd3fbc311bdeaf1ac1 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 28 Apr 2025 11:14:50 +1200 Subject: [PATCH] [docker] Use new base container image (#8582) --- .dockerignore | 3 +- .github/actions/build-image/action.yaml | 29 ++-- .github/actions/restore-python/action.yml | 4 +- .github/dependabot.yml | 1 - .github/workflows/ci-docker.yml | 7 +- .github/workflows/ci.yml | 4 +- .github/workflows/release.yml | 64 +++---- docker/Dockerfile | 203 ++++------------------ docker/build.py | 15 +- pyproject.toml | 1 - requirements.txt | 1 + requirements_optional.txt | 1 - script/setup | 23 +-- script/setup.bat | 4 +- 14 files changed, 108 insertions(+), 252 deletions(-) delete mode 100644 requirements_optional.txt diff --git a/.dockerignore b/.dockerignore index 7998ff877f..ccd466d8cb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -114,4 +114,5 @@ config/ examples/ Dockerfile .git/ -tests/build/ +tests/ +.* diff --git a/.github/actions/build-image/action.yaml b/.github/actions/build-image/action.yaml index c171a0a13c..3d6de54f42 100644 --- a/.github/actions/build-image/action.yaml +++ b/.github/actions/build-image/action.yaml @@ -1,15 +1,11 @@ name: Build Image inputs: - platform: - description: "Platform to build for" - required: true - example: "linux/amd64" target: description: "Target to build" required: true example: "docker" - baseimg: - description: "Base image type" + build_type: + description: "Build type" required: true example: "docker" suffix: @@ -19,6 +15,11 @@ inputs: description: "Version to build" required: true example: "2023.12.0" + base_os: + description: "Base OS to use" + required: false + default: "debian" + example: "debian" runs: using: "composite" steps: @@ -53,22 +54,22 @@ runs: with: context: . file: ./docker/Dockerfile - platforms: ${{ inputs.platform }} target: ${{ inputs.target }} cache-from: type=gha cache-to: ${{ steps.cache-to.outputs.value }} build-args: | - BASEIMGTYPE=${{ inputs.baseimg }} + BUILD_TYPE=${{ inputs.build_type }} BUILD_VERSION=${{ inputs.version }} + BUILD_OS=${{ inputs.base_os }} outputs: | type=image,name=ghcr.io/${{ steps.tags.outputs.image_name }},push-by-digest=true,name-canonical=true,push=true - name: Export ghcr digests shell: bash run: | - mkdir -p /tmp/digests/${{ inputs.target }}/ghcr + mkdir -p /tmp/digests/${{ inputs.build_type }}/ghcr digest="${{ steps.build-ghcr.outputs.digest }}" - touch "/tmp/digests/${{ inputs.target }}/ghcr/${digest#sha256:}" + touch "/tmp/digests/${{ inputs.build_type }}/ghcr/${digest#sha256:}" - name: Build and push to dockerhub by digest id: build-dockerhub @@ -79,19 +80,19 @@ runs: with: context: . file: ./docker/Dockerfile - platforms: ${{ inputs.platform }} target: ${{ inputs.target }} cache-from: type=gha cache-to: ${{ steps.cache-to.outputs.value }} build-args: | - BASEIMGTYPE=${{ inputs.baseimg }} + BUILD_TYPE=${{ inputs.build_type }} BUILD_VERSION=${{ inputs.version }} + BUILD_OS=${{ inputs.base_os }} outputs: | type=image,name=docker.io/${{ steps.tags.outputs.image_name }},push-by-digest=true,name-canonical=true,push=true - name: Export dockerhub digests shell: bash run: | - mkdir -p /tmp/digests/${{ inputs.target }}/dockerhub + mkdir -p /tmp/digests/${{ inputs.build_type }}/dockerhub digest="${{ steps.build-dockerhub.outputs.digest }}" - touch "/tmp/digests/${{ inputs.target }}/dockerhub/${digest#sha256:}" + touch "/tmp/digests/${{ inputs.build_type }}/dockerhub/${digest#sha256:}" diff --git a/.github/actions/restore-python/action.yml b/.github/actions/restore-python/action.yml index b9913605da..082539adaa 100644 --- a/.github/actions/restore-python/action.yml +++ b/.github/actions/restore-python/action.yml @@ -34,7 +34,7 @@ runs: python -m venv venv source venv/bin/activate python --version - pip install -r requirements.txt -r requirements_optional.txt -r requirements_test.txt + pip install -r requirements.txt -r requirements_test.txt pip install -e . - name: Create Python virtual environment if: steps.cache-venv.outputs.cache-hit != 'true' && runner.os == 'Windows' @@ -43,5 +43,5 @@ runs: python -m venv venv ./venv/Scripts/activate python --version - pip install -r requirements.txt -r requirements_optional.txt -r requirements_test.txt + pip install -r requirements.txt -r requirements_test.txt pip install -e . diff --git a/.github/dependabot.yml b/.github/dependabot.yml index bb35f16048..cf507bbaa6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,7 +17,6 @@ updates: docker-actions: applies-to: version-updates patterns: - - "docker/setup-qemu-action" - "docker/login-action" - "docker/setup-buildx-action" - package-ecosystem: github-actions diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 168333f3ff..511ec55f3e 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -37,8 +37,11 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "ubuntu-24.04-arm"] - build_type: ["ha-addon", "docker", "lint"] + os: ["ubuntu-24.04", "ubuntu-24.04-arm"] + build_type: + - "ha-addon" + - "docker" + # - "lint" steps: - uses: actions/checkout@v4.1.7 - name: Set up Python diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b01758323..77fe79fd1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: uses: actions/checkout@v4.1.7 - name: Generate cache-key id: cache-key - run: echo key="${{ hashFiles('requirements.txt', 'requirements_optional.txt', 'requirements_test.txt') }}" >> $GITHUB_OUTPUT + run: echo key="${{ hashFiles('requirements.txt', 'requirements_test.txt') }}" >> $GITHUB_OUTPUT - name: Set up Python ${{ env.DEFAULT_PYTHON }} id: python uses: actions/setup-python@v5.6.0 @@ -58,7 +58,7 @@ jobs: python -m venv venv . venv/bin/activate python --version - pip install -r requirements.txt -r requirements_optional.txt -r requirements_test.txt + pip install -r requirements.txt -r requirements_test.txt pip install -e . ruff: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 417212f40e..359a9bcc53 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,19 +68,22 @@ jobs: uses: pypa/gh-action-pypi-publish@v1.12.4 deploy-docker: - name: Build ESPHome ${{ matrix.platform }} + name: Build ESPHome ${{ matrix.platform.arch }} if: github.repository == 'esphome/esphome' permissions: contents: read packages: write - runs-on: ubuntu-latest + runs-on: ${{ matrix.platform.os }} needs: [init] strategy: fail-fast: false matrix: platform: - - linux/amd64 - - linux/arm64 + - arch: amd64 + os: "ubuntu-24.04" + - arch: arm64 + os: "ubuntu-24.04-arm" + steps: - uses: actions/checkout@v4.1.7 - name: Set up Python @@ -90,9 +93,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.10.0 - - name: Set up QEMU - if: matrix.platform != 'linux/amd64' - uses: docker/setup-qemu-action@v3.6.0 - name: Log in to docker hub uses: docker/login-action@v3.4.0 @@ -109,45 +109,36 @@ jobs: - name: Build docker uses: ./.github/actions/build-image with: - platform: ${{ matrix.platform }} - target: docker - baseimg: docker + target: final + build_type: docker suffix: "" version: ${{ needs.init.outputs.tag }} - name: Build ha-addon uses: ./.github/actions/build-image with: - platform: ${{ matrix.platform }} - target: hassio - baseimg: hassio + target: final + build_type: ha-addon suffix: "hassio" version: ${{ needs.init.outputs.tag }} - - name: Build lint - uses: ./.github/actions/build-image - with: - platform: ${{ matrix.platform }} - target: lint - baseimg: docker - suffix: lint - version: ${{ needs.init.outputs.tag }} - - - name: Sanitize platform name - id: sanitize - run: | - echo "${{ matrix.platform }}" | sed 's|/|-|g' > /tmp/platform - echo name=$(cat /tmp/platform) >> $GITHUB_OUTPUT + # - name: Build lint + # uses: ./.github/actions/build-image + # with: + # target: lint + # build_type: lint + # suffix: lint + # version: ${{ needs.init.outputs.tag }} - name: Upload digests uses: actions/upload-artifact@v4.6.2 with: - name: digests-${{ steps.sanitize.outputs.name }} + name: digests-${{ matrix.platform.arch }} path: /tmp/digests retention-days: 1 deploy-manifest: - name: Publish ESPHome ${{ matrix.image.title }} to ${{ matrix.registry }} + name: Publish ESPHome ${{ matrix.image.build_type }} to ${{ matrix.registry }} runs-on: ubuntu-latest needs: - init @@ -160,15 +151,12 @@ jobs: fail-fast: false matrix: image: - - title: "ha-addon" - target: "hassio" - suffix: "hassio" - - title: "docker" - target: "docker" + - build_type: "docker" suffix: "" - - title: "lint" - target: "lint" - suffix: "lint" + - build_type: "ha-addon" + suffix: "hassio" + # - build_type: "lint" + # suffix: "lint" registry: - ghcr - dockerhub @@ -212,7 +200,7 @@ jobs: done - name: Create manifest list and push - working-directory: /tmp/digests/${{ matrix.image.target }}/${{ matrix.registry }} + working-directory: /tmp/digests/${{ matrix.image.build_type }}/${{ matrix.registry }} run: | docker buildx imagetools create $(jq -Rcnr 'inputs | . / "," | map("-t " + .) | join(" ")' <<< "${{ steps.tags.outputs.tags}}") \ $(printf '${{ steps.tags.outputs.image }}@sha256:%s ' *) diff --git a/docker/Dockerfile b/docker/Dockerfile index 117ec17ae4..39dc1c7f28 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,131 +1,54 @@ -# Build these with the build.py script -# Example: -# python3 docker/build.py --tag dev --arch amd64 --build-type docker build +ARG BUILD_VERSION=dev +ARG BUILD_OS=alpine +ARG BUILD_BASE_VERSION=2025.04.0 +ARG BUILD_TYPE=docker -# One of "docker", "hassio" -ARG BASEIMGTYPE=docker +FROM ghcr.io/esphome/docker-base:${BUILD_OS}-${BUILD_BASE_VERSION} AS base-source-docker +FROM ghcr.io/esphome/docker-base:${BUILD_OS}-ha-addon-${BUILD_BASE_VERSION} AS base-source-ha-addon +ARG BUILD_TYPE +FROM base-source-${BUILD_TYPE} AS base -# https://github.com/hassio-addons/addon-debian-base/releases -FROM ghcr.io/hassio-addons/debian-base:7.2.0 AS base-hassio -# https://hub.docker.com/_/debian?tab=tags&page=1&name=bookworm -FROM debian:12.2-slim AS base-docker +RUN git config --system --add safe.directory "*" -FROM base-${BASEIMGTYPE} AS base +RUN pip install uv==0.6.14 - -ARG TARGETARCH -ARG TARGETVARIANT - - -# Note that --break-system-packages is used below because -# https://peps.python.org/pep-0668/ added a safety check that prevents -# installing packages with the same name as a system package. This is -# not a problem for us because we are not concerned about overwriting -# system packages because we are running in an isolated container. +COPY requirements.txt / RUN \ - apt-get update \ - # Use pinned versions so that we get updates with build caching - && apt-get install -y --no-install-recommends \ - python3-pip=23.0.1+dfsg-1 \ - python3-setuptools=66.1.1-1+deb12u1 \ - python3-venv=3.11.2-1+b1 \ - python3-wheel=0.38.4-2 \ - iputils-ping=3:20221126-1+deb12u1 \ - git=1:2.39.5-0+deb12u2 \ - curl=7.88.1-10+deb12u12 \ - openssh-client=1:9.2p1-2+deb12u5 \ - python3-cffi=1.15.1-5 \ - libcairo2=1.16.0-7 \ - libmagic1=1:5.44-3 \ - patch=2.7.6-7 \ - && rm -rf \ - /tmp/* \ - /var/{cache,log}/* \ - /var/lib/apt/lists/* - -ENV \ - # Fix click python3 lang warning https://click.palletsprojects.com/en/7.x/python3/ - LANG=C.UTF-8 LC_ALL=C.UTF-8 \ - # Store globally installed pio libs in /piolibs - PLATFORMIO_GLOBALLIB_DIR=/piolibs + uv pip install --no-cache-dir \ + -r /requirements.txt RUN \ - pip3 install \ - --break-system-packages --no-cache-dir \ - # Keep platformio version in sync with requirements.txt - platformio==6.1.18 \ - # Change some platformio settings - && platformio settings set enable_telemetry No \ + platformio settings set enable_telemetry No \ && platformio settings set check_platformio_interval 1000000 \ && mkdir -p /piolibs - -# First install requirements to leverage caching when requirements don't change -# tmpfs is for https://github.com/rust-lang/cargo/issues/8719 - -COPY requirements.txt requirements_optional.txt / -RUN --mount=type=tmpfs,target=/root/.cargo < /etc/apt/sources.list.d/llvm.sources.list \ - && apt-get update \ - # Use pinned versions so that we get updates with build caching - && apt-get install -y --no-install-recommends \ - clang-format-13=1:13.0.1-11+b2 \ - patch=2.7.6-7 \ - software-properties-common=0.99.30-4.1~deb12u1 \ - nano=7.2-1+deb12u1 \ - build-essential=12.9 \ - python3-dev=3.11.2-1+b1 \ - clang-tidy-18=1:18.1.8~++20240731024826+3b5b5c1ec4a3-1~exp1~20240731144843.145 \ - && rm -rf \ - /tmp/* \ - /var/{cache,log}/* \ - /var/lib/apt/lists/* - -COPY requirements_test.txt / -RUN pip3 install --break-system-packages --no-cache-dir -r /requirements_test.txt - -VOLUME ["/esphome"] -WORKDIR /esphome +# Copy esphome and install +COPY . /esphome +RUN uv pip install --no-cache-dir -e /esphome diff --git a/docker/build.py b/docker/build.py index cdc25df340..921adac7ab 100755 --- a/docker/build.py +++ b/docker/build.py @@ -54,7 +54,7 @@ manifest_parser = subparsers.add_parser( class DockerParams: build_to: str manifest_to: str - baseimgtype: str + build_type: str platform: str target: str @@ -66,24 +66,19 @@ class DockerParams: TYPE_LINT: "esphome/esphome-lint", }[build_type] build_to = f"{prefix}-{arch}" - baseimgtype = { - TYPE_DOCKER: "docker", - TYPE_HA_ADDON: "hassio", - TYPE_LINT: "docker", - }[build_type] platform = { ARCH_AMD64: "linux/amd64", ARCH_AARCH64: "linux/arm64", }[arch] target = { - TYPE_DOCKER: "docker", - TYPE_HA_ADDON: "hassio", + TYPE_DOCKER: "final", + TYPE_HA_ADDON: "final", TYPE_LINT: "lint", }[build_type] return cls( build_to=build_to, manifest_to=prefix, - baseimgtype=baseimgtype, + build_type=build_type, platform=platform, target=target, ) @@ -145,7 +140,7 @@ def main(): "buildx", "build", "--build-arg", - f"BASEIMGTYPE={params.baseimgtype}", + f"BUILD_TYPE={params.build_type}", "--build-arg", f"BUILD_VERSION={args.tag}", "--cache-from", diff --git a/pyproject.toml b/pyproject.toml index daf702d2e0..e3b10722c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,6 @@ version = {attr = "esphome.const.__version__"} [tool.setuptools.dynamic.optional-dependencies] dev = { file = ["requirements_dev.txt"] } test = { file = ["requirements_test.txt"] } -displays = { file = ["requirements_optional.txt"] } [tool.setuptools.packages.find] include = ["esphome*"] diff --git a/requirements.txt b/requirements.txt index cb1f1da2f2..f09d7894dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ puremagic==1.28 ruamel.yaml==0.18.10 # dashboard_import esphome-glyphsets==0.2.0 pillow==10.4.0 +cairosvg==2.7.1 freetype-py==2.5.1 # esp-idf requires this, but doesn't bundle it by default diff --git a/requirements_optional.txt b/requirements_optional.txt deleted file mode 100644 index 7416753d55..0000000000 --- a/requirements_optional.txt +++ /dev/null @@ -1 +0,0 @@ -cairosvg==2.7.1 diff --git a/script/setup b/script/setup index 3ebf75387f..acc2ec58b4 100755 --- a/script/setup +++ b/script/setup @@ -4,25 +4,28 @@ set -e cd "$(dirname "$0")/.." -location="venv/bin/activate" if [ ! -n "$DEVCONTAINER" ] && [ ! -n "$VIRTUAL_ENV" ] && [ ! "$ESPHOME_NO_VENV" ]; then - python3 -m venv venv - if [ -f venv/Scripts/activate ]; then - location="venv/Scripts/activate" + if [ -x "$(command -v uv)" ]; then + uv venv venv + else + python3 -m venv venv fi - source $location + source venv/bin/activate fi -pip3 install -r requirements.txt -r requirements_optional.txt -r requirements_test.txt -r requirements_dev.txt -pip3 install setuptools wheel -pip3 install -e ".[dev,test,displays]" --config-settings editable_mode=compat +if ! [ -x "$(command -v uv)" ]; then + python3 -m pip install uv +fi + +uv pip install setuptools wheel +uv pip install -e ".[dev,test]" --config-settings editable_mode=compat pre-commit install script/platformio_install_deps.py platformio.ini --libraries --tools --platforms -mkdir .temp +mkdir -p .temp echo echo -echo "Virtual environment created. Run 'source $location' to use it." +echo "Virtual environment created. Run 'source venv/bin/activate' to use it." diff --git a/script/setup.bat b/script/setup.bat index 0b49768139..ea2591bb71 100644 --- a/script/setup.bat +++ b/script/setup.bat @@ -15,9 +15,9 @@ echo Installing required packages... python.exe -m pip install --upgrade pip -pip3 install -r requirements.txt -r requirements_optional.txt -r requirements_test.txt -r requirements_dev.txt +pip3 install -r requirements.txt -r requirements_test.txt -r requirements_dev.txt pip3 install setuptools wheel -pip3 install -e ".[dev,test,displays]" --config-settings editable_mode=compat +pip3 install -e ".[dev,test]" --config-settings editable_mode=compat pre-commit install