From 66be5d4444e7da1a8893ce6dca3c27ab8a3d0196 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Tue, 17 Nov 2026 11:28:18 -0800 Subject: [PATCH 1/3] fix(docker): keep the pip cache out of the image layers, and check the layers --- .github/workflows/test_docker.yml | 4 + utils/docker/Dockerfile.jammy | 3 + utils/docker/Dockerfile.noble | 3 + utils/docker/Dockerfile.resolute | 3 + utils/docker/build.sh | 2 + utils/docker/check_layers.js | 226 ++++++++++++++++++++++++++++++ utils/docker/test_check_layers.sh | 112 +++++++++++++++ 7 files changed, 353 insertions(+) create mode 100755 utils/docker/check_layers.js create mode 100755 utils/docker/test_check_layers.sh diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 001275484..2e92c325b 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -5,6 +5,7 @@ on: - '.github/workflows/test_docker.yml' - 'setup.py' - '**/Dockerfile.*' + - 'utils/docker/**' branches: - main - release-* @@ -13,6 +14,7 @@ on: - '.github/workflows/test_docker.yml' - 'setup.py' - '**/Dockerfile.*' + - 'utils/docker/**' branches: - main - release-* @@ -67,6 +69,8 @@ jobs: # build.sh builds the wheels, and with them the driver. - name: Build Docker image run: bash utils/docker/build.sh --${{ matrix.arch }} ${{ matrix.docker-image-variant }} playwright-python:localbuild-${{ matrix.docker-image-variant }} + - name: Test check_layers.js + run: bash utils/docker/test_check_layers.sh - name: Start the container id: container run: | diff --git a/utils/docker/Dockerfile.jammy b/utils/docker/Dockerfile.jammy index a2428c1d9..03291208c 100644 --- a/utils/docker/Dockerfile.jammy +++ b/utils/docker/Dockerfile.jammy @@ -5,6 +5,9 @@ ARG DEBIAN_FRONTEND=noninteractive ARG TZ=America/Los_Angeles ARG DOCKER_IMAGE_NAME_TEMPLATE="mcr.microsoft.com/playwright/python:v%version%-jammy" ARG UBUNTU_MIRROR_PREFIX +# pip and virtualenv write their caches here during the build. /dev/shm is a tmpfs, +# so no image layer keeps them. The image itself uses the default ~/.cache. +ARG XDG_CACHE_HOME=/dev/shm/cache ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 diff --git a/utils/docker/Dockerfile.noble b/utils/docker/Dockerfile.noble index fcd686bf2..427c4463f 100644 --- a/utils/docker/Dockerfile.noble +++ b/utils/docker/Dockerfile.noble @@ -5,6 +5,9 @@ ARG DEBIAN_FRONTEND=noninteractive ARG TZ=America/Los_Angeles ARG DOCKER_IMAGE_NAME_TEMPLATE="mcr.microsoft.com/playwright/python:v%version%-noble" ARG UBUNTU_MIRROR_PREFIX +# pip and virtualenv write their caches here during the build. /dev/shm is a tmpfs, +# so no image layer keeps them. The image itself uses the default ~/.cache. +ARG XDG_CACHE_HOME=/dev/shm/cache ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 diff --git a/utils/docker/Dockerfile.resolute b/utils/docker/Dockerfile.resolute index c8911634e..76a12e704 100644 --- a/utils/docker/Dockerfile.resolute +++ b/utils/docker/Dockerfile.resolute @@ -5,6 +5,9 @@ ARG DEBIAN_FRONTEND=noninteractive ARG TZ=America/Los_Angeles ARG DOCKER_IMAGE_NAME_TEMPLATE="mcr.microsoft.com/playwright/python:v%version%-resolute" ARG UBUNTU_MIRROR_PREFIX +# pip and virtualenv write their caches here during the build. /dev/shm is a tmpfs, +# so no image layer keeps them. The image itself uses the default ~/.cache. +ARG XDG_CACHE_HOME=/dev/shm/cache ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 diff --git a/utils/docker/build.sh b/utils/docker/build.sh index e9f6fb7c7..a7ebf85a9 100755 --- a/utils/docker/build.sh +++ b/utils/docker/build.sh @@ -69,6 +69,8 @@ for ((attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)); do --build-arg UBUNTU_MIRROR_PREFIX="${UBUNTU_MIRROR_PREFIX}" \ "${SECRET_ARGS[@]}" \ -t "$3" -f "Dockerfile.$2" .; then + # A failure of this check is not transient, so it does not retry the build. + node ./check_layers.js "$3" exit 0 fi if (( attempt < MAX_ATTEMPTS )); then diff --git a/utils/docker/check_layers.js b/utils/docker/check_layers.js new file mode 100755 index 000000000..7d9ab847b --- /dev/null +++ b/utils/docker/check_layers.js @@ -0,0 +1,226 @@ +#!/usr/bin/env node +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * The same file is in microsoft/playwright (utils/docker/check_layers.js) and in + * the language ports. Keep the copies the same. + */ + +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { execFileSync, spawnSync } = require('child_process'); + +const SCRIPT_NAME = path.basename(__filename); + +// A file that a later layer deletes is still in the layer that created it, and +// everyone who pulls the image can read it there. So the check reads every layer, +// not the file system of the final image. +// +// Each pattern is matched against the path of a file in a layer, for example +// "root/.npm/_logs/2026-01-01-debug-0.log". +const FORBIDDEN = [ + // npm and yarn: the cache index and the debug logs record download URLs, + // including signed URLs of a private registry. + /(^|\/)\.npm\//, + /(^|\/)_cacache\//, + /(^|\/)_logs\/.*-debug-\d+\.log$/, + /(^|\/)\.yarn\//, + /(^|\/)\.cache\/yarn\//, + // pip: the HTTP cache of the package index. + /(^|\/)\.cache\/pip\//, + // Maven and Gradle: settings.xml and gradle.properties can have credentials of + // a repository, and the local repository records where each artifact came from. + /(^|\/)\.m2\//, + /(^|\/)\.gradle\//, + // .NET and NuGet: the config can have credentials of a feed. + /(^|\/)\.dotnet\//, + /(^|\/)\.nuget\//, + /(^|\/)\.local\/share\/NuGet\//, + // Registry and package manager credentials. + /(^|\/)\.npmrc$/, + /(^|\/)\.yarnrc(\.yml)?$/, + /(^|\/)\.pypirc$/, + /(^|\/)pip\.conf$/, + /(^|\/)\.netrc$/, + // Source control. + /(^|\/)\.git\//, + /(^|\/)\.git-credentials$/, + // Cloud, container and ssh credentials. + /(^|\/)\.ssh\//, + /(^|\/)\.aws\//, + /(^|\/)\.azure\//, + /(^|\/)\.kube\//, + /(^|\/)\.docker\//, + /(^|\/)\.config\/gcloud\//, + // Private keys and key stores. Certificates (.pem, .crt) are public, and the + // image needs the CA certificates, so they are not in this list. + /\.key$/, + /\.(p12|pfx|jks|keystore)$/, + /(^|\/)id_(rsa|dsa|ecdsa|ed25519)$/, + // Local environment files. + /(^|\/)\.env(\.[^/]*)?$/, +]; + +// Files that match a pattern above, but are a part of a package and hold no secret. +const ALLOWED = [ + // The npm package has its own .npmrc with settings for npm development. + /(^|\/)node_modules\/npm\/\.npmrc$/, +]; + +function usage() { + return `usage: ${SCRIPT_NAME} [--list-files] [--base-image=] playwright:localbuild-noble + +Check that no layer of the docker image contains files that can hold +tokens or credentials, such as the npm cache, .npmrc, a .git directory or a private key. + + --list-files also print every file that the layers of our Dockerfile add. + Layers of the base image are not listed. + --base-image= the image in the FROM line of the Dockerfile. The check then + skips its layers: we cannot change them. Use it when the base + image has files that the check forbids, as the .NET SDK image + does. Without it, the check reads all layers. +`; +} + +const args = process.argv.slice(2); +if (!args.length || args.includes('--help') || args.includes('-h')) { + console.log(usage()); + process.exit(args.length ? 0 : 1); +} +const listFiles = args.includes('--list-files'); +const baseImageArg = args.find(arg => arg.startsWith('--base-image=')); +const baseImage = baseImageArg ? baseImageArg.substring('--base-image='.length) : undefined; +const images = args.filter(arg => arg !== '--list-files' && arg !== baseImageArg); +if (images.length !== 1 || images[0].startsWith('-')) { + console.error(usage()); + process.exit(1); +} +const image = images[0]; + +const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'check-layers-')); +process.on('exit', () => fs.rmSync(tmpDir, { recursive: true, force: true })); + +console.log(`Checking the layers of ${image} for files that can hold credentials...`); +// Unpack the image without a copy of the archive on disk. With pipefail, a +// failure of "docker save" fails the command too. +const saved = spawnSync('bash', ['-o', 'pipefail', '-c', 'docker save "$0" | tar -x -C "$1"', image, tmpDir], { stdio: 'inherit' }); +if (saved.status !== 0) { + console.error(`ERROR: cannot save and unpack ${image}`); + process.exit(1); +} + +let failed = false; +const layers = readLayers(tmpDir); +if (!layers.length) { + console.error(`ERROR: found no layers in ${image}`); + process.exit(1); +} +const baseLayerCount = baseImage ? countBaseLayers(baseImage, layers) : 0; +layers.forEach((layer, index) => { + if (index < baseLayerCount) + return; + const layerFile = path.join(tmpDir, layer.file); + const files = listLayerFiles(layerFile); + if (listFiles && (baseImage || layer.ours)) { + console.log(`\n=== layer ${index + 1}: ${layer.step}`); + console.log(`=== ${files.length} file(s). A ".wh." name marks a file that the layer deletes.`); + console.log(files.join('\n')); + } + const matches = withoutEmptyFiles(layerFile, files.filter(file => FORBIDDEN.some(re => re.test(file)) && !ALLOWED.some(re => re.test(file)))); + if (!matches.length) + return; + failed = true; + console.log(`\nERROR: layer ${index + 1} contains ${matches.length} forbidden file(s).`); + console.log(` step: ${layer.step}`); + for (const file of matches.slice(0, 20)) + console.log(` ${file}`); +}); + +if (failed) { + console.error('\nThese files stay in the image even when a later layer deletes them.'); + console.error('Do not create them, or delete them in the RUN step that creates them.'); + process.exit(1); +} +console.log(`\nChecked ${layers.length - baseLayerCount} layer(s), found no forbidden files.` + (baseLayerCount ? ` Skipped ${baseLayerCount} layer(s) of ${baseImage}.` : '')); + +/** + * Returns the layers of an unpacked "docker save" archive, bottom layer first. + * @returns {{ file: string, diffId: string, platform: string, ours: boolean, step: string }[]} + */ +function readLayers(dir) { + const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'manifest.json'), 'utf8'))[0]; + const config = JSON.parse(fs.readFileSync(path.join(dir, manifest.Config), 'utf8')); + // The history also has the steps that make no layer, such as ENV and ARG. + const steps = (config.history || []).filter(step => !step.empty_layer); + return manifest.Layers.map((file, i) => { + const step = steps[i] || {}; + // BuildKit puts the build arguments in front of a RUN command. Remove them. + const command = (step.created_by || '?').replace(/^RUN \|\d+ .*? \/bin\/sh -c /, 'RUN ').replace(/\s+/g, ' '); + return { + file, + diffId: config.rootfs.diff_ids[i], + platform: `${config.os}/${config.architecture}`, + // BuildKit marks the steps of the Dockerfile that it runs. The base image has no mark. + ours: (step.comment || '').includes('buildkit.dockerfile'), + step: command.length > 100 ? command.slice(0, 100) + '...' : command, + }; + }); +} + +/** + * Returns the paths of the files in a layer. tar reads both plain and compressed layers. + */ +function listLayerFiles(layerFile) { + const output = execFileSync('tar', ['-tf', layerFile], { encoding: 'utf8', maxBuffer: 1024 * 1024 * 1024 }); + return output.split('\n') + // Paths in a layer can start with "./". + .map(line => line.replace(/^\.\//, '')) + // Skip directory entries: an empty directory, such as the ~/.ssh that a + // package creates, holds no secret. + .filter(line => line && !line.endsWith('/')); +} + +/** + * Returns the number of layers that the image has from its base image. + */ +function countBaseLayers(baseImage, layers) { + // Pull the base image, to compare with the same version that the build used. + // An image that exists only on this machine cannot be pulled. Use it as it is. + spawnSync('docker', ['pull', '--quiet', '--platform', layers[0].platform, baseImage], { stdio: 'ignore' }); + const inspected = spawnSync('docker', ['image', 'inspect', '--format', '{{json .RootFS.Layers}}', baseImage], { encoding: 'utf8' }); + if (inspected.status !== 0) { + console.error(`ERROR: cannot find the base image ${baseImage}`); + process.exit(1); + } + const baseDiffIds = JSON.parse(inspected.stdout); + if (baseDiffIds.length >= layers.length || baseDiffIds.some((diffId, i) => diffId !== layers[i].diffId)) { + console.error(`ERROR: the layers of ${baseImage} are not the first layers of ${image}`); + process.exit(1); + } + return baseDiffIds.length; +} + +/** + * Removes the empty files from the list. An empty file holds no secret, and BuildKit + * can leave one at the target of a secret mount, for example /root/.npmrc. + */ +function withoutEmptyFiles(layerFile, files) { + // Each call of tar reads the layer again. A layer with many forbidden files fails + // the check with or without the empty ones. + if (files.length > 20) + return files; + return files.filter(file => execFileSync('tar', ['-xOf', layerFile, file], { maxBuffer: 1024 * 1024 * 1024 }).length > 0); +} diff --git a/utils/docker/test_check_layers.sh b/utils/docker/test_check_layers.sh new file mode 100755 index 000000000..e6c28378d --- /dev/null +++ b/utils/docker/test_check_layers.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -e +set +x + +if [[ ($1 == '--help') || ($1 == '-h') ]]; then + echo "usage: $(basename $0)" + echo + echo "Test check_layers.js with small images that this script builds." + echo + exit 0 +fi + +# The images that we publish are clean, so a build never sees check_layers.js fail. +# This test gives it an image with forbidden files, to know that it still finds them. + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +TMP_DIR="$(mktemp -d)" + +function cleanup() { + rm -rf "${TMP_DIR}" + docker rmi -f playwright:check-layers-bad playwright:check-layers-good \ + playwright:check-layers-child playwright:check-layers-base > /dev/null 2>&1 || true +} +trap cleanup EXIT + +cd "${TMP_DIR}" +echo "placeholder" > file +touch empty + +# 1. Forbidden files in a lower layer must fail the check. +FORBIDDEN_FILES=( + root/.npmrc + certs/server.key + home/pwuser/id_ed25519 + root/.cache/pip/http-v2/a/b/c + etc/pip.conf + root/.m2/settings.xml + root/.dotnet/TelemetryStorageService/a.trn + root/.nuget/NuGet/NuGet.Config +) +echo "FROM scratch" > Dockerfile.bad +for FORBIDDEN_FILE in "${FORBIDDEN_FILES[@]}"; do + echo "COPY file /${FORBIDDEN_FILE}" >> Dockerfile.bad +done +echo "COPY file /readme" >> Dockerfile.bad +docker build -q -t playwright:check-layers-bad -f Dockerfile.bad . > /dev/null +if node "${SCRIPT_DIR}/check_layers.js" playwright:check-layers-bad > output.txt 2>&1; then + cat output.txt + echo "FAILED: check_layers.js passed for an image with forbidden files in its layers" + exit 1 +fi +# The check must fail because it found the files, not because it crashed. +for FORBIDDEN_FILE in "${FORBIDDEN_FILES[@]}"; do + if ! grep -q -x -F " ${FORBIDDEN_FILE}" output.txt; then + cat output.txt + echo "FAILED: check_layers.js did not report ${FORBIDDEN_FILE}" + exit 1 + fi +done + +# 2. An image with only allowed files must pass the check. An empty file is +# allowed too: BuildKit can leave one at the target of a secret mount. +cat > Dockerfile.good <<'DOCKERFILE' +FROM scratch +COPY file /usr/local/lib/node_modules/npm/.npmrc +COPY file /etc/ssl/certs/ca.pem +COPY file /home/pwuser/.ssh-notes/id_ed25519.pub +COPY empty /root/.npmrc +COPY empty /etc/pip.conf +COPY file /readme +DOCKERFILE +docker build -q -t playwright:check-layers-good -f Dockerfile.good . > /dev/null +if ! node "${SCRIPT_DIR}/check_layers.js" playwright:check-layers-good > output.txt 2>&1; then + cat output.txt + echo "FAILED: check_layers.js failed for an image with no forbidden files" + exit 1 +fi + +# 3. With --base-image, forbidden files in the base image pass, and forbidden +# files in our layers still fail. +cat > Dockerfile.base <<'DOCKERFILE' +FROM scratch +COPY file /root/.dotnet/sentinel +DOCKERFILE +docker build -q -t playwright:check-layers-base -f Dockerfile.base . > /dev/null +cat > Dockerfile.child <<'DOCKERFILE' +FROM playwright:check-layers-base +COPY file /readme +DOCKERFILE +docker build -q -t playwright:check-layers-child -f Dockerfile.child . > /dev/null +if node "${SCRIPT_DIR}/check_layers.js" playwright:check-layers-child > output.txt 2>&1; then + cat output.txt + echo "FAILED: check_layers.js passed for a forbidden file in the base image, without --base-image" + exit 1 +fi +if ! node "${SCRIPT_DIR}/check_layers.js" --base-image=playwright:check-layers-base playwright:check-layers-child > output.txt 2>&1; then + cat output.txt + echo "FAILED: check_layers.js failed for a forbidden file in the base image, with --base-image" + exit 1 +fi +cat > Dockerfile.child <<'DOCKERFILE' +FROM playwright:check-layers-base +COPY file /root/.dotnet/other +DOCKERFILE +docker build -q -t playwright:check-layers-child -f Dockerfile.child . > /dev/null +if node "${SCRIPT_DIR}/check_layers.js" --base-image=playwright:check-layers-base playwright:check-layers-child > output.txt 2>&1; then + cat output.txt + echo "FAILED: check_layers.js passed for a forbidden file in our layer, with --base-image" + exit 1 +fi + +echo "PASSED: check_layers.js finds forbidden files and accepts clean images" From 196f7a626951c9fefa3068c3afe7a1384998232a Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Tue, 17 Nov 2026 11:40:02 -0800 Subject: [PATCH 2/3] devops(docker): scan the image for secrets with Trivy --- .github/workflows/test_docker.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 2e92c325b..a698630a4 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -69,6 +69,17 @@ jobs: # build.sh builds the wheels, and with them the driver. - name: Build Docker image run: bash utils/docker/build.sh --${{ matrix.arch }} ${{ matrix.docker-image-variant }} playwright-python:localbuild-${{ matrix.docker-image-variant }} + # check_layers.js looks at the paths of the files. This scan looks at their content, + # in all layers, and finds tokens and keys in files with any name. + - name: Scan the image for secrets + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: playwright-python:localbuild-${{ matrix.docker-image-variant }} + scanners: secret + version: v0.76.2 + exit-code: '1' + # The secret scan needs no vulnerability database. + cache: 'false' - name: Test check_layers.js run: bash utils/docker/test_check_layers.sh - name: Start the container From 63a3180a2dff75c483740424993b1a18288593b7 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Tue, 17 Nov 2026 12:10:01 -0800 Subject: [PATCH 3/3] devops(docker): check the image layers for caches only, leave secrets to Trivy --- .github/workflows/test_docker.yml | 4 +- utils/docker/check_layers.js | 130 +++++++----------------------- utils/docker/test_check_layers.sh | 62 +++----------- 3 files changed, 41 insertions(+), 155 deletions(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index a698630a4..8fa87a0b1 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -69,8 +69,8 @@ jobs: # build.sh builds the wheels, and with them the driver. - name: Build Docker image run: bash utils/docker/build.sh --${{ matrix.arch }} ${{ matrix.docker-image-variant }} playwright-python:localbuild-${{ matrix.docker-image-variant }} - # check_layers.js looks at the paths of the files. This scan looks at their content, - # in all layers, and finds tokens and keys in files with any name. + # check_layers.js looks at the paths of the files, for caches and other leftovers. + # This scan looks at the content of the files, in all layers, and finds secrets. - name: Scan the image for secrets uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: diff --git a/utils/docker/check_layers.js b/utils/docker/check_layers.js index 7d9ab847b..49fa44720 100755 --- a/utils/docker/check_layers.js +++ b/utils/docker/check_layers.js @@ -14,8 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. * - * The same file is in microsoft/playwright (utils/docker/check_layers.js) and in - * the language ports. Keep the copies the same. + * The same file is in microsoft/playwright, microsoft/playwright-python and + * microsoft/playwright-mcp, in utils/docker/. Keep the copies the same. */ const fs = require('fs'); @@ -25,73 +25,38 @@ const { execFileSync, spawnSync } = require('child_process'); const SCRIPT_NAME = path.basename(__filename); -// A file that a later layer deletes is still in the layer that created it, and -// everyone who pulls the image can read it there. So the check reads every layer, -// not the file system of the final image. +// A file that a later layer deletes is still in the layer that created it. It makes +// the image larger, and everyone who pulls the image can read it there. So the check +// reads every layer, not the file system of the final image. +// +// This check looks only at the paths of the files. It does not look for secrets: +// the Trivy scan in the Docker tests does that, by content. // // Each pattern is matched against the path of a file in a layer, for example // "root/.npm/_logs/2026-01-01-debug-0.log". const FORBIDDEN = [ - // npm and yarn: the cache index and the debug logs record download URLs, - // including signed URLs of a private registry. + // npm: the cache and the debug logs. /(^|\/)\.npm\//, /(^|\/)_cacache\//, /(^|\/)_logs\/.*-debug-\d+\.log$/, + // Node.js: the compile cache that npm turns on. It is in the temporary directory. + /(^|\/)node-compile-cache\//, + // yarn. /(^|\/)\.yarn\//, - /(^|\/)\.cache\/yarn\//, - // pip: the HTTP cache of the package index. - /(^|\/)\.cache\/pip\//, - // Maven and Gradle: settings.xml and gradle.properties can have credentials of - // a repository, and the local repository records where each artifact came from. - /(^|\/)\.m2\//, - /(^|\/)\.gradle\//, - // .NET and NuGet: the config can have credentials of a feed. - /(^|\/)\.dotnet\//, - /(^|\/)\.nuget\//, - /(^|\/)\.local\/share\/NuGet\//, - // Registry and package manager credentials. - /(^|\/)\.npmrc$/, - /(^|\/)\.yarnrc(\.yml)?$/, - /(^|\/)\.pypirc$/, - /(^|\/)pip\.conf$/, - /(^|\/)\.netrc$/, + // The cache directory of a user: pip, virtualenv, yarn and other tools write here. + /(^|\/)\.cache\//, // Source control. /(^|\/)\.git\//, - /(^|\/)\.git-credentials$/, - // Cloud, container and ssh credentials. - /(^|\/)\.ssh\//, - /(^|\/)\.aws\//, - /(^|\/)\.azure\//, - /(^|\/)\.kube\//, - /(^|\/)\.docker\//, - /(^|\/)\.config\/gcloud\//, - // Private keys and key stores. Certificates (.pem, .crt) are public, and the - // image needs the CA certificates, so they are not in this list. - /\.key$/, - /\.(p12|pfx|jks|keystore)$/, - /(^|\/)id_(rsa|dsa|ecdsa|ed25519)$/, - // Local environment files. - /(^|\/)\.env(\.[^/]*)?$/, -]; - -// Files that match a pattern above, but are a part of a package and hold no secret. -const ALLOWED = [ - // The npm package has its own .npmrc with settings for npm development. - /(^|\/)node_modules\/npm\/\.npmrc$/, ]; function usage() { - return `usage: ${SCRIPT_NAME} [--list-files] [--base-image=] playwright:localbuild-noble + return `usage: ${SCRIPT_NAME} [--list-files] playwright:localbuild-noble -Check that no layer of the docker image contains files that can hold -tokens or credentials, such as the npm cache, .npmrc, a .git directory or a private key. +Check that no layer of the docker image contains files that a build step +must not leave behind, such as a package manager cache or a .git directory. - --list-files also print every file that the layers of our Dockerfile add. - Layers of the base image are not listed. - --base-image= the image in the FROM line of the Dockerfile. The check then - skips its layers: we cannot change them. Use it when the base - image has files that the check forbids, as the .NET SDK image - does. Without it, the check reads all layers. + --list-files also print every file that the layers of our Dockerfile add. + Layers of the base image are not listed. `; } @@ -101,9 +66,7 @@ if (!args.length || args.includes('--help') || args.includes('-h')) { process.exit(args.length ? 0 : 1); } const listFiles = args.includes('--list-files'); -const baseImageArg = args.find(arg => arg.startsWith('--base-image=')); -const baseImage = baseImageArg ? baseImageArg.substring('--base-image='.length) : undefined; -const images = args.filter(arg => arg !== '--list-files' && arg !== baseImageArg); +const images = args.filter(arg => arg !== '--list-files'); if (images.length !== 1 || images[0].startsWith('-')) { console.error(usage()); process.exit(1); @@ -113,7 +76,7 @@ const image = images[0]; const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'check-layers-')); process.on('exit', () => fs.rmSync(tmpDir, { recursive: true, force: true })); -console.log(`Checking the layers of ${image} for files that can hold credentials...`); +console.log(`Checking the layers of ${image} for caches and other files that must not be in the image...`); // Unpack the image without a copy of the archive on disk. With pipefail, a // failure of "docker save" fails the command too. const saved = spawnSync('bash', ['-o', 'pipefail', '-c', 'docker save "$0" | tar -x -C "$1"', image, tmpDir], { stdio: 'inherit' }); @@ -128,18 +91,14 @@ if (!layers.length) { console.error(`ERROR: found no layers in ${image}`); process.exit(1); } -const baseLayerCount = baseImage ? countBaseLayers(baseImage, layers) : 0; layers.forEach((layer, index) => { - if (index < baseLayerCount) - return; - const layerFile = path.join(tmpDir, layer.file); - const files = listLayerFiles(layerFile); - if (listFiles && (baseImage || layer.ours)) { + const files = listLayerFiles(path.join(tmpDir, layer.file)); + if (listFiles && layer.ours) { console.log(`\n=== layer ${index + 1}: ${layer.step}`); console.log(`=== ${files.length} file(s). A ".wh." name marks a file that the layer deletes.`); console.log(files.join('\n')); } - const matches = withoutEmptyFiles(layerFile, files.filter(file => FORBIDDEN.some(re => re.test(file)) && !ALLOWED.some(re => re.test(file)))); + const matches = files.filter(file => FORBIDDEN.some(re => re.test(file))); if (!matches.length) return; failed = true; @@ -154,11 +113,11 @@ if (failed) { console.error('Do not create them, or delete them in the RUN step that creates them.'); process.exit(1); } -console.log(`\nChecked ${layers.length - baseLayerCount} layer(s), found no forbidden files.` + (baseLayerCount ? ` Skipped ${baseLayerCount} layer(s) of ${baseImage}.` : '')); +console.log(`\nChecked ${layers.length} layer(s), found no forbidden files.`); /** * Returns the layers of an unpacked "docker save" archive, bottom layer first. - * @returns {{ file: string, diffId: string, platform: string, ours: boolean, step: string }[]} + * @returns {{ file: string, ours: boolean, step: string }[]} */ function readLayers(dir) { const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'manifest.json'), 'utf8'))[0]; @@ -171,8 +130,6 @@ function readLayers(dir) { const command = (step.created_by || '?').replace(/^RUN \|\d+ .*? \/bin\/sh -c /, 'RUN ').replace(/\s+/g, ' '); return { file, - diffId: config.rootfs.diff_ids[i], - platform: `${config.os}/${config.architecture}`, // BuildKit marks the steps of the Dockerfile that it runs. The base image has no mark. ours: (step.comment || '').includes('buildkit.dockerfile'), step: command.length > 100 ? command.slice(0, 100) + '...' : command, @@ -188,39 +145,6 @@ function listLayerFiles(layerFile) { return output.split('\n') // Paths in a layer can start with "./". .map(line => line.replace(/^\.\//, '')) - // Skip directory entries: an empty directory, such as the ~/.ssh that a - // package creates, holds no secret. + // Skip directory entries: an empty directory is not a problem. .filter(line => line && !line.endsWith('/')); } - -/** - * Returns the number of layers that the image has from its base image. - */ -function countBaseLayers(baseImage, layers) { - // Pull the base image, to compare with the same version that the build used. - // An image that exists only on this machine cannot be pulled. Use it as it is. - spawnSync('docker', ['pull', '--quiet', '--platform', layers[0].platform, baseImage], { stdio: 'ignore' }); - const inspected = spawnSync('docker', ['image', 'inspect', '--format', '{{json .RootFS.Layers}}', baseImage], { encoding: 'utf8' }); - if (inspected.status !== 0) { - console.error(`ERROR: cannot find the base image ${baseImage}`); - process.exit(1); - } - const baseDiffIds = JSON.parse(inspected.stdout); - if (baseDiffIds.length >= layers.length || baseDiffIds.some((diffId, i) => diffId !== layers[i].diffId)) { - console.error(`ERROR: the layers of ${baseImage} are not the first layers of ${image}`); - process.exit(1); - } - return baseDiffIds.length; -} - -/** - * Removes the empty files from the list. An empty file holds no secret, and BuildKit - * can leave one at the target of a secret mount, for example /root/.npmrc. - */ -function withoutEmptyFiles(layerFile, files) { - // Each call of tar reads the layer again. A layer with many forbidden files fails - // the check with or without the empty ones. - if (files.length > 20) - return files; - return files.filter(file => execFileSync('tar', ['-xOf', layerFile, file], { maxBuffer: 1024 * 1024 * 1024 }).length > 0); -} diff --git a/utils/docker/test_check_layers.sh b/utils/docker/test_check_layers.sh index e6c28378d..0c81e9ed4 100755 --- a/utils/docker/test_check_layers.sh +++ b/utils/docker/test_check_layers.sh @@ -5,7 +5,7 @@ set +x if [[ ($1 == '--help') || ($1 == '-h') ]]; then echo "usage: $(basename $0)" echo - echo "Test check_layers.js with small images that this script builds." + echo "Test check_layers.js with two small images that this script builds." echo exit 0 fi @@ -18,25 +18,22 @@ TMP_DIR="$(mktemp -d)" function cleanup() { rm -rf "${TMP_DIR}" - docker rmi -f playwright:check-layers-bad playwright:check-layers-good \ - playwright:check-layers-child playwright:check-layers-base > /dev/null 2>&1 || true + docker rmi -f playwright:check-layers-bad playwright:check-layers-good > /dev/null 2>&1 || true } trap cleanup EXIT cd "${TMP_DIR}" echo "placeholder" > file -touch empty # 1. Forbidden files in a lower layer must fail the check. FORBIDDEN_FILES=( - root/.npmrc - certs/server.key - home/pwuser/id_ed25519 + root/.npm/_cacache/index-v5/aa/bb/cc + root/.npm/_logs/2026-01-01T00_00_00_000Z-debug-0.log + tmp/node-compile-cache/v26.0.0-x64/aabbcc + root/.yarn/berry/cache/a.zip root/.cache/pip/http-v2/a/b/c - etc/pip.conf - root/.m2/settings.xml - root/.dotnet/TelemetryStorageService/a.trn - root/.nuget/NuGet/NuGet.Config + home/pwuser/.cache/virtualenv/a + src/.git/config ) echo "FROM scratch" > Dockerfile.bad for FORBIDDEN_FILE in "${FORBIDDEN_FILES[@]}"; do @@ -58,15 +55,13 @@ for FORBIDDEN_FILE in "${FORBIDDEN_FILES[@]}"; do fi done -# 2. An image with only allowed files must pass the check. An empty file is -# allowed too: BuildKit can leave one at the target of a secret mount. +# 2. An image with no forbidden files must pass the check. Files that can hold +# a secret, such as .npmrc, pass too: the Trivy scan looks for secrets. cat > Dockerfile.good <<'DOCKERFILE' FROM scratch COPY file /usr/local/lib/node_modules/npm/.npmrc COPY file /etc/ssl/certs/ca.pem -COPY file /home/pwuser/.ssh-notes/id_ed25519.pub -COPY empty /root/.npmrc -COPY empty /etc/pip.conf +COPY file /var/cache/fontconfig/aabbcc.cache-9 COPY file /readme DOCKERFILE docker build -q -t playwright:check-layers-good -f Dockerfile.good . > /dev/null @@ -76,37 +71,4 @@ if ! node "${SCRIPT_DIR}/check_layers.js" playwright:check-layers-good > output. exit 1 fi -# 3. With --base-image, forbidden files in the base image pass, and forbidden -# files in our layers still fail. -cat > Dockerfile.base <<'DOCKERFILE' -FROM scratch -COPY file /root/.dotnet/sentinel -DOCKERFILE -docker build -q -t playwright:check-layers-base -f Dockerfile.base . > /dev/null -cat > Dockerfile.child <<'DOCKERFILE' -FROM playwright:check-layers-base -COPY file /readme -DOCKERFILE -docker build -q -t playwright:check-layers-child -f Dockerfile.child . > /dev/null -if node "${SCRIPT_DIR}/check_layers.js" playwright:check-layers-child > output.txt 2>&1; then - cat output.txt - echo "FAILED: check_layers.js passed for a forbidden file in the base image, without --base-image" - exit 1 -fi -if ! node "${SCRIPT_DIR}/check_layers.js" --base-image=playwright:check-layers-base playwright:check-layers-child > output.txt 2>&1; then - cat output.txt - echo "FAILED: check_layers.js failed for a forbidden file in the base image, with --base-image" - exit 1 -fi -cat > Dockerfile.child <<'DOCKERFILE' -FROM playwright:check-layers-base -COPY file /root/.dotnet/other -DOCKERFILE -docker build -q -t playwright:check-layers-child -f Dockerfile.child . > /dev/null -if node "${SCRIPT_DIR}/check_layers.js" --base-image=playwright:check-layers-base playwright:check-layers-child > output.txt 2>&1; then - cat output.txt - echo "FAILED: check_layers.js passed for a forbidden file in our layer, with --base-image" - exit 1 -fi - -echo "PASSED: check_layers.js finds forbidden files and accepts clean images" +echo "PASSED: check_layers.js finds forbidden files and accepts a clean image"