From 4b603ad9cd26f71bd17d52c2f6923ce6ba163c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joel=20H=C3=B6ner?= Date: Sat, 17 Feb 2024 17:48:57 +0100 Subject: [PATCH] dockerTools: configurable compression schema This commit adds support for swapping out the compression algorithm used in all major docker-tools commands that generate images. The default algorithm remains unchanged (gzip). --- .../images/dockertools.section.md | 7 ++ nixos/tests/docker-tools.nix | 21 +++++ pkgs/build-support/docker/default.nix | 79 +++++++++++++++---- pkgs/build-support/docker/examples.nix | 22 ++++++ 4 files changed, 115 insertions(+), 14 deletions(-) diff --git a/doc/build-helpers/images/dockertools.section.md b/doc/build-helpers/images/dockertools.section.md index 9317146b8f94..831a42f25423 100644 --- a/doc/build-helpers/images/dockertools.section.md +++ b/doc/build-helpers/images/dockertools.section.md @@ -178,6 +178,13 @@ Similarly, if you encounter errors similar to `Error_Protocol ("certificate has _Default value:_ 0. +`compressor` (String; _optional_) + +: Selects the algorithm used to compress the image. + + _Default value:_ `"gz"`.\ + _Possible values:_ `"none"`, `"gz"`, `"zstd"`. + `contents` **DEPRECATED** : This attribute is deprecated, and users are encouraged to use `copyToRoot` instead. diff --git a/nixos/tests/docker-tools.nix b/nixos/tests/docker-tools.nix index 90af817e75ed..0d28a39712c6 100644 --- a/nixos/tests/docker-tools.nix +++ b/nixos/tests/docker-tools.nix @@ -128,6 +128,15 @@ in { docker.succeed("docker images --format '{{.Tag}}' | grep -F '${examples.nixLayered.imageTag}'") docker.succeed("docker rmi ${examples.nixLayered.imageName}") + with subtest("Check that images with alternative compression schemas load"): + docker.succeed( + "docker load --input='${examples.bashZstdCompressed}'", + "docker rmi ${examples.bashZstdCompressed.imageName}", + ) + docker.succeed( + "docker load --input='${examples.bashUncompressed}'", + "docker rmi ${examples.bashUncompressed.imageName}", + ) with subtest( "Check if the nix store is correctly initialized by listing " @@ -449,6 +458,18 @@ in { "docker run --rm ${examples.layeredImageWithFakeRootCommands.imageName} /hello/bin/layeredImageWithFakeRootCommands-hello" ) + with subtest("mergeImage correctly deals with varying compression schemas in inputs"): + docker.succeed("docker load --input='${examples.mergeVaryingCompressor}'") + + for sub_image, tag in [ + ("${examples.redis.imageName}", "${examples.redis.imageTag}"), + ("${examples.bashUncompressed.imageName}", "${examples.bashUncompressed.imageTag}"), + ("${examples.bashZstdCompressed.imageName}", "${examples.bashZstdCompressed.imageTag}"), + ]: + docker.succeed(f"docker images --format '{{{{.Repository}}}}-{{{{.Tag}}}}' | grep -F '{sub_image}-{tag}'") + docker.succeed(f"docker rmi {sub_image}") + + with subtest("exportImage produces a valid tarball"): docker.succeed( "tar -tf ${examples.exportBash} | grep '\./bin/bash' > /dev/null" diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 05a1a6fbbdaf..611c373105e5 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -8,6 +8,7 @@ , proot , fakeNss , fakeroot +, file , go , jq , jshon @@ -34,6 +35,7 @@ , writeText , writeTextDir , writePython3 +, zstd }: let @@ -76,6 +78,30 @@ let # mapping from the go package. defaultArchitecture = go.GOARCH; + compressors = { + none = { + ext = ""; + nativeInputs = [ ]; + compress = "cat"; + decompress = "cat"; + }; + gz = { + ext = ".gz"; + nativeInputs = [ pigz ]; + compress = "pigz -p$NIX_BUILD_CORES -nTR"; + decompress = "pigz -d -p$NIX_BUILD_CORES"; + }; + zstd = { + ext = ".zst"; + nativeInputs = [ zstd ]; + compress = "zstd -T$NIX_BUILD_CORES"; + decompress = "zstd -d -T$NIX_BUILD_CORES"; + }; + }; + + compressorForImage = compressor: imageName: compressors.${compressor} or + (throw "in docker image ${imageName}: compressor must be one of: [${toString builtins.attrNames compressors}]"); + in rec { examples = callPackage ./examples.nix { @@ -487,16 +513,17 @@ rec { ''; }; - buildLayeredImage = lib.makeOverridable ({ name, ... }@args: + buildLayeredImage = lib.makeOverridable ({ name, compressor ? "gz", ... }@args: let stream = streamLayeredImage args; + compress = compressorForImage compressor name; in - runCommand "${baseNameOf name}.tar.gz" + runCommand "${baseNameOf name}.tar${compress.ext}" { inherit (stream) imageName; passthru = { inherit (stream) imageTag; }; - nativeBuildInputs = [ pigz ]; - } "${stream} | pigz -nTR > $out" + nativeBuildInputs = compress.nativeInputs; + } "${stream} | ${compress.compress} > $out" ); # 1. extract the base image @@ -539,6 +566,8 @@ rec { buildVMMemorySize ? 512 , # Time of creation of the image. created ? "1970-01-01T00:00:01Z" + , # Compressor to use. One of: none, gz, zstd. + compressor ? "gz" , # Deprecated. contents ? null , @@ -574,6 +603,8 @@ rec { in if created == "now" then impure else pure; + compress = compressorForImage compressor name; + layer = if runAsRoot == null then @@ -590,9 +621,9 @@ rec { extraCommands; copyToRoot = rootContents; }; - result = runCommand "docker-image-${baseName}.tar.gz" + result = runCommand "docker-image-${baseName}.tar${compress.ext}" { - nativeBuildInputs = [ jshon pigz jq moreutils ]; + nativeBuildInputs = [ jshon jq moreutils ] ++ compress.nativeInputs; # Image name must be lowercase imageName = lib.toLower name; imageTag = lib.optionalString (tag != null) tag; @@ -746,7 +777,7 @@ rec { chmod -R a-w image echo "Cooking the image..." - tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | pigz -nTR > $out + tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | ${compress.compress} > $out echo "Finished." ''; @@ -761,16 +792,28 @@ rec { mergeImages = images: runCommand "merge-docker-images" { inherit images; - nativeBuildInputs = [ pigz jq ]; + nativeBuildInputs = [ file jq ] + ++ compressors.none.nativeInputs + ++ compressors.gz.nativeInputs + ++ compressors.zstd.nativeInputs; } '' mkdir image inputs # Extract images repos=() manifests=() + last_image_mime="application/gzip" for item in $images; do name=$(basename $item) mkdir inputs/$name - tar -I pigz -xf $item -C inputs/$name + + last_image_mime=$(file --mime-type -b $item) + case $last_image_mime in + "application/x-tar") ${compressors.none.decompress};; + "application/zstd") ${compressors.zstd.decompress};; + "application/gzip") ${compressors.gz.decompress};; + *) echo "error: unexpected layer type $last_image_mime" >&2; exit 1;; + esac < $item | tar -xC inputs/$name + if [ -f inputs/$name/repositories ]; then repos+=(inputs/$name/repositories) fi @@ -787,7 +830,14 @@ rec { mv repositories image/repositories mv manifest.json image/manifest.json # Create tarball and gzip - tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | pigz -nTR > $out + tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | ( + case $last_image_mime in + "application/x-tar") ${compressors.none.compress};; + "application/zstd") ${compressors.zstd.compress};; + "application/gzip") ${compressors.gz.compress};; + # `*)` not needed; already checked. + esac + ) > $out ''; @@ -1238,14 +1288,15 @@ rec { }; # Wrapper around streamNixShellImage to build an image from the result - buildNixShellImage = { drv, ... }@args: + buildNixShellImage = { drv, compressor ? "gz", ... }@args: let stream = streamNixShellImage args; + compress = compressorForImage compressor drv.name; in - runCommand "${drv.name}-env.tar.gz" + runCommand "${drv.name}-env.tar${compress.ext}" { inherit (stream) imageName; passthru = { inherit (stream) imageTag; }; - nativeBuildInputs = [ pigz ]; - } "${stream} | pigz -nTR > $out"; + nativeBuildInputs = compress.nativeInputs; + } "${stream} | ${compress.compress} > $out"; } diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix index 5784e650dc2e..64dd112a950f 100644 --- a/pkgs/build-support/docker/examples.nix +++ b/pkgs/build-support/docker/examples.nix @@ -480,6 +480,22 @@ rec { layerC = layerOnTopOf layerB "c"; in layerC; + bashUncompressed = pkgs.dockerTools.buildImage { + name = "bash-uncompressed"; + tag = "latest"; + compressor = "none"; + # Not recommended. Use `buildEnv` between copy and packages to avoid file duplication. + copyToRoot = pkgs.bashInteractive; + }; + + bashZstdCompressed = pkgs.dockerTools.buildImage { + name = "bash-zstd"; + tag = "latest"; + compressor = "zstd"; + # Not recommended. Use `buildEnv` between copy and packages to avoid file duplication. + copyToRoot = pkgs.bashInteractive; + }; + # buildImage without explicit tag bashNoTag = pkgs.dockerTools.buildImage { name = "bash-no-tag"; @@ -614,6 +630,12 @@ rec { layeredImageWithFakeRootCommands ]; + mergeVaryingCompressor = pkgs.dockerTools.mergeImages [ + redis + bashUncompressed + bashZstdCompressed + ]; + helloOnRoot = pkgs.dockerTools.streamLayeredImage { name = "hello"; tag = "latest";