From 506b21518dfaa747b022ff1850de86b8e5dc44d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Fri, 19 Jan 2024 12:56:44 +0000 Subject: [PATCH] ceph: Fix tests by adding back old required python libs. See #281858. Fixes #241482. Also fix test putting cluster in unhealthy `POOL_APP_NOT_ENABLED` state; this seems to be the default state with Ceph 18.2.1 at least, and it does not hurt to fix it now already in the way the Ceph docs say. Also revert "nixosTests.ceph-single-node: remove dashboard check" This reverts commit 41b27d7f4b3d9609c5b80ca67925c5827322f00d. --- nixos/tests/ceph-multi-node.nix | 8 +++ nixos/tests/ceph-single-node-bluestore.nix | 8 +++ nixos/tests/ceph-single-node.nix | 29 +++++---- .../python-modules/cryptography/default.nix | 4 +- pkgs/tools/filesystems/ceph/default.nix | 65 ++++++++++++++++++- ...or-CVE-2023-49083-on-cryptography-40.patch | 44 +++++++++++++ 6 files changed, 142 insertions(+), 16 deletions(-) create mode 100644 pkgs/tools/filesystems/ceph/python-cryptography-Cherry-pick-fix-for-CVE-2023-49083-on-cryptography-40.patch diff --git a/nixos/tests/ceph-multi-node.nix b/nixos/tests/ceph-multi-node.nix index 556546beee76..b1352a4bc8f4 100644 --- a/nixos/tests/ceph-multi-node.nix +++ b/nixos/tests/ceph-multi-node.nix @@ -185,6 +185,14 @@ let monA.succeed( "ceph osd pool create multi-node-test 32 32", "ceph osd pool ls | grep 'multi-node-test'", + + # We need to enable an application on the pool, otherwise it will + # stay unhealthy in state POOL_APP_NOT_ENABLED. + # Creating a CephFS would do this automatically, but we haven't done that here. + # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application + # We use the custom application name "nixos-test" for this. + "ceph osd pool application enable multi-node-test nixos-test", + "ceph osd pool rename multi-node-test multi-node-other-test", "ceph osd pool ls | grep 'multi-node-other-test'", ) diff --git a/nixos/tests/ceph-single-node-bluestore.nix b/nixos/tests/ceph-single-node-bluestore.nix index acaae4cf300e..8bd1a78244a2 100644 --- a/nixos/tests/ceph-single-node-bluestore.nix +++ b/nixos/tests/ceph-single-node-bluestore.nix @@ -145,6 +145,14 @@ let monA.succeed( "ceph osd pool create single-node-test 32 32", "ceph osd pool ls | grep 'single-node-test'", + + # We need to enable an application on the pool, otherwise it will + # stay unhealthy in state POOL_APP_NOT_ENABLED. + # Creating a CephFS would do this automatically, but we haven't done that here. + # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application + # We use the custom application name "nixos-test" for this. + "ceph osd pool application enable single-node-test nixos-test", + "ceph osd pool rename single-node-test single-node-other-test", "ceph osd pool ls | grep 'single-node-other-test'", ) diff --git a/nixos/tests/ceph-single-node.nix b/nixos/tests/ceph-single-node.nix index a3a4072365af..c34ec511dc6d 100644 --- a/nixos/tests/ceph-single-node.nix +++ b/nixos/tests/ceph-single-node.nix @@ -145,6 +145,14 @@ let monA.succeed( "ceph osd pool create single-node-test 32 32", "ceph osd pool ls | grep 'single-node-test'", + + # We need to enable an application on the pool, otherwise it will + # stay unhealthy in state POOL_APP_NOT_ENABLED. + # Creating a CephFS would do this automatically, but we haven't done that here. + # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application + # We use the custom application name "nixos-test" for this. + "ceph osd pool application enable single-node-test nixos-test", + "ceph osd pool rename single-node-test single-node-other-test", "ceph osd pool ls | grep 'single-node-other-test'", ) @@ -182,19 +190,16 @@ let monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'") monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'") - # This test has been commented out due to the upstream issue with pyo3 - # that has broken this dashboard - # Reference: https://www.spinics.net/lists/ceph-users/msg77812.html # Enable the dashboard and recheck health - # monA.succeed( - # "ceph mgr module enable dashboard", - # "ceph config set mgr mgr/dashboard/ssl false", - # # default is 8080 but it's better to be explicit - # "ceph config set mgr mgr/dashboard/server_port 8080", - # ) - # monA.wait_for_open_port(8080) - # monA.wait_until_succeeds("curl -q --fail http://localhost:8080") - # monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'") + monA.succeed( + "ceph mgr module enable dashboard", + "ceph config set mgr mgr/dashboard/ssl false", + # default is 8080 but it's better to be explicit + "ceph config set mgr mgr/dashboard/server_port 8080", + ) + monA.wait_for_open_port(8080) + monA.wait_until_succeeds("curl -q --fail http://localhost:8080") + monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'") ''; in { name = "basic-single-node-ceph-cluster"; diff --git a/pkgs/development/python-modules/cryptography/default.nix b/pkgs/development/python-modules/cryptography/default.nix index 4acb8c6dcd19..770561126fb3 100644 --- a/pkgs/development/python-modules/cryptography/default.nix +++ b/pkgs/development/python-modules/cryptography/default.nix @@ -4,6 +4,7 @@ , callPackage , cargo , cffi +, cryptography-vectors ? (callPackage ./vectors.nix { }) , fetchPypi , hypothesis , iso8601 @@ -24,9 +25,6 @@ , setuptoolsRustBuildHook }: -let - cryptography-vectors = callPackage ./vectors.nix { }; -in buildPythonPackage rec { pname = "cryptography"; version = "41.0.7"; # Also update the hash in vectors.nix diff --git a/pkgs/tools/filesystems/ceph/default.nix b/pkgs/tools/filesystems/ceph/default.nix index f38cd4be880c..fa87a6abfe75 100644 --- a/pkgs/tools/filesystems/ceph/default.nix +++ b/pkgs/tools/filesystems/ceph/default.nix @@ -50,6 +50,9 @@ , zlib , zstd +# Dependencies of overridden Python dependencies, hopefully we can remove these soon. +, rustPlatform + # Optional Dependencies , curl ? null , expat ? null @@ -166,7 +169,67 @@ let # Watch out for python <> boost compatibility python = python310.override { - packageOverrides = self: super: { + packageOverrides = self: super: let cryptographyOverrideVersion = "40.0.1"; in { + # Ceph does not support `cryptography` > 40 yet: + # * https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602 + # * Upstream issue: https://tracker.ceph.com/issues/63529 + # > Python Sub-Interpreter Model Used by ceph-mgr Incompatible With Python Modules Based on PyO3 + # + # We pin the older `cryptography` 40 here; + # this also forces us to pin an older `pyopenssl` because the current one + # is not compatible with older `cryptography`, see: + # https://github.com/pyca/pyopenssl/blob/d9752e44127ba36041b045417af8a0bf16ec4f1e/CHANGELOG.rst#2320-2023-05-30 + cryptography = super.cryptography.overridePythonAttrs (old: rec { + version = cryptographyOverrideVersion; + + src = fetchPypi { + inherit (old) pname; + version = cryptographyOverrideVersion; + hash = "sha256-KAPy+LHpX2FEGZJsfm9V2CivxhTKXtYVQ4d65mjMNHI="; + }; + + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src; + sourceRoot = let cargoRoot = "src/rust"; in "${old.pname}-${cryptographyOverrideVersion}/${cargoRoot}"; + name = "${old.pname}-${cryptographyOverrideVersion}"; + hash = "sha256-gFfDTc2QWBWHBCycVH1dYlCsWQMVcRZfOBIau+njtDU="; + }; + + patches = (old.patches or []) ++ [ + # Fix https://nvd.nist.gov/vuln/detail/CVE-2023-49083 which has no upstream backport. + # See https://github.com/pyca/cryptography/commit/f09c261ca10a31fe41b1262306db7f8f1da0e48a#diff-f5134bf8f3cf0a5cc8601df55e50697acc866c603a38caff98802bd8e17976c5R1893 + ./python-cryptography-Cherry-pick-fix-for-CVE-2023-49083-on-cryptography-40.patch + ]; + + # Tests would require overriding `cryptography-vectors`, which is not currently + # possible/desired, see: https://github.com/NixOS/nixpkgs/pull/281858#pullrequestreview-1841421866 + doCheck = false; + }); + + # This is the most recent version of `pyopenssl` that's still compatible with `cryptography` 40. + # See https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602 + pyopenssl = super.pyopenssl.overridePythonAttrs (old: rec { + version = "23.1.1"; + src = fetchPypi { + pname = "pyOpenSSL"; + inherit version; + hash = "sha256-hBSYub7GFiOxtsR+u8AjZ8B9YODhlfGXkIF/EMyNsLc="; + }; + }); + + # Ceph does not support `kubernetes` >= 19, see: + # https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1900324090 + kubernetes = super.kubernetes.overridePythonAttrs (old: rec { + version = "18.20.0"; + src = fetchFromGitHub { + owner = "kubernetes-client"; + repo = "python"; + rev = "v${version}"; + sha256 = "1sawp62j7h0yksmg9jlv4ik9b9i1a1w9syywc9mv8x89wibf5ql1"; + fetchSubmodules = true; + }; + }); + }; }; diff --git a/pkgs/tools/filesystems/ceph/python-cryptography-Cherry-pick-fix-for-CVE-2023-49083-on-cryptography-40.patch b/pkgs/tools/filesystems/ceph/python-cryptography-Cherry-pick-fix-for-CVE-2023-49083-on-cryptography-40.patch new file mode 100644 index 000000000000..7ca05cf7450f --- /dev/null +++ b/pkgs/tools/filesystems/ceph/python-cryptography-Cherry-pick-fix-for-CVE-2023-49083-on-cryptography-40.patch @@ -0,0 +1,44 @@ +commit 08dbad8552ecca85d3e38072f94eb864b143d218 +Author: Niklas Hambüchen +Date: Mon Jan 22 12:52:06 2024 +0000 + + Cherry-pick fix for CVE-2023-49083 + + Cherry-Picked-From: f09c261ca10a31fe41b1262306db7f8f1da0e48a + Cherry-Picked-By: Niklas Hambüchen + +diff --git a/src/cryptography/hazmat/backends/openssl/backend.py b/src/cryptography/hazmat/backends/openssl/backend.py +index a3fe1bce4..58e7207c4 100644 +--- a/src/cryptography/hazmat/backends/openssl/backend.py ++++ b/src/cryptography/hazmat/backends/openssl/backend.py +@@ -2383,9 +2383,12 @@ class Backend: + _Reasons.UNSUPPORTED_SERIALIZATION, + ) + ++ certs: list[x509.Certificate] = [] ++ if p7.d.sign == self._ffi.NULL: ++ return certs ++ + sk_x509 = p7.d.sign.cert + num = self._lib.sk_X509_num(sk_x509) +- certs = [] + for i in range(num): + x509 = self._lib.sk_X509_value(sk_x509, i) + self.openssl_assert(x509 != self._ffi.NULL) +diff --git a/tests/hazmat/primitives/test_pkcs7.py b/tests/hazmat/primitives/test_pkcs7.py +index 4e61c5ef5..d8170bfb9 100644 +--- a/tests/hazmat/primitives/test_pkcs7.py ++++ b/tests/hazmat/primitives/test_pkcs7.py +@@ -89,6 +89,12 @@ class TestPKCS7Loading: + mode="rb", + ) + ++ def test_load_pkcs7_empty_certificates(self, backend): ++ der = b"\x30\x0B\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x07\x02" ++ ++ certificates = pkcs7.load_der_pkcs7_certificates(der) ++ assert certificates == [] ++ + + # We have no public verification API and won't be adding one until we get + # some requirements from users so this function exists to give us basic