ceph: Fix tests by adding back old required python libs. See #281858.

Fixes #241482.

Also fix test putting cluster in unhealthy `POOL_APP_NOT_ENABLED` state;
this seems to be the default state with Ceph 18.2.1 at least,
and it does not hurt to fix it now already in the way the Ceph docs say.

Also revert "nixosTests.ceph-single-node: remove dashboard check"

This reverts commit 41b27d7f4b.
This commit is contained in:
Niklas Hambüchen 2024-01-19 12:56:44 +00:00
parent bf2c85fa69
commit 506b21518d
6 changed files with 142 additions and 16 deletions

View File

@ -185,6 +185,14 @@ let
monA.succeed( monA.succeed(
"ceph osd pool create multi-node-test 32 32", "ceph osd pool create multi-node-test 32 32",
"ceph osd pool ls | grep 'multi-node-test'", "ceph osd pool ls | grep 'multi-node-test'",
# We need to enable an application on the pool, otherwise it will
# stay unhealthy in state POOL_APP_NOT_ENABLED.
# Creating a CephFS would do this automatically, but we haven't done that here.
# See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
# We use the custom application name "nixos-test" for this.
"ceph osd pool application enable multi-node-test nixos-test",
"ceph osd pool rename multi-node-test multi-node-other-test", "ceph osd pool rename multi-node-test multi-node-other-test",
"ceph osd pool ls | grep 'multi-node-other-test'", "ceph osd pool ls | grep 'multi-node-other-test'",
) )

View File

@ -145,6 +145,14 @@ let
monA.succeed( monA.succeed(
"ceph osd pool create single-node-test 32 32", "ceph osd pool create single-node-test 32 32",
"ceph osd pool ls | grep 'single-node-test'", "ceph osd pool ls | grep 'single-node-test'",
# We need to enable an application on the pool, otherwise it will
# stay unhealthy in state POOL_APP_NOT_ENABLED.
# Creating a CephFS would do this automatically, but we haven't done that here.
# See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
# We use the custom application name "nixos-test" for this.
"ceph osd pool application enable single-node-test nixos-test",
"ceph osd pool rename single-node-test single-node-other-test", "ceph osd pool rename single-node-test single-node-other-test",
"ceph osd pool ls | grep 'single-node-other-test'", "ceph osd pool ls | grep 'single-node-other-test'",
) )

View File

@ -145,6 +145,14 @@ let
monA.succeed( monA.succeed(
"ceph osd pool create single-node-test 32 32", "ceph osd pool create single-node-test 32 32",
"ceph osd pool ls | grep 'single-node-test'", "ceph osd pool ls | grep 'single-node-test'",
# We need to enable an application on the pool, otherwise it will
# stay unhealthy in state POOL_APP_NOT_ENABLED.
# Creating a CephFS would do this automatically, but we haven't done that here.
# See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
# We use the custom application name "nixos-test" for this.
"ceph osd pool application enable single-node-test nixos-test",
"ceph osd pool rename single-node-test single-node-other-test", "ceph osd pool rename single-node-test single-node-other-test",
"ceph osd pool ls | grep 'single-node-other-test'", "ceph osd pool ls | grep 'single-node-other-test'",
) )
@ -182,19 +190,16 @@ let
monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'") monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'") monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
# This test has been commented out due to the upstream issue with pyo3
# that has broken this dashboard
# Reference: https://www.spinics.net/lists/ceph-users/msg77812.html
# Enable the dashboard and recheck health # Enable the dashboard and recheck health
# monA.succeed( monA.succeed(
# "ceph mgr module enable dashboard", "ceph mgr module enable dashboard",
# "ceph config set mgr mgr/dashboard/ssl false", "ceph config set mgr mgr/dashboard/ssl false",
# # default is 8080 but it's better to be explicit # default is 8080 but it's better to be explicit
# "ceph config set mgr mgr/dashboard/server_port 8080", "ceph config set mgr mgr/dashboard/server_port 8080",
# ) )
# monA.wait_for_open_port(8080) monA.wait_for_open_port(8080)
# monA.wait_until_succeeds("curl -q --fail http://localhost:8080") monA.wait_until_succeeds("curl -q --fail http://localhost:8080")
# monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'") monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
''; '';
in { in {
name = "basic-single-node-ceph-cluster"; name = "basic-single-node-ceph-cluster";

View File

@ -4,6 +4,7 @@
, callPackage , callPackage
, cargo , cargo
, cffi , cffi
, cryptography-vectors ? (callPackage ./vectors.nix { })
, fetchPypi , fetchPypi
, hypothesis , hypothesis
, iso8601 , iso8601
@ -24,9 +25,6 @@
, setuptoolsRustBuildHook , setuptoolsRustBuildHook
}: }:
let
cryptography-vectors = callPackage ./vectors.nix { };
in
buildPythonPackage rec { buildPythonPackage rec {
pname = "cryptography"; pname = "cryptography";
version = "41.0.7"; # Also update the hash in vectors.nix version = "41.0.7"; # Also update the hash in vectors.nix

View File

@ -50,6 +50,9 @@
, zlib , zlib
, zstd , zstd
# Dependencies of overridden Python dependencies, hopefully we can remove these soon.
, rustPlatform
# Optional Dependencies # Optional Dependencies
, curl ? null , curl ? null
, expat ? null , expat ? null
@ -166,7 +169,67 @@ let
# Watch out for python <> boost compatibility # Watch out for python <> boost compatibility
python = python310.override { python = python310.override {
packageOverrides = self: super: { packageOverrides = self: super: let cryptographyOverrideVersion = "40.0.1"; in {
# Ceph does not support `cryptography` > 40 yet:
# * https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602
# * Upstream issue: https://tracker.ceph.com/issues/63529
# > Python Sub-Interpreter Model Used by ceph-mgr Incompatible With Python Modules Based on PyO3
#
# We pin the older `cryptography` 40 here;
# this also forces us to pin an older `pyopenssl` because the current one
# is not compatible with older `cryptography`, see:
# https://github.com/pyca/pyopenssl/blob/d9752e44127ba36041b045417af8a0bf16ec4f1e/CHANGELOG.rst#2320-2023-05-30
cryptography = super.cryptography.overridePythonAttrs (old: rec {
version = cryptographyOverrideVersion;
src = fetchPypi {
inherit (old) pname;
version = cryptographyOverrideVersion;
hash = "sha256-KAPy+LHpX2FEGZJsfm9V2CivxhTKXtYVQ4d65mjMNHI=";
};
cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
sourceRoot = let cargoRoot = "src/rust"; in "${old.pname}-${cryptographyOverrideVersion}/${cargoRoot}";
name = "${old.pname}-${cryptographyOverrideVersion}";
hash = "sha256-gFfDTc2QWBWHBCycVH1dYlCsWQMVcRZfOBIau+njtDU=";
};
patches = (old.patches or []) ++ [
# Fix https://nvd.nist.gov/vuln/detail/CVE-2023-49083 which has no upstream backport.
# See https://github.com/pyca/cryptography/commit/f09c261ca10a31fe41b1262306db7f8f1da0e48a#diff-f5134bf8f3cf0a5cc8601df55e50697acc866c603a38caff98802bd8e17976c5R1893
./python-cryptography-Cherry-pick-fix-for-CVE-2023-49083-on-cryptography-40.patch
];
# Tests would require overriding `cryptography-vectors`, which is not currently
# possible/desired, see: https://github.com/NixOS/nixpkgs/pull/281858#pullrequestreview-1841421866
doCheck = false;
});
# This is the most recent version of `pyopenssl` that's still compatible with `cryptography` 40.
# See https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602
pyopenssl = super.pyopenssl.overridePythonAttrs (old: rec {
version = "23.1.1";
src = fetchPypi {
pname = "pyOpenSSL";
inherit version;
hash = "sha256-hBSYub7GFiOxtsR+u8AjZ8B9YODhlfGXkIF/EMyNsLc=";
};
});
# Ceph does not support `kubernetes` >= 19, see:
# https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1900324090
kubernetes = super.kubernetes.overridePythonAttrs (old: rec {
version = "18.20.0";
src = fetchFromGitHub {
owner = "kubernetes-client";
repo = "python";
rev = "v${version}";
sha256 = "1sawp62j7h0yksmg9jlv4ik9b9i1a1w9syywc9mv8x89wibf5ql1";
fetchSubmodules = true;
};
});
}; };
}; };

View File

@ -0,0 +1,44 @@
commit 08dbad8552ecca85d3e38072f94eb864b143d218
Author: Niklas Hambüchen <mail@nh2.me>
Date: Mon Jan 22 12:52:06 2024 +0000
Cherry-pick fix for CVE-2023-49083
Cherry-Picked-From: f09c261ca10a31fe41b1262306db7f8f1da0e48a
Cherry-Picked-By: Niklas Hambüchen <mail@nh2.me>
diff --git a/src/cryptography/hazmat/backends/openssl/backend.py b/src/cryptography/hazmat/backends/openssl/backend.py
index a3fe1bce4..58e7207c4 100644
--- a/src/cryptography/hazmat/backends/openssl/backend.py
+++ b/src/cryptography/hazmat/backends/openssl/backend.py
@@ -2383,9 +2383,12 @@ class Backend:
_Reasons.UNSUPPORTED_SERIALIZATION,
)
+ certs: list[x509.Certificate] = []
+ if p7.d.sign == self._ffi.NULL:
+ return certs
+
sk_x509 = p7.d.sign.cert
num = self._lib.sk_X509_num(sk_x509)
- certs = []
for i in range(num):
x509 = self._lib.sk_X509_value(sk_x509, i)
self.openssl_assert(x509 != self._ffi.NULL)
diff --git a/tests/hazmat/primitives/test_pkcs7.py b/tests/hazmat/primitives/test_pkcs7.py
index 4e61c5ef5..d8170bfb9 100644
--- a/tests/hazmat/primitives/test_pkcs7.py
+++ b/tests/hazmat/primitives/test_pkcs7.py
@@ -89,6 +89,12 @@ class TestPKCS7Loading:
mode="rb",
)
+ def test_load_pkcs7_empty_certificates(self, backend):
+ der = b"\x30\x0B\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x07\x02"
+
+ certificates = pkcs7.load_der_pkcs7_certificates(der)
+ assert certificates == []
+
# We have no public verification API and won't be adding one until we get
# some requirements from users so this function exists to give us basic