From 8da0f9f79dd9f75ff215bade607d2a4b009009e5 Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Wed, 12 Apr 2023 20:06:39 +0200 Subject: [PATCH 1/9] gitlab-ci: assert that NM_TEST_SELECT_RUN is valid in "run-test.sh" The script now fails, if the user passes an invalid "$NM_TEST_SELECT_RUN" or if the script references an invalid name. --- .gitlab-ci/run-test.sh | 44 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci/run-test.sh b/.gitlab-ci/run-test.sh index d991010dc..e2f48e8b2 100755 --- a/.gitlab-ci/run-test.sh +++ b/.gitlab-ci/run-test.sh @@ -2,6 +2,11 @@ set -ex +die() { + printf "%s\n" "$*" >&2 + exit 1 +} + export PAGER=cat export OMP_NUM_THREADS=1 @@ -44,9 +49,47 @@ meson --version # to run that test as part of the build. Disable it. export NMTST_SKIP_CHECK_GITLAB_CI=1 +# Assert that "$1" is one of the valid values for NM_TEST_SELECT_RUN. die() otherwise. +check_run_assert() { + { set +x; } 2>/dev/null + local run="$1" + local a + + # These are the supported $NM_TEST_SELECT_RUN values. + local _CHECK_RUN_LIST=( + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + ) + + if [ "$run" = '' ] ; then + set -x + return 0 + fi + + for a in "${_CHECK_RUN_LIST[@]}" ; do + if [ "$a" = "$run" ] ; then + set -x + return 0 + fi + done + die "invalid NM_TEST_SELECT_RUN value \"$1\"" +} + +check_run_assert "$NM_TEST_SELECT_RUN" + check_run() { local test_no="$1" + check_run_assert "$test_no" + # Usually, we run the build several times. However, for testing # the build script manually, it can be useful to explicitly select # one step to run. For example, if step 3 is known to fail, you @@ -55,7 +98,6 @@ check_run() { test -z "$NM_TEST_SELECT_RUN" -o "$NM_TEST_SELECT_RUN" = "$test_no" } - check_run_clean() { if ! check_run "$1" ; then return 1 From d39fb6f0148af24d3325b9ea91e3c0a0ea873306 Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Wed, 12 Apr 2023 21:20:56 +0200 Subject: [PATCH 2/9] gitlab-ci: use clearer names for NM_TEST_SELECT_RUN in "run-test.sh" --- .gitlab-ci/run-test.sh | 67 +++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/.gitlab-ci/run-test.sh b/.gitlab-ci/run-test.sh index e2f48e8b2..50d090361 100755 --- a/.gitlab-ci/run-test.sh +++ b/.gitlab-ci/run-test.sh @@ -57,19 +57,21 @@ check_run_assert() { # These are the supported $NM_TEST_SELECT_RUN values. local _CHECK_RUN_LIST=( - 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 + autotools+gcc+docs+valgrind + meson+gcc+docs+valgrind + autotools+clang + meson+clang + autotools+gcc+docs+el7+py2 + rpm+autotools + rpm+meson + tarball + subtree + + all + none ) - if [ "$run" = '' ] ; then + if [ "$run" = all ] ; then set -x return 0 fi @@ -83,6 +85,7 @@ check_run_assert() { die "invalid NM_TEST_SELECT_RUN value \"$1\"" } +[ -z "$NM_TEST_SELECT_RUN" ] && NM_TEST_SELECT_RUN=all check_run_assert "$NM_TEST_SELECT_RUN" check_run() { @@ -93,9 +96,9 @@ check_run() { # Usually, we run the build several times. However, for testing # the build script manually, it can be useful to explicitly select # one step to run. For example, if step 3 is known to fail, you - # can still manually run step 4 by setting NM_TEST_SELECT_RUN=4. + # can still manually run step A by setting NM_TEST_SELECT_RUN=A. - test -z "$NM_TEST_SELECT_RUN" -o "$NM_TEST_SELECT_RUN" = "$test_no" + test "$NM_TEST_SELECT_RUN" = all -o "$NM_TEST_SELECT_RUN" = "$test_no" } check_run_clean() { @@ -106,21 +109,21 @@ check_run_clean() { return 0 } -if check_run_clean 1 ; then +if check_run_clean autotools+gcc+docs+valgrind ; then BUILD_TYPE=autotools CC=gcc WITH_DOCS=1 WITH_VALGRIND=1 contrib/scripts/nm-ci-run.sh mv build/INST/share/gtk-doc/html "$ARTIFACT_DIR/docs-html" fi -check_run_clean 2 && BUILD_TYPE=meson CC=gcc WITH_DOCS=1 WITH_VALGRIND=1 contrib/scripts/nm-ci-run.sh -check_run_clean 3 && BUILD_TYPE=autotools CC=clang WITH_DOCS=0 contrib/scripts/nm-ci-run.sh -check_run_clean 4 && BUILD_TYPE=meson CC=clang WITH_DOCS=0 contrib/scripts/nm-ci-run.sh +check_run_clean meson+gcc+docs+valgrind && BUILD_TYPE=meson CC=gcc WITH_DOCS=1 WITH_VALGRIND=1 contrib/scripts/nm-ci-run.sh +check_run_clean autotools+clang && BUILD_TYPE=autotools CC=clang WITH_DOCS=0 contrib/scripts/nm-ci-run.sh +check_run_clean meson+clang && BUILD_TYPE=meson CC=clang WITH_DOCS=0 contrib/scripts/nm-ci-run.sh -check_run_clean 5 && test $IS_CENTOS_7 = 1 && PYTHON=python2 BUILD_TYPE=autotools CC=gcc WITH_DOCS=1 contrib/scripts/nm-ci-run.sh +check_run_clean autotools+gcc+docs+el7+py2 && test $IS_CENTOS_7 = 1 && PYTHON=python2 BUILD_TYPE=autotools CC=gcc WITH_DOCS=1 contrib/scripts/nm-ci-run.sh -check_run_clean 6 && test $IS_FEDORA = 1 -o $IS_CENTOS = 1 && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -W meson -check_run_clean 7 && test $IS_FEDORA = 1 && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -w meson +check_run_clean rpm+autotools && test $IS_FEDORA = 1 -o $IS_CENTOS = 1 && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -W meson +check_run_clean rpm+meson && test $IS_FEDORA = 1 && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -w meson -if check_run_clean 8 && [ "$NM_BUILD_TARBALL" = 1 ]; then +if check_run_clean tarball && [ "$NM_BUILD_TARBALL" = 1 ]; then SIGN_SOURCE=0 ./contrib/fedora/rpm/build_clean.sh -r mv ./NetworkManager-1*.tar.xz "$ARTIFACT_DIR/" mv ./contrib/fedora/rpm/latest/SRPMS/NetworkManager-1*.src.rpm "$ARTIFACT_DIR/" @@ -152,7 +155,7 @@ test_subtree() { popd } -if check_run_clean 10; then +if check_run_clean subtree; then for d in c-list c-rbtree c-siphash c-stdaux n-acd n-dhcp4 ; do for cc in gcc clang; do test_subtree "$d" "$cc" @@ -162,18 +165,16 @@ fi ############################################################################### -if [ -z "$NM_TEST_SELECT_RUN" ] ; then +if [ "$NM_BUILD_TARBALL" = 1 ]; then do_clean - if [ "$NM_BUILD_TARBALL" = 1 ]; then - if check_run 1 ; then - mv "$ARTIFACT_DIR/docs-html/" ./ - fi - if check_run 8 ; then - mv \ - "$ARTIFACT_DIR"/NetworkManager-1*.tar.xz \ - "$ARTIFACT_DIR"/NetworkManager-1*.src.rpm \ - ./ - fi + if check_run autotools+gcc+docs+valgrind ; then + mv "$ARTIFACT_DIR/docs-html/" ./ + fi + if check_run tarball ; then + mv \ + "$ARTIFACT_DIR"/NetworkManager-1*.tar.xz \ + "$ARTIFACT_DIR"/NetworkManager-1*.src.rpm \ + ./ fi fi From b06ddab9d42a3ca3379ff6f6d5997a0fd5cc5bea Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Tue, 11 Apr 2023 12:23:30 +0200 Subject: [PATCH 3/9] gitlab-ci: add multiple stages/tiers for tests We have many test configurations (i.e. distros like fedora:37, debian:9). Almost all of them run manually triggered, because running them every time would be wasteful. Still, even as we trigger those tests only seldom, whenever we trigger them all together, they consume still too many resources of the freedesktop.org gitlab infrastructure. One possibility would be to just drop old distros (e.g. fedora:30). Which tests are setup in gitlab-ci is constantly refined and adjusted. So dropping some distros is not necessarily wrong and bound to happen eventually. However, I also don't find it great to just disable tests that are still passing. If we want to avoid consuming too many resources, we can just choose not to run those tests. We don't need to enforce that by deleting tests. Once deleted, such a configuration cannot be tested anymore as it would be too cumbersome to recreate the setup manually. Instead, introduce stages/tiers to clearer mark configuration that we should test even less frequently. Note that it is still required from the developer to not trigger too many tests at once, to not monopolize the CI resources. The stages should make that clearer to see, but don't solve it. Deleting tests might solve it, but only if we delete a significant number of those tests, which seems not desirable. --- .gitlab-ci.yml | 573 ++++++++++++++++++++++------------------- .gitlab-ci/ci.template | 71 +++-- .gitlab-ci/config.yml | 49 +++- 3 files changed, 382 insertions(+), 311 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 71fe4e4cd..eeae024b6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,8 @@ .templates_sha: &template_sha ffe4d1b10aab7534489f0c4bbc4c5899df17d3f2 # see https://docs.gitlab.com/ee/ci/yaml/#includefile + + include: # Alpine container builder template - project: 'freedesktop/ci-templates' @@ -39,7 +41,9 @@ include: stages: - prep - - test + - tier1 + - tier2 + - tier3 - deploy - triage - container_clean @@ -53,17 +57,17 @@ variables: # # This is done by running `ci-fairy generate-template` and possibly bumping # ".default_tag". - FEDORA_TAG: '2023-01-18.0-82ad875db2dc' - UBUNTU_TAG: '2023-01-18.0-b674114b79c1' - DEBIAN_TAG: '2023-01-18.0-b674114b79c1' - CENTOS_TAG: '2023-01-18.0-82ad875db2dc' - ALPINE_TAG: '2023-01-18.0-14c807942fa4' + ALPINE_TAG: '2023-04-11.0-c319e2a8ce5b' + CENTOS_TAG: '2023-04-11.0-3322a2e54777' + DEBIAN_TAG: '2023-04-11.0-b529fd8eba83' + FEDORA_TAG: '2023-04-11.0-3322a2e54777' + UBUNTU_TAG: '2023-04-11.0-b529fd8eba83' + ALPINE_EXEC: 'bash .gitlab-ci/alpine-install.sh' + CENTOS_EXEC: 'bash .gitlab-ci/fedora-install.sh' + DEBIAN_EXEC: 'bash .gitlab-ci/debian-install.sh' FEDORA_EXEC: 'bash .gitlab-ci/fedora-install.sh' UBUNTU_EXEC: 'bash .gitlab-ci/debian-install.sh' - DEBIAN_EXEC: 'bash .gitlab-ci/debian-install.sh' - CENTOS_EXEC: 'bash .gitlab-ci/fedora-install.sh' - ALPINE_EXEC: 'bash .gitlab-ci/alpine-install.sh' .nm_artifacts: variables: @@ -93,6 +97,36 @@ variables: # Build a container for each distribution + version. The ci-templates # will re-use the containers if the tag doesn't change. +fedora:37@container-prep: + extends: + - .fdo.container-build@fedora + stage: prep + variables: + GIT_STRATEGY: none + FDO_DISTRIBUTION_VERSION: '37' + FDO_DISTRIBUTION_TAG: $FEDORA_TAG + FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC + +fedora:36@container-prep: + extends: + - .fdo.container-build@fedora + stage: prep + variables: + GIT_STRATEGY: none + FDO_DISTRIBUTION_VERSION: '36' + FDO_DISTRIBUTION_TAG: $FEDORA_TAG + FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC + +fedora:38@container-prep: + extends: + - .fdo.container-build@fedora + stage: prep + variables: + GIT_STRATEGY: none + FDO_DISTRIBUTION_VERSION: '38' + FDO_DISTRIBUTION_TAG: $FEDORA_TAG + FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC + fedora:30@container-prep: extends: - .fdo.container-build@fedora @@ -153,36 +187,6 @@ fedora:35@container-prep: FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC -fedora:36@container-prep: - extends: - - .fdo.container-build@fedora - stage: prep - variables: - GIT_STRATEGY: none - FDO_DISTRIBUTION_VERSION: '36' - FDO_DISTRIBUTION_TAG: $FEDORA_TAG - FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC - -fedora:37@container-prep: - extends: - - .fdo.container-build@fedora - stage: prep - variables: - GIT_STRATEGY: none - FDO_DISTRIBUTION_VERSION: '37' - FDO_DISTRIBUTION_TAG: $FEDORA_TAG - FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC - -fedora:38@container-prep: - extends: - - .fdo.container-build@fedora - stage: prep - variables: - GIT_STRATEGY: none - FDO_DISTRIBUTION_VERSION: '38' - FDO_DISTRIBUTION_TAG: $FEDORA_TAG - FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC - ubuntu:18.04@container-prep: extends: - .fdo.container-build@ubuntu @@ -233,16 +237,6 @@ ubuntu:rolling@container-prep: FDO_DISTRIBUTION_TAG: $UBUNTU_TAG FDO_DISTRIBUTION_EXEC: $UBUNTU_EXEC -debian:9@container-prep: - extends: - - .fdo.container-build@debian - stage: prep - variables: - GIT_STRATEGY: none - FDO_DISTRIBUTION_VERSION: '9' - FDO_DISTRIBUTION_TAG: $DEBIAN_TAG - FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC - debian:10@container-prep: extends: - .fdo.container-build@debian @@ -263,16 +257,6 @@ debian:11@container-prep: FDO_DISTRIBUTION_TAG: $DEBIAN_TAG FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC -debian:testing@container-prep: - extends: - - .fdo.container-build@debian - stage: prep - variables: - GIT_STRATEGY: none - FDO_DISTRIBUTION_VERSION: 'testing' - FDO_DISTRIBUTION_TAG: $DEBIAN_TAG - FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC - debian:sid@container-prep: extends: - .fdo.container-build@debian @@ -283,6 +267,26 @@ debian:sid@container-prep: FDO_DISTRIBUTION_TAG: $DEBIAN_TAG FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC +debian:9@container-prep: + extends: + - .fdo.container-build@debian + stage: prep + variables: + GIT_STRATEGY: none + FDO_DISTRIBUTION_VERSION: '9' + FDO_DISTRIBUTION_TAG: $DEBIAN_TAG + FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC + +debian:testing@container-prep: + extends: + - .fdo.container-build@debian + stage: prep + variables: + GIT_STRATEGY: none + FDO_DISTRIBUTION_VERSION: 'testing' + FDO_DISTRIBUTION_TAG: $DEBIAN_TAG + FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC + centos:7.5.1804@container-prep: extends: - .fdo.container-build@centos @@ -293,6 +297,36 @@ centos:7.5.1804@container-prep: FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC +centos:7.9.2009@container-prep: + extends: + - .fdo.container-build@centos + stage: prep + variables: + GIT_STRATEGY: none + FDO_DISTRIBUTION_VERSION: '7.9.2009' + FDO_DISTRIBUTION_TAG: $CENTOS_TAG + FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC + +centos:8.1.1911@container-prep: + extends: + - .fdo.container-build@centos + stage: prep + variables: + GIT_STRATEGY: none + FDO_DISTRIBUTION_VERSION: '8.1.1911' + FDO_DISTRIBUTION_TAG: $CENTOS_TAG + FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC + +centos:8.3.2011@container-prep: + extends: + - .fdo.container-build@centos + stage: prep + variables: + GIT_STRATEGY: none + FDO_DISTRIBUTION_VERSION: '8.3.2011' + FDO_DISTRIBUTION_TAG: $CENTOS_TAG + FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC + centos:7.6.1810@container-prep: extends: - .fdo.container-build@centos @@ -323,26 +357,6 @@ centos:7.8.2003@container-prep: FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC -centos:7.9.2009@container-prep: - extends: - - .fdo.container-build@centos - stage: prep - variables: - GIT_STRATEGY: none - FDO_DISTRIBUTION_VERSION: '7.9.2009' - FDO_DISTRIBUTION_TAG: $CENTOS_TAG - FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC - -centos:8.1.1911@container-prep: - extends: - - .fdo.container-build@centos - stage: prep - variables: - GIT_STRATEGY: none - FDO_DISTRIBUTION_VERSION: '8.1.1911' - FDO_DISTRIBUTION_TAG: $CENTOS_TAG - FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC - centos:8.2.2004@container-prep: extends: - .fdo.container-build@centos @@ -353,16 +367,6 @@ centos:8.2.2004@container-prep: FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC -centos:8.3.2011@container-prep: - extends: - - .fdo.container-build@centos - stage: prep - variables: - GIT_STRATEGY: none - FDO_DISTRIBUTION_VERSION: '8.3.2011' - FDO_DISTRIBUTION_TAG: $CENTOS_TAG - FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC - alpine:latest@container-prep: extends: - .fdo.container-build@alpine @@ -407,6 +411,33 @@ alpine:latest@container-prep: only: - schedules +fedora:37@container-clean: + extends: + - .container-clean + variables: + GIT_STRATEGY: none + CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/fedora/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG + FDO_DISTRIBUTION_VERSION: '37' + FDO_DISTRIBUTION_TAG: $FEDORA_TAG + +fedora:36@container-clean: + extends: + - .container-clean + variables: + GIT_STRATEGY: none + CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/fedora/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG + FDO_DISTRIBUTION_VERSION: '36' + FDO_DISTRIBUTION_TAG: $FEDORA_TAG + +fedora:38@container-clean: + extends: + - .container-clean + variables: + GIT_STRATEGY: none + CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/fedora/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG + FDO_DISTRIBUTION_VERSION: '38' + FDO_DISTRIBUTION_TAG: $FEDORA_TAG + fedora:30@container-clean: extends: - .container-clean @@ -461,33 +492,6 @@ fedora:35@container-clean: FDO_DISTRIBUTION_VERSION: '35' FDO_DISTRIBUTION_TAG: $FEDORA_TAG -fedora:36@container-clean: - extends: - - .container-clean - variables: - GIT_STRATEGY: none - CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/fedora/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG - FDO_DISTRIBUTION_VERSION: '36' - FDO_DISTRIBUTION_TAG: $FEDORA_TAG - -fedora:37@container-clean: - extends: - - .container-clean - variables: - GIT_STRATEGY: none - CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/fedora/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG - FDO_DISTRIBUTION_VERSION: '37' - FDO_DISTRIBUTION_TAG: $FEDORA_TAG - -fedora:38@container-clean: - extends: - - .container-clean - variables: - GIT_STRATEGY: none - CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/fedora/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG - FDO_DISTRIBUTION_VERSION: '38' - FDO_DISTRIBUTION_TAG: $FEDORA_TAG - ubuntu:18.04@container-clean: extends: - .container-clean @@ -533,15 +537,6 @@ ubuntu:rolling@container-clean: FDO_DISTRIBUTION_VERSION: 'rolling' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG -debian:9@container-clean: - extends: - - .container-clean - variables: - GIT_STRATEGY: none - CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/debian/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG - FDO_DISTRIBUTION_VERSION: '9' - FDO_DISTRIBUTION_TAG: $DEBIAN_TAG - debian:10@container-clean: extends: - .container-clean @@ -560,15 +555,6 @@ debian:11@container-clean: FDO_DISTRIBUTION_VERSION: '11' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG -debian:testing@container-clean: - extends: - - .container-clean - variables: - GIT_STRATEGY: none - CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/debian/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG - FDO_DISTRIBUTION_VERSION: 'testing' - FDO_DISTRIBUTION_TAG: $DEBIAN_TAG - debian:sid@container-clean: extends: - .container-clean @@ -578,6 +564,24 @@ debian:sid@container-clean: FDO_DISTRIBUTION_VERSION: 'sid' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG +debian:9@container-clean: + extends: + - .container-clean + variables: + GIT_STRATEGY: none + CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/debian/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG + FDO_DISTRIBUTION_VERSION: '9' + FDO_DISTRIBUTION_TAG: $DEBIAN_TAG + +debian:testing@container-clean: + extends: + - .container-clean + variables: + GIT_STRATEGY: none + CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/debian/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG + FDO_DISTRIBUTION_VERSION: 'testing' + FDO_DISTRIBUTION_TAG: $DEBIAN_TAG + centos:7.5.1804@container-clean: extends: - .container-clean @@ -587,6 +591,33 @@ centos:7.5.1804@container-clean: FDO_DISTRIBUTION_VERSION: '7.5.1804' FDO_DISTRIBUTION_TAG: $CENTOS_TAG +centos:7.9.2009@container-clean: + extends: + - .container-clean + variables: + GIT_STRATEGY: none + CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/centos/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG + FDO_DISTRIBUTION_VERSION: '7.9.2009' + FDO_DISTRIBUTION_TAG: $CENTOS_TAG + +centos:8.1.1911@container-clean: + extends: + - .container-clean + variables: + GIT_STRATEGY: none + CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/centos/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG + FDO_DISTRIBUTION_VERSION: '8.1.1911' + FDO_DISTRIBUTION_TAG: $CENTOS_TAG + +centos:8.3.2011@container-clean: + extends: + - .container-clean + variables: + GIT_STRATEGY: none + CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/centos/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG + FDO_DISTRIBUTION_VERSION: '8.3.2011' + FDO_DISTRIBUTION_TAG: $CENTOS_TAG + centos:7.6.1810@container-clean: extends: - .container-clean @@ -614,24 +645,6 @@ centos:7.8.2003@container-clean: FDO_DISTRIBUTION_VERSION: '7.8.2003' FDO_DISTRIBUTION_TAG: $CENTOS_TAG -centos:7.9.2009@container-clean: - extends: - - .container-clean - variables: - GIT_STRATEGY: none - CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/centos/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG - FDO_DISTRIBUTION_VERSION: '7.9.2009' - FDO_DISTRIBUTION_TAG: $CENTOS_TAG - -centos:8.1.1911@container-clean: - extends: - - .container-clean - variables: - GIT_STRATEGY: none - CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/centos/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG - FDO_DISTRIBUTION_VERSION: '8.1.1911' - FDO_DISTRIBUTION_TAG: $CENTOS_TAG - centos:8.2.2004@container-clean: extends: - .container-clean @@ -641,15 +654,6 @@ centos:8.2.2004@container-clean: FDO_DISTRIBUTION_VERSION: '8.2.2004' FDO_DISTRIBUTION_TAG: $CENTOS_TAG -centos:8.3.2011@container-clean: - extends: - - .container-clean - variables: - GIT_STRATEGY: none - CURRENT_CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/centos/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG - FDO_DISTRIBUTION_VERSION: '8.3.2011' - FDO_DISTRIBUTION_TAG: $CENTOS_TAG - alpine:latest@container-clean: extends: - .container-clean @@ -667,7 +671,6 @@ alpine:latest@container-clean: ################################################################# .build@template: - stage: test script: - env - r=0 @@ -684,11 +687,50 @@ alpine:latest@container-clean: ################################################################# +t_fedora:37: + extends: + - .build@template + - .fdo.distribution-image@fedora + - .nm_artifacts + stage: tier1 + variables: + FDO_DISTRIBUTION_VERSION: '37' + FDO_DISTRIBUTION_TAG: $FEDORA_TAG + needs: + - "fedora:37@container-prep" + +t_fedora:36: + extends: + - .build@template + - .fdo.distribution-image@fedora + - .nm_artifacts_debug + stage: tier2 + variables: + FDO_DISTRIBUTION_VERSION: '36' + FDO_DISTRIBUTION_TAG: $FEDORA_TAG + needs: + - "fedora:36@container-prep" + when: manual + +t_fedora:38: + extends: + - .build@template + - .fdo.distribution-image@fedora + - .nm_artifacts_debug + stage: tier2 + variables: + FDO_DISTRIBUTION_VERSION: '38' + FDO_DISTRIBUTION_TAG: $FEDORA_TAG + needs: + - "fedora:38@container-prep" + when: manual + t_fedora:30: extends: - .build@template - .fdo.distribution-image@fedora - .nm_artifacts_debug + stage: tier3 variables: FDO_DISTRIBUTION_VERSION: '30' FDO_DISTRIBUTION_TAG: $FEDORA_TAG @@ -701,6 +743,7 @@ t_fedora:31: - .build@template - .fdo.distribution-image@fedora - .nm_artifacts_debug + stage: tier3 variables: FDO_DISTRIBUTION_VERSION: '31' FDO_DISTRIBUTION_TAG: $FEDORA_TAG @@ -713,6 +756,7 @@ t_fedora:32: - .build@template - .fdo.distribution-image@fedora - .nm_artifacts_debug + stage: tier3 variables: FDO_DISTRIBUTION_VERSION: '32' FDO_DISTRIBUTION_TAG: $FEDORA_TAG @@ -725,6 +769,7 @@ t_fedora:33: - .build@template - .fdo.distribution-image@fedora - .nm_artifacts_debug + stage: tier3 variables: FDO_DISTRIBUTION_VERSION: '33' FDO_DISTRIBUTION_TAG: $FEDORA_TAG @@ -737,6 +782,7 @@ t_fedora:34: - .build@template - .fdo.distribution-image@fedora - .nm_artifacts_debug + stage: tier3 variables: FDO_DISTRIBUTION_VERSION: '34' FDO_DISTRIBUTION_TAG: $FEDORA_TAG @@ -749,6 +795,7 @@ t_fedora:35: - .build@template - .fdo.distribution-image@fedora - .nm_artifacts_debug + stage: tier3 variables: FDO_DISTRIBUTION_VERSION: '35' FDO_DISTRIBUTION_TAG: $FEDORA_TAG @@ -756,46 +803,12 @@ t_fedora:35: - "fedora:35@container-prep" when: manual -t_fedora:36: - extends: - - .build@template - - .fdo.distribution-image@fedora - - .nm_artifacts_debug - variables: - FDO_DISTRIBUTION_VERSION: '36' - FDO_DISTRIBUTION_TAG: $FEDORA_TAG - needs: - - "fedora:36@container-prep" - when: manual - -t_fedora:37: - extends: - - .build@template - - .fdo.distribution-image@fedora - - .nm_artifacts - variables: - FDO_DISTRIBUTION_VERSION: '37' - FDO_DISTRIBUTION_TAG: $FEDORA_TAG - needs: - - "fedora:37@container-prep" - -t_fedora:38: - extends: - - .build@template - - .fdo.distribution-image@fedora - - .nm_artifacts_debug - variables: - FDO_DISTRIBUTION_VERSION: '38' - FDO_DISTRIBUTION_TAG: $FEDORA_TAG - needs: - - "fedora:38@container-prep" - when: manual - t_ubuntu:18.04: extends: - .build@template - .fdo.distribution-image@ubuntu - .nm_artifacts_debug + stage: tier2 variables: FDO_DISTRIBUTION_VERSION: '18.04' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG @@ -808,6 +821,7 @@ t_ubuntu:20.04: - .build@template - .fdo.distribution-image@ubuntu - .nm_artifacts_debug + stage: tier2 variables: FDO_DISTRIBUTION_VERSION: '20.04' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG @@ -820,6 +834,7 @@ t_ubuntu:22.04: - .build@template - .fdo.distribution-image@ubuntu - .nm_artifacts_debug + stage: tier2 variables: FDO_DISTRIBUTION_VERSION: '22.04' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG @@ -832,6 +847,7 @@ t_ubuntu:devel: - .build@template - .fdo.distribution-image@ubuntu - .nm_artifacts_debug + stage: tier2 variables: FDO_DISTRIBUTION_VERSION: 'devel' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG @@ -844,6 +860,7 @@ t_ubuntu:rolling: - .build@template - .fdo.distribution-image@ubuntu - .nm_artifacts_debug + stage: tier2 variables: FDO_DISTRIBUTION_VERSION: 'rolling' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG @@ -851,23 +868,12 @@ t_ubuntu:rolling: - "ubuntu:rolling@container-prep" when: manual -t_debian:9: - extends: - - .build@template - - .fdo.distribution-image@debian - - .nm_artifacts_debug - variables: - FDO_DISTRIBUTION_VERSION: '9' - FDO_DISTRIBUTION_TAG: $DEBIAN_TAG - needs: - - "debian:9@container-prep" - when: manual - t_debian:10: extends: - .build@template - .fdo.distribution-image@debian - .nm_artifacts_debug + stage: tier2 variables: FDO_DISTRIBUTION_VERSION: '10' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG @@ -880,6 +886,7 @@ t_debian:11: - .build@template - .fdo.distribution-image@debian - .nm_artifacts_debug + stage: tier2 variables: FDO_DISTRIBUTION_VERSION: '11' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG @@ -887,23 +894,12 @@ t_debian:11: - "debian:11@container-prep" when: manual -t_debian:testing: - extends: - - .build@template - - .fdo.distribution-image@debian - - .nm_artifacts_debug - variables: - FDO_DISTRIBUTION_VERSION: 'testing' - FDO_DISTRIBUTION_TAG: $DEBIAN_TAG - needs: - - "debian:testing@container-prep" - when: manual - t_debian:sid: extends: - .build@template - .fdo.distribution-image@debian - .nm_artifacts_debug + stage: tier2 variables: FDO_DISTRIBUTION_VERSION: 'sid' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG @@ -911,11 +907,38 @@ t_debian:sid: - "debian:sid@container-prep" when: manual +t_debian:9: + extends: + - .build@template + - .fdo.distribution-image@debian + - .nm_artifacts_debug + stage: tier3 + variables: + FDO_DISTRIBUTION_VERSION: '9' + FDO_DISTRIBUTION_TAG: $DEBIAN_TAG + needs: + - "debian:9@container-prep" + when: manual + +t_debian:testing: + extends: + - .build@template + - .fdo.distribution-image@debian + - .nm_artifacts_debug + stage: tier3 + variables: + FDO_DISTRIBUTION_VERSION: 'testing' + FDO_DISTRIBUTION_TAG: $DEBIAN_TAG + needs: + - "debian:testing@container-prep" + when: manual + t_centos:7.5.1804: extends: - .build@template - .fdo.distribution-image@centos - .nm_artifacts_debug + stage: tier2 variables: FDO_DISTRIBUTION_VERSION: '7.5.1804' FDO_DISTRIBUTION_TAG: $CENTOS_TAG @@ -923,47 +946,12 @@ t_centos:7.5.1804: - "centos:7.5.1804@container-prep" when: manual -t_centos:7.6.1810: - extends: - - .build@template - - .fdo.distribution-image@centos - - .nm_artifacts_debug - variables: - FDO_DISTRIBUTION_VERSION: '7.6.1810' - FDO_DISTRIBUTION_TAG: $CENTOS_TAG - needs: - - "centos:7.6.1810@container-prep" - when: manual - -t_centos:7.7.1908: - extends: - - .build@template - - .fdo.distribution-image@centos - - .nm_artifacts_debug - variables: - FDO_DISTRIBUTION_VERSION: '7.7.1908' - FDO_DISTRIBUTION_TAG: $CENTOS_TAG - needs: - - "centos:7.7.1908@container-prep" - when: manual - -t_centos:7.8.2003: - extends: - - .build@template - - .fdo.distribution-image@centos - - .nm_artifacts_debug - variables: - FDO_DISTRIBUTION_VERSION: '7.8.2003' - FDO_DISTRIBUTION_TAG: $CENTOS_TAG - needs: - - "centos:7.8.2003@container-prep" - when: manual - t_centos:7.9.2009: extends: - .build@template - .fdo.distribution-image@centos - .nm_artifacts_debug + stage: tier2 variables: FDO_DISTRIBUTION_VERSION: '7.9.2009' FDO_DISTRIBUTION_TAG: $CENTOS_TAG @@ -976,6 +964,7 @@ t_centos:8.1.1911: - .build@template - .fdo.distribution-image@centos - .nm_artifacts_debug + stage: tier2 variables: FDO_DISTRIBUTION_VERSION: '8.1.1911' FDO_DISTRIBUTION_TAG: $CENTOS_TAG @@ -983,23 +972,12 @@ t_centos:8.1.1911: - "centos:8.1.1911@container-prep" when: manual -t_centos:8.2.2004: - extends: - - .build@template - - .fdo.distribution-image@centos - - .nm_artifacts_debug - variables: - FDO_DISTRIBUTION_VERSION: '8.2.2004' - FDO_DISTRIBUTION_TAG: $CENTOS_TAG - needs: - - "centos:8.2.2004@container-prep" - when: manual - t_centos:8.3.2011: extends: - .build@template - .fdo.distribution-image@centos - .nm_artifacts_debug + stage: tier2 variables: FDO_DISTRIBUTION_VERSION: '8.3.2011' FDO_DISTRIBUTION_TAG: $CENTOS_TAG @@ -1007,11 +985,64 @@ t_centos:8.3.2011: - "centos:8.3.2011@container-prep" when: manual +t_centos:7.6.1810: + extends: + - .build@template + - .fdo.distribution-image@centos + - .nm_artifacts_debug + stage: tier3 + variables: + FDO_DISTRIBUTION_VERSION: '7.6.1810' + FDO_DISTRIBUTION_TAG: $CENTOS_TAG + needs: + - "centos:7.6.1810@container-prep" + when: manual + +t_centos:7.7.1908: + extends: + - .build@template + - .fdo.distribution-image@centos + - .nm_artifacts_debug + stage: tier3 + variables: + FDO_DISTRIBUTION_VERSION: '7.7.1908' + FDO_DISTRIBUTION_TAG: $CENTOS_TAG + needs: + - "centos:7.7.1908@container-prep" + when: manual + +t_centos:7.8.2003: + extends: + - .build@template + - .fdo.distribution-image@centos + - .nm_artifacts_debug + stage: tier3 + variables: + FDO_DISTRIBUTION_VERSION: '7.8.2003' + FDO_DISTRIBUTION_TAG: $CENTOS_TAG + needs: + - "centos:7.8.2003@container-prep" + when: manual + +t_centos:8.2.2004: + extends: + - .build@template + - .fdo.distribution-image@centos + - .nm_artifacts_debug + stage: tier3 + variables: + FDO_DISTRIBUTION_VERSION: '8.2.2004' + FDO_DISTRIBUTION_TAG: $CENTOS_TAG + needs: + - "centos:8.2.2004@container-prep" + when: manual + t_alpine:latest: extends: - .build@template - .fdo.distribution-image@alpine - .nm_artifacts_debug + stage: tier2 variables: FDO_DISTRIBUTION_VERSION: 'latest' FDO_DISTRIBUTION_TAG: $ALPINE_TAG @@ -1033,7 +1064,7 @@ check-patch: FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - "fedora:37@container-prep" - stage: test + stage: tier1 script: - date '+%Y%m%d-%H%M%S'; NM_CHECKPATCH_FETCH_UPSTREAM=1 contrib/scripts/checkpatch-feature-branch.sh allow_failure: true @@ -1046,7 +1077,7 @@ check-tree: FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - "fedora:37@container-prep" - stage: test + stage: tier1 script: - date '+%Y%m%d-%H%M%S'; contrib/scripts/nm-python-black-format.sh --check - date '+%Y%m%d-%H%M%S'; git ls-files -z -- 'po/*.po' | xargs -0 -n1 msgfmt -vc diff --git a/.gitlab-ci/ci.template b/.gitlab-ci/ci.template index 02b33ff3b..e06b9210c 100644 --- a/.gitlab-ci/ci.template +++ b/.gitlab-ci/ci.template @@ -17,17 +17,37 @@ .templates_sha: &template_sha ffe4d1b10aab7534489f0c4bbc4c5899df17d3f2 # see https://docs.gitlab.com/ee/ci/yaml/#includefile +{# Group distros by their common (name,base_type,tag) tuples.#} +{% set distro_groups = [] %} +{% for distro in distributions %} + {% set g = {'name':distro.name,'base_type':distro.base_type,'tag':distro.tag} %} + {% if g not in distro_groups %} + {% do distro_groups.append(g) %} + {% endif %} +{% endfor %} + +{# The "default_distro" builds our pages and is used for check-{tree,patch} tests. It is the first distro with tier 1. #} +{% set default_distro = [] %} +{% for distro in distributions %} + {% if distro.tier == 1 and default_distro|length == 0 %} + {% do default_distro.append(distro) %} + {% endif %} +{% endfor %} +{% set default_distro = default_distro[0] %} + include: -{% for distro in distributions|sort(attribute="name") %} - # {{ distro.name.capitalize() }} container builder template +{% for distro_group in distro_groups|sort(attribute='name') %} + # {{ distro_group.name.capitalize() }} container builder template - project: 'freedesktop/ci-templates' ref: *template_sha - file: '/templates/{{distro.name}}.yml' + file: '/templates/{{distro_group.name}}.yml' {% endfor %} stages: - prep - - test + - tier1 + - tier2 + - tier3 - deploy - triage - container_clean @@ -41,17 +61,17 @@ variables: # # This is done by running `ci-fairy generate-template` and possibly bumping # ".default_tag". -{% for distro in distributions %} - {{"%-13s"| format(distro.name.upper() + '_TAG:')}}'{{distro.tag}}-{{ +{% for distro_group in distro_groups|sort(attribute='name') %} + {{"%-13s"| format(distro_group.name.upper() + '_TAG:')}}'{{distro_group.tag}}-{{ (ci_fairy.hashfiles('./.gitlab-ci/config.yml', './.gitlab-ci/ci.template', - './.gitlab-ci/' + distro.base_type + '-install.sh', - './contrib/' + distro.base_type + '/REQUIRED_PACKAGES'))[0:12] + './.gitlab-ci/' + distro_group.base_type + '-install.sh', + './contrib/' + distro_group.base_type + '/REQUIRED_PACKAGES'))[0:12] }}' {% endfor %} -{% for distro in distributions %} - {{"%-13s"| format(distro.name.upper() + '_EXEC:')}}'bash .gitlab-ci/{{distro.base_type}}-install.sh' +{% for distro_group in distro_groups|sort(attribute='name') %} + {{"%-13s"| format(distro_group.name.upper() + '_EXEC:')}}'bash .gitlab-ci/{{distro_group.base_type}}-install.sh' {% endfor %} .nm_artifacts: @@ -151,7 +171,6 @@ variables: ################################################################# .build@template: - stage: test script: - env - r=0 @@ -174,12 +193,12 @@ t_{{distro.name}}:{{version}}: extends: - .build@template - .fdo.distribution-image@{{distro.name}} -{% if distro.name == pages_build.name and - version == pages_build.version %} +{% if distro == default_distro %} - .nm_artifacts {% else %} - .nm_artifacts_debug {% endif %} + stage: tier{{distro.tier}} variables: FDO_DISTRIBUTION_VERSION: '{{version}}' FDO_DISTRIBUTION_TAG: ${{distro.name.upper()}}_TAG @@ -191,7 +210,7 @@ t_{{distro.name}}:{{version}}: {% endif %} needs: - "{{distro.name}}:{{version}}@container-prep" -{% if not version in distro.get('always', []) and (distro.name != pages_build.name or version != pages_build.version) %} +{% if distro.tier > 1 %} when: manual {% endif %} {% endfor %} @@ -205,26 +224,26 @@ t_{{distro.name}}:{{version}}: check-patch: extends: - - .fdo.distribution-image@{{pages_build.name}} + - .fdo.distribution-image@{{default_distro.name}} variables: - FDO_DISTRIBUTION_VERSION: '{{pages_build.version}}' - FDO_DISTRIBUTION_TAG: ${{pages_build.name.upper()}}_TAG + FDO_DISTRIBUTION_VERSION: '{{default_distro.versions[0]}}' + FDO_DISTRIBUTION_TAG: ${{default_distro.name.upper()}}_TAG needs: - - "{{pages_build.name}}:{{pages_build.version}}@container-prep" - stage: test + - "{{default_distro.name}}:{{default_distro.versions[0]}}@container-prep" + stage: tier1 script: - date '+%Y%m%d-%H%M%S'; NM_CHECKPATCH_FETCH_UPSTREAM=1 contrib/scripts/checkpatch-feature-branch.sh allow_failure: true check-tree: extends: - - .fdo.distribution-image@{{pages_build.name}} + - .fdo.distribution-image@{{default_distro.name}} variables: - FDO_DISTRIBUTION_VERSION: '{{pages_build.version}}' - FDO_DISTRIBUTION_TAG: ${{pages_build.name.upper()}}_TAG + FDO_DISTRIBUTION_VERSION: '{{default_distro.versions[0]}}' + FDO_DISTRIBUTION_TAG: ${{default_distro.name.upper()}}_TAG needs: - - "{{pages_build.name}}:{{pages_build.version}}@container-prep" - stage: test + - "{{default_distro.name}}:{{default_distro.versions[0]}}@container-prep" + stage: tier1 script: - date '+%Y%m%d-%H%M%S'; contrib/scripts/nm-python-black-format.sh --check - date '+%Y%m%d-%H%M%S'; git ls-files -z -- 'po/*.po' | xargs -0 -n1 msgfmt -vc @@ -242,9 +261,9 @@ pages: only: - main dependencies: - - t_{{pages_build.name}}:{{pages_build.version}} + - t_{{default_distro.name}}:{{default_distro.versions[0]}} needs: - - t_{{pages_build.name}}:{{pages_build.version}} + - t_{{default_distro.name}}:{{default_distro.versions[0]}} triage:issues: stage: triage diff --git a/.gitlab-ci/config.yml b/.gitlab-ci/config.yml index 117785c8c..7508fec74 100644 --- a/.gitlab-ci/config.yml +++ b/.gitlab-ci/config.yml @@ -8,7 +8,7 @@ # # We're happy to rebuild all containers when one changes. -.default_tag: &default_tag '2023-01-18.0' +.default_tag: &default_tag '2023-04-11.0' # The list of all distributions we want to create job for. @@ -17,6 +17,21 @@ distributions: - name: fedora tag: *default_tag base_type: fedora + # Tier1 is used to build the pages and check-{tree,patch} + tier: 1 + versions: + - '37' + - name: fedora + tag: *default_tag + base_type: fedora + tier: 2 + versions: + - '36' + - '38' + - name: fedora + tag: *default_tag + base_type: fedora + tier: 3 versions: - '30' - '31' @@ -24,12 +39,10 @@ distributions: - '33' - '34' - '35' - - '36' - - '37' - - '38' - name: ubuntu tag: *default_tag base_type: debian + tier: 2 versions: - '18.04' - '20.04' @@ -39,31 +52,39 @@ distributions: - name: debian tag: *default_tag base_type: debian + tier: 2 versions: - - '9' - '10' - '11' - - 'testing' - 'sid' + - name: debian + tag: *default_tag + base_type: debian + tier: 3 + versions: + - '9' + - 'testing' - name: centos tag: *default_tag base_type: fedora + tier: 2 versions: - '7.5.1804' + - '7.9.2009' + - '8.1.1911' + - '8.3.2011' + - name: centos + tag: *default_tag + base_type: fedora + tier: 3 + versions: - '7.6.1810' - '7.7.1908' - '7.8.2003' - - '7.9.2009' - - '8.1.1911' - '8.2.2004' - - '8.3.2011' - name: alpine tag: *default_tag base_type: alpine + tier: 2 versions: - 'latest' - -# specifies which of the above distros is used as source for pages -pages_build: - name: fedora - version: '37' From e41fe546f7eac2b44f98ba64eea6c09fe4ebca9b Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Wed, 12 Apr 2023 17:01:19 +0200 Subject: [PATCH 4/9] gitlab-ci: use parallel:matrix for tier1 tests The benefit is that instead of one long running job for fedora:37 (the current tier1 test), we have several smaller. A minor downside is, that if the build is broken, then usually the very first test would already fail. Previously, that meant that the follow up tests were skipped. Now, they run all in parallel. However, test failures should be the exception, so the wasted resources are probably irrelevant. The upside is, that we can see which tests fail, and we run them much faster (in parallel). This is only done for the tier1 test, because those tests are started automatically. Other tiers need to be triggered manually, which already means a lot of clicking. Making those also matrix tests, would result in an insane amount of clicking. As those other tests are run much more seldom, having them huge is probably fine. --- .gitlab-ci.yml | 25 ++++++++++++++++++------- .gitlab-ci/ci.template | 17 +++++++++++++++-- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eeae024b6..2add66a05 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,11 +57,11 @@ variables: # # This is done by running `ci-fairy generate-template` and possibly bumping # ".default_tag". - ALPINE_TAG: '2023-04-11.0-c319e2a8ce5b' - CENTOS_TAG: '2023-04-11.0-3322a2e54777' - DEBIAN_TAG: '2023-04-11.0-b529fd8eba83' - FEDORA_TAG: '2023-04-11.0-3322a2e54777' - UBUNTU_TAG: '2023-04-11.0-b529fd8eba83' + ALPINE_TAG: '2023-04-11.0-5bd75d53d745' + CENTOS_TAG: '2023-04-11.0-5bda0c516ecd' + DEBIAN_TAG: '2023-04-11.0-8dcd2b8ca112' + FEDORA_TAG: '2023-04-11.0-5bda0c516ecd' + UBUNTU_TAG: '2023-04-11.0-8dcd2b8ca112' ALPINE_EXEC: 'bash .gitlab-ci/alpine-install.sh' CENTOS_EXEC: 'bash .gitlab-ci/fedora-install.sh' @@ -693,6 +693,17 @@ t_fedora:37: - .fdo.distribution-image@fedora - .nm_artifacts stage: tier1 + parallel: + matrix: + - NM_TEST_SELECT_RUN: + - autotools+gcc+docs+valgrind + - meson+gcc+docs+valgrind + - autotools+clang + - meson+clang + - rpm+autotools + - rpm+meson + - tarball + - subtree variables: FDO_DISTRIBUTION_VERSION: '37' FDO_DISTRIBUTION_TAG: $FEDORA_TAG @@ -1095,9 +1106,9 @@ pages: only: - main dependencies: - - t_fedora:37 + - "t_fedora:37: [autotools+gcc+docs+valgrind]" needs: - - t_fedora:37 + - "t_fedora:37: [autotools+gcc+docs+valgrind]" triage:issues: stage: triage diff --git a/.gitlab-ci/ci.template b/.gitlab-ci/ci.template index e06b9210c..b58d5c6c8 100644 --- a/.gitlab-ci/ci.template +++ b/.gitlab-ci/ci.template @@ -199,6 +199,19 @@ t_{{distro.name}}:{{version}}: - .nm_artifacts_debug {% endif %} stage: tier{{distro.tier}} +{% if distro.tier <= 1 %} + parallel: + matrix: + - NM_TEST_SELECT_RUN: + - autotools+gcc+docs+valgrind + - meson+gcc+docs+valgrind + - autotools+clang + - meson+clang + - rpm+autotools + - rpm+meson + - tarball + - subtree +{% endif %} variables: FDO_DISTRIBUTION_VERSION: '{{version}}' FDO_DISTRIBUTION_TAG: ${{distro.name.upper()}}_TAG @@ -261,9 +274,9 @@ pages: only: - main dependencies: - - t_{{default_distro.name}}:{{default_distro.versions[0]}} + - "t_{{default_distro.name}}:{{default_distro.versions[0]}}: [autotools+gcc+docs+valgrind]" needs: - - t_{{default_distro.name}}:{{default_distro.versions[0]}} + - "t_{{default_distro.name}}:{{default_distro.versions[0]}}: [autotools+gcc+docs+valgrind]" triage:issues: stage: triage From 31c05da92c44ab22bfeafc3e492affe5dad89202 Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Wed, 12 Apr 2023 21:05:29 +0200 Subject: [PATCH 5/9] gitlab-ci: rename "@container-prep" tests to "@prep" The long name looks verbose and takes away space on the web page. Shorten the name. --- .gitlab-ci.yml | 126 ++++++++++++++++++++--------------------- .gitlab-ci/ci.template | 8 +-- 2 files changed, 67 insertions(+), 67 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2add66a05..6e5a487aa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,11 +57,11 @@ variables: # # This is done by running `ci-fairy generate-template` and possibly bumping # ".default_tag". - ALPINE_TAG: '2023-04-11.0-5bd75d53d745' - CENTOS_TAG: '2023-04-11.0-5bda0c516ecd' - DEBIAN_TAG: '2023-04-11.0-8dcd2b8ca112' - FEDORA_TAG: '2023-04-11.0-5bda0c516ecd' - UBUNTU_TAG: '2023-04-11.0-8dcd2b8ca112' + ALPINE_TAG: '2023-04-11.0-63b57b80e21f' + CENTOS_TAG: '2023-04-11.0-b6ce4d55572b' + DEBIAN_TAG: '2023-04-11.0-de33b560b02a' + FEDORA_TAG: '2023-04-11.0-b6ce4d55572b' + UBUNTU_TAG: '2023-04-11.0-de33b560b02a' ALPINE_EXEC: 'bash .gitlab-ci/alpine-install.sh' CENTOS_EXEC: 'bash .gitlab-ci/fedora-install.sh' @@ -97,7 +97,7 @@ variables: # Build a container for each distribution + version. The ci-templates # will re-use the containers if the tag doesn't change. -fedora:37@container-prep: +fedora:37@prep: extends: - .fdo.container-build@fedora stage: prep @@ -107,7 +107,7 @@ fedora:37@container-prep: FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC -fedora:36@container-prep: +fedora:36@prep: extends: - .fdo.container-build@fedora stage: prep @@ -117,7 +117,7 @@ fedora:36@container-prep: FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC -fedora:38@container-prep: +fedora:38@prep: extends: - .fdo.container-build@fedora stage: prep @@ -127,7 +127,7 @@ fedora:38@container-prep: FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC -fedora:30@container-prep: +fedora:30@prep: extends: - .fdo.container-build@fedora stage: prep @@ -137,7 +137,7 @@ fedora:30@container-prep: FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC -fedora:31@container-prep: +fedora:31@prep: extends: - .fdo.container-build@fedora stage: prep @@ -147,7 +147,7 @@ fedora:31@container-prep: FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC -fedora:32@container-prep: +fedora:32@prep: extends: - .fdo.container-build@fedora stage: prep @@ -157,7 +157,7 @@ fedora:32@container-prep: FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC -fedora:33@container-prep: +fedora:33@prep: extends: - .fdo.container-build@fedora stage: prep @@ -167,7 +167,7 @@ fedora:33@container-prep: FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC -fedora:34@container-prep: +fedora:34@prep: extends: - .fdo.container-build@fedora stage: prep @@ -177,7 +177,7 @@ fedora:34@container-prep: FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC -fedora:35@container-prep: +fedora:35@prep: extends: - .fdo.container-build@fedora stage: prep @@ -187,7 +187,7 @@ fedora:35@container-prep: FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC -ubuntu:18.04@container-prep: +ubuntu:18.04@prep: extends: - .fdo.container-build@ubuntu stage: prep @@ -197,7 +197,7 @@ ubuntu:18.04@container-prep: FDO_DISTRIBUTION_TAG: $UBUNTU_TAG FDO_DISTRIBUTION_EXEC: $UBUNTU_EXEC -ubuntu:20.04@container-prep: +ubuntu:20.04@prep: extends: - .fdo.container-build@ubuntu stage: prep @@ -207,7 +207,7 @@ ubuntu:20.04@container-prep: FDO_DISTRIBUTION_TAG: $UBUNTU_TAG FDO_DISTRIBUTION_EXEC: $UBUNTU_EXEC -ubuntu:22.04@container-prep: +ubuntu:22.04@prep: extends: - .fdo.container-build@ubuntu stage: prep @@ -217,7 +217,7 @@ ubuntu:22.04@container-prep: FDO_DISTRIBUTION_TAG: $UBUNTU_TAG FDO_DISTRIBUTION_EXEC: $UBUNTU_EXEC -ubuntu:devel@container-prep: +ubuntu:devel@prep: extends: - .fdo.container-build@ubuntu stage: prep @@ -227,7 +227,7 @@ ubuntu:devel@container-prep: FDO_DISTRIBUTION_TAG: $UBUNTU_TAG FDO_DISTRIBUTION_EXEC: $UBUNTU_EXEC -ubuntu:rolling@container-prep: +ubuntu:rolling@prep: extends: - .fdo.container-build@ubuntu stage: prep @@ -237,7 +237,7 @@ ubuntu:rolling@container-prep: FDO_DISTRIBUTION_TAG: $UBUNTU_TAG FDO_DISTRIBUTION_EXEC: $UBUNTU_EXEC -debian:10@container-prep: +debian:10@prep: extends: - .fdo.container-build@debian stage: prep @@ -247,7 +247,7 @@ debian:10@container-prep: FDO_DISTRIBUTION_TAG: $DEBIAN_TAG FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC -debian:11@container-prep: +debian:11@prep: extends: - .fdo.container-build@debian stage: prep @@ -257,7 +257,7 @@ debian:11@container-prep: FDO_DISTRIBUTION_TAG: $DEBIAN_TAG FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC -debian:sid@container-prep: +debian:sid@prep: extends: - .fdo.container-build@debian stage: prep @@ -267,7 +267,7 @@ debian:sid@container-prep: FDO_DISTRIBUTION_TAG: $DEBIAN_TAG FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC -debian:9@container-prep: +debian:9@prep: extends: - .fdo.container-build@debian stage: prep @@ -277,7 +277,7 @@ debian:9@container-prep: FDO_DISTRIBUTION_TAG: $DEBIAN_TAG FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC -debian:testing@container-prep: +debian:testing@prep: extends: - .fdo.container-build@debian stage: prep @@ -287,7 +287,7 @@ debian:testing@container-prep: FDO_DISTRIBUTION_TAG: $DEBIAN_TAG FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC -centos:7.5.1804@container-prep: +centos:7.5.1804@prep: extends: - .fdo.container-build@centos stage: prep @@ -297,7 +297,7 @@ centos:7.5.1804@container-prep: FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC -centos:7.9.2009@container-prep: +centos:7.9.2009@prep: extends: - .fdo.container-build@centos stage: prep @@ -307,7 +307,7 @@ centos:7.9.2009@container-prep: FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC -centos:8.1.1911@container-prep: +centos:8.1.1911@prep: extends: - .fdo.container-build@centos stage: prep @@ -317,7 +317,7 @@ centos:8.1.1911@container-prep: FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC -centos:8.3.2011@container-prep: +centos:8.3.2011@prep: extends: - .fdo.container-build@centos stage: prep @@ -327,7 +327,7 @@ centos:8.3.2011@container-prep: FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC -centos:7.6.1810@container-prep: +centos:7.6.1810@prep: extends: - .fdo.container-build@centos stage: prep @@ -337,7 +337,7 @@ centos:7.6.1810@container-prep: FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC -centos:7.7.1908@container-prep: +centos:7.7.1908@prep: extends: - .fdo.container-build@centos stage: prep @@ -347,7 +347,7 @@ centos:7.7.1908@container-prep: FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC -centos:7.8.2003@container-prep: +centos:7.8.2003@prep: extends: - .fdo.container-build@centos stage: prep @@ -357,7 +357,7 @@ centos:7.8.2003@container-prep: FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC -centos:8.2.2004@container-prep: +centos:8.2.2004@prep: extends: - .fdo.container-build@centos stage: prep @@ -367,7 +367,7 @@ centos:8.2.2004@container-prep: FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC -alpine:latest@container-prep: +alpine:latest@prep: extends: - .fdo.container-build@alpine stage: prep @@ -708,7 +708,7 @@ t_fedora:37: FDO_DISTRIBUTION_VERSION: '37' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:37@container-prep" + - "fedora:37@prep" t_fedora:36: extends: @@ -720,7 +720,7 @@ t_fedora:36: FDO_DISTRIBUTION_VERSION: '36' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:36@container-prep" + - "fedora:36@prep" when: manual t_fedora:38: @@ -733,7 +733,7 @@ t_fedora:38: FDO_DISTRIBUTION_VERSION: '38' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:38@container-prep" + - "fedora:38@prep" when: manual t_fedora:30: @@ -746,7 +746,7 @@ t_fedora:30: FDO_DISTRIBUTION_VERSION: '30' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:30@container-prep" + - "fedora:30@prep" when: manual t_fedora:31: @@ -759,7 +759,7 @@ t_fedora:31: FDO_DISTRIBUTION_VERSION: '31' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:31@container-prep" + - "fedora:31@prep" when: manual t_fedora:32: @@ -772,7 +772,7 @@ t_fedora:32: FDO_DISTRIBUTION_VERSION: '32' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:32@container-prep" + - "fedora:32@prep" when: manual t_fedora:33: @@ -785,7 +785,7 @@ t_fedora:33: FDO_DISTRIBUTION_VERSION: '33' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:33@container-prep" + - "fedora:33@prep" when: manual t_fedora:34: @@ -798,7 +798,7 @@ t_fedora:34: FDO_DISTRIBUTION_VERSION: '34' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:34@container-prep" + - "fedora:34@prep" when: manual t_fedora:35: @@ -811,7 +811,7 @@ t_fedora:35: FDO_DISTRIBUTION_VERSION: '35' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:35@container-prep" + - "fedora:35@prep" when: manual t_ubuntu:18.04: @@ -824,7 +824,7 @@ t_ubuntu:18.04: FDO_DISTRIBUTION_VERSION: '18.04' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG needs: - - "ubuntu:18.04@container-prep" + - "ubuntu:18.04@prep" when: manual t_ubuntu:20.04: @@ -837,7 +837,7 @@ t_ubuntu:20.04: FDO_DISTRIBUTION_VERSION: '20.04' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG needs: - - "ubuntu:20.04@container-prep" + - "ubuntu:20.04@prep" when: manual t_ubuntu:22.04: @@ -850,7 +850,7 @@ t_ubuntu:22.04: FDO_DISTRIBUTION_VERSION: '22.04' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG needs: - - "ubuntu:22.04@container-prep" + - "ubuntu:22.04@prep" when: manual t_ubuntu:devel: @@ -863,7 +863,7 @@ t_ubuntu:devel: FDO_DISTRIBUTION_VERSION: 'devel' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG needs: - - "ubuntu:devel@container-prep" + - "ubuntu:devel@prep" when: manual t_ubuntu:rolling: @@ -876,7 +876,7 @@ t_ubuntu:rolling: FDO_DISTRIBUTION_VERSION: 'rolling' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG needs: - - "ubuntu:rolling@container-prep" + - "ubuntu:rolling@prep" when: manual t_debian:10: @@ -889,7 +889,7 @@ t_debian:10: FDO_DISTRIBUTION_VERSION: '10' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG needs: - - "debian:10@container-prep" + - "debian:10@prep" when: manual t_debian:11: @@ -902,7 +902,7 @@ t_debian:11: FDO_DISTRIBUTION_VERSION: '11' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG needs: - - "debian:11@container-prep" + - "debian:11@prep" when: manual t_debian:sid: @@ -915,7 +915,7 @@ t_debian:sid: FDO_DISTRIBUTION_VERSION: 'sid' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG needs: - - "debian:sid@container-prep" + - "debian:sid@prep" when: manual t_debian:9: @@ -928,7 +928,7 @@ t_debian:9: FDO_DISTRIBUTION_VERSION: '9' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG needs: - - "debian:9@container-prep" + - "debian:9@prep" when: manual t_debian:testing: @@ -941,7 +941,7 @@ t_debian:testing: FDO_DISTRIBUTION_VERSION: 'testing' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG needs: - - "debian:testing@container-prep" + - "debian:testing@prep" when: manual t_centos:7.5.1804: @@ -954,7 +954,7 @@ t_centos:7.5.1804: FDO_DISTRIBUTION_VERSION: '7.5.1804' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:7.5.1804@container-prep" + - "centos:7.5.1804@prep" when: manual t_centos:7.9.2009: @@ -967,7 +967,7 @@ t_centos:7.9.2009: FDO_DISTRIBUTION_VERSION: '7.9.2009' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:7.9.2009@container-prep" + - "centos:7.9.2009@prep" when: manual t_centos:8.1.1911: @@ -980,7 +980,7 @@ t_centos:8.1.1911: FDO_DISTRIBUTION_VERSION: '8.1.1911' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:8.1.1911@container-prep" + - "centos:8.1.1911@prep" when: manual t_centos:8.3.2011: @@ -993,7 +993,7 @@ t_centos:8.3.2011: FDO_DISTRIBUTION_VERSION: '8.3.2011' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:8.3.2011@container-prep" + - "centos:8.3.2011@prep" when: manual t_centos:7.6.1810: @@ -1006,7 +1006,7 @@ t_centos:7.6.1810: FDO_DISTRIBUTION_VERSION: '7.6.1810' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:7.6.1810@container-prep" + - "centos:7.6.1810@prep" when: manual t_centos:7.7.1908: @@ -1019,7 +1019,7 @@ t_centos:7.7.1908: FDO_DISTRIBUTION_VERSION: '7.7.1908' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:7.7.1908@container-prep" + - "centos:7.7.1908@prep" when: manual t_centos:7.8.2003: @@ -1032,7 +1032,7 @@ t_centos:7.8.2003: FDO_DISTRIBUTION_VERSION: '7.8.2003' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:7.8.2003@container-prep" + - "centos:7.8.2003@prep" when: manual t_centos:8.2.2004: @@ -1045,7 +1045,7 @@ t_centos:8.2.2004: FDO_DISTRIBUTION_VERSION: '8.2.2004' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:8.2.2004@container-prep" + - "centos:8.2.2004@prep" when: manual t_alpine:latest: @@ -1058,7 +1058,7 @@ t_alpine:latest: FDO_DISTRIBUTION_VERSION: 'latest' FDO_DISTRIBUTION_TAG: $ALPINE_TAG needs: - - "alpine:latest@container-prep" + - "alpine:latest@prep" when: manual ################################################################# @@ -1074,7 +1074,7 @@ check-patch: FDO_DISTRIBUTION_VERSION: '37' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:37@container-prep" + - "fedora:37@prep" stage: tier1 script: - date '+%Y%m%d-%H%M%S'; NM_CHECKPATCH_FETCH_UPSTREAM=1 contrib/scripts/checkpatch-feature-branch.sh @@ -1087,7 +1087,7 @@ check-tree: FDO_DISTRIBUTION_VERSION: '37' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:37@container-prep" + - "fedora:37@prep" stage: tier1 script: - date '+%Y%m%d-%H%M%S'; contrib/scripts/nm-python-black-format.sh --check diff --git a/.gitlab-ci/ci.template b/.gitlab-ci/ci.template index b58d5c6c8..e2b2b0b1f 100644 --- a/.gitlab-ci/ci.template +++ b/.gitlab-ci/ci.template @@ -104,7 +104,7 @@ variables: {% for distro in distributions %} {% for version in distro.versions %} -{{distro.name}}:{{version}}@container-prep: +{{distro.name}}:{{version}}@prep: extends: - .fdo.container-build@{{distro.name}} stage: prep @@ -222,7 +222,7 @@ t_{{distro.name}}:{{version}}: {% endfor %} {% endif %} needs: - - "{{distro.name}}:{{version}}@container-prep" + - "{{distro.name}}:{{version}}@prep" {% if distro.tier > 1 %} when: manual {% endif %} @@ -242,7 +242,7 @@ check-patch: FDO_DISTRIBUTION_VERSION: '{{default_distro.versions[0]}}' FDO_DISTRIBUTION_TAG: ${{default_distro.name.upper()}}_TAG needs: - - "{{default_distro.name}}:{{default_distro.versions[0]}}@container-prep" + - "{{default_distro.name}}:{{default_distro.versions[0]}}@prep" stage: tier1 script: - date '+%Y%m%d-%H%M%S'; NM_CHECKPATCH_FETCH_UPSTREAM=1 contrib/scripts/checkpatch-feature-branch.sh @@ -255,7 +255,7 @@ check-tree: FDO_DISTRIBUTION_VERSION: '{{default_distro.versions[0]}}' FDO_DISTRIBUTION_TAG: ${{default_distro.name.upper()}}_TAG needs: - - "{{default_distro.name}}:{{default_distro.versions[0]}}@container-prep" + - "{{default_distro.name}}:{{default_distro.versions[0]}}@prep" stage: tier1 script: - date '+%Y%m%d-%H%M%S'; contrib/scripts/nm-python-black-format.sh --check From 8e37037e8830a40c97d6bb8992b3c4b9eb3fcc09 Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Thu, 13 Apr 2023 08:19:25 +0200 Subject: [PATCH 6/9] gitlab-ci: drop "tag"/"default_tag" from ci templates The tag we actually use already contains a hash of the input files and is generated (by `ci-fairy generate-templates`). There is no need for having this fixed prefix. As also seens by having a date there, which is maintained badly and meaningless. Drop it. --- .gitlab-ci.yml | 10 +++++----- .gitlab-ci/ci.template | 6 +++--- .gitlab-ci/config.yml | 16 +--------------- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6e5a487aa..36fc20458 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,11 +57,11 @@ variables: # # This is done by running `ci-fairy generate-template` and possibly bumping # ".default_tag". - ALPINE_TAG: '2023-04-11.0-63b57b80e21f' - CENTOS_TAG: '2023-04-11.0-b6ce4d55572b' - DEBIAN_TAG: '2023-04-11.0-de33b560b02a' - FEDORA_TAG: '2023-04-11.0-b6ce4d55572b' - UBUNTU_TAG: '2023-04-11.0-de33b560b02a' + ALPINE_TAG: 'tag-0a615b61395f' + CENTOS_TAG: 'tag-87fa88557da7' + DEBIAN_TAG: 'tag-2edf40356382' + FEDORA_TAG: 'tag-87fa88557da7' + UBUNTU_TAG: 'tag-2edf40356382' ALPINE_EXEC: 'bash .gitlab-ci/alpine-install.sh' CENTOS_EXEC: 'bash .gitlab-ci/fedora-install.sh' diff --git a/.gitlab-ci/ci.template b/.gitlab-ci/ci.template index e2b2b0b1f..521896ea0 100644 --- a/.gitlab-ci/ci.template +++ b/.gitlab-ci/ci.template @@ -17,10 +17,10 @@ .templates_sha: &template_sha ffe4d1b10aab7534489f0c4bbc4c5899df17d3f2 # see https://docs.gitlab.com/ee/ci/yaml/#includefile -{# Group distros by their common (name,base_type,tag) tuples.#} +{# Group distros by their common (name,base_type) tuples.#} {% set distro_groups = [] %} {% for distro in distributions %} - {% set g = {'name':distro.name,'base_type':distro.base_type,'tag':distro.tag} %} + {% set g = {'name':distro.name,'base_type':distro.base_type} %} {% if g not in distro_groups %} {% do distro_groups.append(g) %} {% endif %} @@ -62,7 +62,7 @@ variables: # This is done by running `ci-fairy generate-template` and possibly bumping # ".default_tag". {% for distro_group in distro_groups|sort(attribute='name') %} - {{"%-13s"| format(distro_group.name.upper() + '_TAG:')}}'{{distro_group.tag}}-{{ + {{"%-13s"| format(distro_group.name.upper() + '_TAG:')}}'tag-{{ (ci_fairy.hashfiles('./.gitlab-ci/config.yml', './.gitlab-ci/ci.template', './.gitlab-ci/' + distro_group.base_type + '-install.sh', diff --git a/.gitlab-ci/config.yml b/.gitlab-ci/config.yml index 7508fec74..91c7d59c1 100644 --- a/.gitlab-ci/config.yml +++ b/.gitlab-ci/config.yml @@ -7,29 +7,21 @@ # https://gitlab.freedesktop.org/freedesktop/ci-templates # -# We're happy to rebuild all containers when one changes. -.default_tag: &default_tag '2023-04-11.0' - - # The list of all distributions we want to create job for. -# The template generates manual jobs for all these. distributions: - name: fedora - tag: *default_tag base_type: fedora - # Tier1 is used to build the pages and check-{tree,patch} + # The first tier:1 in the list is used to build the pages and check-{tree,patch} tier: 1 versions: - '37' - name: fedora - tag: *default_tag base_type: fedora tier: 2 versions: - '36' - '38' - name: fedora - tag: *default_tag base_type: fedora tier: 3 versions: @@ -40,7 +32,6 @@ distributions: - '34' - '35' - name: ubuntu - tag: *default_tag base_type: debian tier: 2 versions: @@ -50,7 +41,6 @@ distributions: - 'devel' - 'rolling' - name: debian - tag: *default_tag base_type: debian tier: 2 versions: @@ -58,14 +48,12 @@ distributions: - '11' - 'sid' - name: debian - tag: *default_tag base_type: debian tier: 3 versions: - '9' - 'testing' - name: centos - tag: *default_tag base_type: fedora tier: 2 versions: @@ -74,7 +62,6 @@ distributions: - '8.1.1911' - '8.3.2011' - name: centos - tag: *default_tag base_type: fedora tier: 3 versions: @@ -83,7 +70,6 @@ distributions: - '7.8.2003' - '8.2.2004' - name: alpine - tag: *default_tag base_type: alpine tier: 2 versions: From afe098a9285b8304d9eaa126803bb419baa3e3eb Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Thu, 13 Apr 2023 08:34:16 +0200 Subject: [PATCH 7/9] gitlab-ci: extract base_type for distros to reduce redundant information The distro.name is not just a pretty name, its the name under which we fetch the container. It is thus a well-known name, that we can rely on. The "base_type" only depends on the distro name, and it makes no sense to ever choose a different name. Tracking it in the "distributions" array is thus redundant. Move the mapping of distro.name to the base type to a separate place. --- .gitlab-ci.yml | 10 +++++----- .gitlab-ci/ci.template | 10 +++++----- .gitlab-ci/config.yml | 18 +++++++++--------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36fc20458..fa99241d9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,11 +57,11 @@ variables: # # This is done by running `ci-fairy generate-template` and possibly bumping # ".default_tag". - ALPINE_TAG: 'tag-0a615b61395f' - CENTOS_TAG: 'tag-87fa88557da7' - DEBIAN_TAG: 'tag-2edf40356382' - FEDORA_TAG: 'tag-87fa88557da7' - UBUNTU_TAG: 'tag-2edf40356382' + ALPINE_TAG: 'tag-93cf50e3b143' + CENTOS_TAG: 'tag-43e358544423' + DEBIAN_TAG: 'tag-08e17e4980fb' + FEDORA_TAG: 'tag-43e358544423' + UBUNTU_TAG: 'tag-08e17e4980fb' ALPINE_EXEC: 'bash .gitlab-ci/alpine-install.sh' CENTOS_EXEC: 'bash .gitlab-ci/fedora-install.sh' diff --git a/.gitlab-ci/ci.template b/.gitlab-ci/ci.template index 521896ea0..da7a51f36 100644 --- a/.gitlab-ci/ci.template +++ b/.gitlab-ci/ci.template @@ -17,10 +17,10 @@ .templates_sha: &template_sha ffe4d1b10aab7534489f0c4bbc4c5899df17d3f2 # see https://docs.gitlab.com/ee/ci/yaml/#includefile -{# Group distros by their common (name,base_type) tuples.#} +{# Group distros by their common (name,) tuples.#} {% set distro_groups = [] %} {% for distro in distributions %} - {% set g = {'name':distro.name,'base_type':distro.base_type} %} + {% set g = {'name':distro.name} %} {% if g not in distro_groups %} {% do distro_groups.append(g) %} {% endif %} @@ -65,13 +65,13 @@ variables: {{"%-13s"| format(distro_group.name.upper() + '_TAG:')}}'tag-{{ (ci_fairy.hashfiles('./.gitlab-ci/config.yml', './.gitlab-ci/ci.template', - './.gitlab-ci/' + distro_group.base_type + '-install.sh', - './contrib/' + distro_group.base_type + '/REQUIRED_PACKAGES'))[0:12] + './.gitlab-ci/' + base_types[distro_group.name] + '-install.sh', + './contrib/' + base_types[distro_group.name] + '/REQUIRED_PACKAGES'))[0:12] }}' {% endfor %} {% for distro_group in distro_groups|sort(attribute='name') %} - {{"%-13s"| format(distro_group.name.upper() + '_EXEC:')}}'bash .gitlab-ci/{{distro_group.base_type}}-install.sh' + {{"%-13s"| format(distro_group.name.upper() + '_EXEC:')}}'bash .gitlab-ci/{{base_types[distro_group.name]}}-install.sh' {% endfor %} .nm_artifacts: diff --git a/.gitlab-ci/config.yml b/.gitlab-ci/config.yml index 91c7d59c1..8aafe0c48 100644 --- a/.gitlab-ci/config.yml +++ b/.gitlab-ci/config.yml @@ -7,22 +7,28 @@ # https://gitlab.freedesktop.org/freedesktop/ci-templates # +# Some distros are fairly similar, and we reuse similar scripts. +# The base type maps the distro name to their base. +base_types: + fedora: fedora + centos: fedora + debian: debian + ubuntu: debian + alpine: alpine + # The list of all distributions we want to create job for. distributions: - name: fedora - base_type: fedora # The first tier:1 in the list is used to build the pages and check-{tree,patch} tier: 1 versions: - '37' - name: fedora - base_type: fedora tier: 2 versions: - '36' - '38' - name: fedora - base_type: fedora tier: 3 versions: - '30' @@ -32,7 +38,6 @@ distributions: - '34' - '35' - name: ubuntu - base_type: debian tier: 2 versions: - '18.04' @@ -41,20 +46,17 @@ distributions: - 'devel' - 'rolling' - name: debian - base_type: debian tier: 2 versions: - '10' - '11' - 'sid' - name: debian - base_type: debian tier: 3 versions: - '9' - 'testing' - name: centos - base_type: fedora tier: 2 versions: - '7.5.1804' @@ -62,7 +64,6 @@ distributions: - '8.1.1911' - '8.3.2011' - name: centos - base_type: fedora tier: 3 versions: - '7.6.1810' @@ -70,7 +71,6 @@ distributions: - '7.8.2003' - '8.2.2004' - name: alpine - base_type: alpine tier: 2 versions: - 'latest' From 5475f57d397122f885f71cedf5c9890f187fc324 Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Thu, 13 Apr 2023 08:43:25 +0200 Subject: [PATCH 8/9] gitlab-ci: make tier tests automatic to simplify starting them manually We want that the tier2+ tests are only run manually. As those tests depend on the respective prep step, there are 3 possibilities: 1) make prep manual and the tier test automatic. That is what we would want, because then we can just manually trigger the prep step (one click). However, in the past this didn't work. 2) make the prep automatic and the test manual. That works, the downside is that we often run the prep step when its not needed. This is what we used to do to workaround 1). 3) make prep and the test manual. Then there are no unnecessary tests run, but triggering a manual test is cumbersome. First click to start the prep step, then wait, then click again. Revisit this. It seems 1) is working now. Yeay. Also rename the prep stages, so that it's clear to which tier they belong. I guess, I could move them instead to prep1, prep2, prep3 stages, but then there are a lot of columns on the web site. --- .gitlab-ci.yml | 180 ++++++++++++++++++++--------------------- .gitlab-ci/ci.template | 14 ++-- 2 files changed, 97 insertions(+), 97 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fa99241d9..437f1961e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,11 +57,11 @@ variables: # # This is done by running `ci-fairy generate-template` and possibly bumping # ".default_tag". - ALPINE_TAG: 'tag-93cf50e3b143' - CENTOS_TAG: 'tag-43e358544423' - DEBIAN_TAG: 'tag-08e17e4980fb' - FEDORA_TAG: 'tag-43e358544423' - UBUNTU_TAG: 'tag-08e17e4980fb' + ALPINE_TAG: 'tag-c6ac275362b2' + CENTOS_TAG: 'tag-80e000f258af' + DEBIAN_TAG: 'tag-f961edd7a574' + FEDORA_TAG: 'tag-80e000f258af' + UBUNTU_TAG: 'tag-f961edd7a574' ALPINE_EXEC: 'bash .gitlab-ci/alpine-install.sh' CENTOS_EXEC: 'bash .gitlab-ci/fedora-install.sh' @@ -97,7 +97,7 @@ variables: # Build a container for each distribution + version. The ci-templates # will re-use the containers if the tag doesn't change. -fedora:37@prep: +tier1:fedora:37@prep: extends: - .fdo.container-build@fedora stage: prep @@ -107,7 +107,7 @@ fedora:37@prep: FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC -fedora:36@prep: +tier2:fedora:36@prep: extends: - .fdo.container-build@fedora stage: prep @@ -116,8 +116,9 @@ fedora:36@prep: FDO_DISTRIBUTION_VERSION: '36' FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC + when: manual -fedora:38@prep: +tier2:fedora:38@prep: extends: - .fdo.container-build@fedora stage: prep @@ -126,8 +127,9 @@ fedora:38@prep: FDO_DISTRIBUTION_VERSION: '38' FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC + when: manual -fedora:30@prep: +tier3:fedora:30@prep: extends: - .fdo.container-build@fedora stage: prep @@ -136,8 +138,9 @@ fedora:30@prep: FDO_DISTRIBUTION_VERSION: '30' FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC + when: manual -fedora:31@prep: +tier3:fedora:31@prep: extends: - .fdo.container-build@fedora stage: prep @@ -146,8 +149,9 @@ fedora:31@prep: FDO_DISTRIBUTION_VERSION: '31' FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC + when: manual -fedora:32@prep: +tier3:fedora:32@prep: extends: - .fdo.container-build@fedora stage: prep @@ -156,8 +160,9 @@ fedora:32@prep: FDO_DISTRIBUTION_VERSION: '32' FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC + when: manual -fedora:33@prep: +tier3:fedora:33@prep: extends: - .fdo.container-build@fedora stage: prep @@ -166,8 +171,9 @@ fedora:33@prep: FDO_DISTRIBUTION_VERSION: '33' FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC + when: manual -fedora:34@prep: +tier3:fedora:34@prep: extends: - .fdo.container-build@fedora stage: prep @@ -176,8 +182,9 @@ fedora:34@prep: FDO_DISTRIBUTION_VERSION: '34' FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC + when: manual -fedora:35@prep: +tier3:fedora:35@prep: extends: - .fdo.container-build@fedora stage: prep @@ -186,8 +193,9 @@ fedora:35@prep: FDO_DISTRIBUTION_VERSION: '35' FDO_DISTRIBUTION_TAG: $FEDORA_TAG FDO_DISTRIBUTION_EXEC: $FEDORA_EXEC + when: manual -ubuntu:18.04@prep: +tier2:ubuntu:18.04@prep: extends: - .fdo.container-build@ubuntu stage: prep @@ -196,8 +204,9 @@ ubuntu:18.04@prep: FDO_DISTRIBUTION_VERSION: '18.04' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG FDO_DISTRIBUTION_EXEC: $UBUNTU_EXEC + when: manual -ubuntu:20.04@prep: +tier2:ubuntu:20.04@prep: extends: - .fdo.container-build@ubuntu stage: prep @@ -206,8 +215,9 @@ ubuntu:20.04@prep: FDO_DISTRIBUTION_VERSION: '20.04' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG FDO_DISTRIBUTION_EXEC: $UBUNTU_EXEC + when: manual -ubuntu:22.04@prep: +tier2:ubuntu:22.04@prep: extends: - .fdo.container-build@ubuntu stage: prep @@ -216,8 +226,9 @@ ubuntu:22.04@prep: FDO_DISTRIBUTION_VERSION: '22.04' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG FDO_DISTRIBUTION_EXEC: $UBUNTU_EXEC + when: manual -ubuntu:devel@prep: +tier2:ubuntu:devel@prep: extends: - .fdo.container-build@ubuntu stage: prep @@ -226,8 +237,9 @@ ubuntu:devel@prep: FDO_DISTRIBUTION_VERSION: 'devel' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG FDO_DISTRIBUTION_EXEC: $UBUNTU_EXEC + when: manual -ubuntu:rolling@prep: +tier2:ubuntu:rolling@prep: extends: - .fdo.container-build@ubuntu stage: prep @@ -236,8 +248,9 @@ ubuntu:rolling@prep: FDO_DISTRIBUTION_VERSION: 'rolling' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG FDO_DISTRIBUTION_EXEC: $UBUNTU_EXEC + when: manual -debian:10@prep: +tier2:debian:10@prep: extends: - .fdo.container-build@debian stage: prep @@ -246,8 +259,9 @@ debian:10@prep: FDO_DISTRIBUTION_VERSION: '10' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC + when: manual -debian:11@prep: +tier2:debian:11@prep: extends: - .fdo.container-build@debian stage: prep @@ -256,8 +270,9 @@ debian:11@prep: FDO_DISTRIBUTION_VERSION: '11' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC + when: manual -debian:sid@prep: +tier2:debian:sid@prep: extends: - .fdo.container-build@debian stage: prep @@ -266,8 +281,9 @@ debian:sid@prep: FDO_DISTRIBUTION_VERSION: 'sid' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC + when: manual -debian:9@prep: +tier3:debian:9@prep: extends: - .fdo.container-build@debian stage: prep @@ -276,8 +292,9 @@ debian:9@prep: FDO_DISTRIBUTION_VERSION: '9' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC + when: manual -debian:testing@prep: +tier3:debian:testing@prep: extends: - .fdo.container-build@debian stage: prep @@ -286,8 +303,9 @@ debian:testing@prep: FDO_DISTRIBUTION_VERSION: 'testing' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG FDO_DISTRIBUTION_EXEC: $DEBIAN_EXEC + when: manual -centos:7.5.1804@prep: +tier2:centos:7.5.1804@prep: extends: - .fdo.container-build@centos stage: prep @@ -296,8 +314,9 @@ centos:7.5.1804@prep: FDO_DISTRIBUTION_VERSION: '7.5.1804' FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC + when: manual -centos:7.9.2009@prep: +tier2:centos:7.9.2009@prep: extends: - .fdo.container-build@centos stage: prep @@ -306,8 +325,9 @@ centos:7.9.2009@prep: FDO_DISTRIBUTION_VERSION: '7.9.2009' FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC + when: manual -centos:8.1.1911@prep: +tier2:centos:8.1.1911@prep: extends: - .fdo.container-build@centos stage: prep @@ -316,8 +336,9 @@ centos:8.1.1911@prep: FDO_DISTRIBUTION_VERSION: '8.1.1911' FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC + when: manual -centos:8.3.2011@prep: +tier2:centos:8.3.2011@prep: extends: - .fdo.container-build@centos stage: prep @@ -326,8 +347,9 @@ centos:8.3.2011@prep: FDO_DISTRIBUTION_VERSION: '8.3.2011' FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC + when: manual -centos:7.6.1810@prep: +tier3:centos:7.6.1810@prep: extends: - .fdo.container-build@centos stage: prep @@ -336,8 +358,9 @@ centos:7.6.1810@prep: FDO_DISTRIBUTION_VERSION: '7.6.1810' FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC + when: manual -centos:7.7.1908@prep: +tier3:centos:7.7.1908@prep: extends: - .fdo.container-build@centos stage: prep @@ -346,8 +369,9 @@ centos:7.7.1908@prep: FDO_DISTRIBUTION_VERSION: '7.7.1908' FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC + when: manual -centos:7.8.2003@prep: +tier3:centos:7.8.2003@prep: extends: - .fdo.container-build@centos stage: prep @@ -356,8 +380,9 @@ centos:7.8.2003@prep: FDO_DISTRIBUTION_VERSION: '7.8.2003' FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC + when: manual -centos:8.2.2004@prep: +tier3:centos:8.2.2004@prep: extends: - .fdo.container-build@centos stage: prep @@ -366,8 +391,9 @@ centos:8.2.2004@prep: FDO_DISTRIBUTION_VERSION: '8.2.2004' FDO_DISTRIBUTION_TAG: $CENTOS_TAG FDO_DISTRIBUTION_EXEC: $CENTOS_EXEC + when: manual -alpine:latest@prep: +tier2:alpine:latest@prep: extends: - .fdo.container-build@alpine stage: prep @@ -376,6 +402,7 @@ alpine:latest@prep: FDO_DISTRIBUTION_VERSION: 'latest' FDO_DISTRIBUTION_TAG: $ALPINE_TAG FDO_DISTRIBUTION_EXEC: $ALPINE_EXEC + when: manual ################################################################# # # @@ -708,7 +735,7 @@ t_fedora:37: FDO_DISTRIBUTION_VERSION: '37' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:37@prep" + - "tier1:fedora:37@prep" t_fedora:36: extends: @@ -720,8 +747,7 @@ t_fedora:36: FDO_DISTRIBUTION_VERSION: '36' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:36@prep" - when: manual + - "tier2:fedora:36@prep" t_fedora:38: extends: @@ -733,8 +759,7 @@ t_fedora:38: FDO_DISTRIBUTION_VERSION: '38' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:38@prep" - when: manual + - "tier2:fedora:38@prep" t_fedora:30: extends: @@ -746,8 +771,7 @@ t_fedora:30: FDO_DISTRIBUTION_VERSION: '30' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:30@prep" - when: manual + - "tier3:fedora:30@prep" t_fedora:31: extends: @@ -759,8 +783,7 @@ t_fedora:31: FDO_DISTRIBUTION_VERSION: '31' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:31@prep" - when: manual + - "tier3:fedora:31@prep" t_fedora:32: extends: @@ -772,8 +795,7 @@ t_fedora:32: FDO_DISTRIBUTION_VERSION: '32' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:32@prep" - when: manual + - "tier3:fedora:32@prep" t_fedora:33: extends: @@ -785,8 +807,7 @@ t_fedora:33: FDO_DISTRIBUTION_VERSION: '33' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:33@prep" - when: manual + - "tier3:fedora:33@prep" t_fedora:34: extends: @@ -798,8 +819,7 @@ t_fedora:34: FDO_DISTRIBUTION_VERSION: '34' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:34@prep" - when: manual + - "tier3:fedora:34@prep" t_fedora:35: extends: @@ -811,8 +831,7 @@ t_fedora:35: FDO_DISTRIBUTION_VERSION: '35' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:35@prep" - when: manual + - "tier3:fedora:35@prep" t_ubuntu:18.04: extends: @@ -824,8 +843,7 @@ t_ubuntu:18.04: FDO_DISTRIBUTION_VERSION: '18.04' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG needs: - - "ubuntu:18.04@prep" - when: manual + - "tier2:ubuntu:18.04@prep" t_ubuntu:20.04: extends: @@ -837,8 +855,7 @@ t_ubuntu:20.04: FDO_DISTRIBUTION_VERSION: '20.04' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG needs: - - "ubuntu:20.04@prep" - when: manual + - "tier2:ubuntu:20.04@prep" t_ubuntu:22.04: extends: @@ -850,8 +867,7 @@ t_ubuntu:22.04: FDO_DISTRIBUTION_VERSION: '22.04' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG needs: - - "ubuntu:22.04@prep" - when: manual + - "tier2:ubuntu:22.04@prep" t_ubuntu:devel: extends: @@ -863,8 +879,7 @@ t_ubuntu:devel: FDO_DISTRIBUTION_VERSION: 'devel' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG needs: - - "ubuntu:devel@prep" - when: manual + - "tier2:ubuntu:devel@prep" t_ubuntu:rolling: extends: @@ -876,8 +891,7 @@ t_ubuntu:rolling: FDO_DISTRIBUTION_VERSION: 'rolling' FDO_DISTRIBUTION_TAG: $UBUNTU_TAG needs: - - "ubuntu:rolling@prep" - when: manual + - "tier2:ubuntu:rolling@prep" t_debian:10: extends: @@ -889,8 +903,7 @@ t_debian:10: FDO_DISTRIBUTION_VERSION: '10' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG needs: - - "debian:10@prep" - when: manual + - "tier2:debian:10@prep" t_debian:11: extends: @@ -902,8 +915,7 @@ t_debian:11: FDO_DISTRIBUTION_VERSION: '11' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG needs: - - "debian:11@prep" - when: manual + - "tier2:debian:11@prep" t_debian:sid: extends: @@ -915,8 +927,7 @@ t_debian:sid: FDO_DISTRIBUTION_VERSION: 'sid' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG needs: - - "debian:sid@prep" - when: manual + - "tier2:debian:sid@prep" t_debian:9: extends: @@ -928,8 +939,7 @@ t_debian:9: FDO_DISTRIBUTION_VERSION: '9' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG needs: - - "debian:9@prep" - when: manual + - "tier3:debian:9@prep" t_debian:testing: extends: @@ -941,8 +951,7 @@ t_debian:testing: FDO_DISTRIBUTION_VERSION: 'testing' FDO_DISTRIBUTION_TAG: $DEBIAN_TAG needs: - - "debian:testing@prep" - when: manual + - "tier3:debian:testing@prep" t_centos:7.5.1804: extends: @@ -954,8 +963,7 @@ t_centos:7.5.1804: FDO_DISTRIBUTION_VERSION: '7.5.1804' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:7.5.1804@prep" - when: manual + - "tier2:centos:7.5.1804@prep" t_centos:7.9.2009: extends: @@ -967,8 +975,7 @@ t_centos:7.9.2009: FDO_DISTRIBUTION_VERSION: '7.9.2009' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:7.9.2009@prep" - when: manual + - "tier2:centos:7.9.2009@prep" t_centos:8.1.1911: extends: @@ -980,8 +987,7 @@ t_centos:8.1.1911: FDO_DISTRIBUTION_VERSION: '8.1.1911' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:8.1.1911@prep" - when: manual + - "tier2:centos:8.1.1911@prep" t_centos:8.3.2011: extends: @@ -993,8 +999,7 @@ t_centos:8.3.2011: FDO_DISTRIBUTION_VERSION: '8.3.2011' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:8.3.2011@prep" - when: manual + - "tier2:centos:8.3.2011@prep" t_centos:7.6.1810: extends: @@ -1006,8 +1011,7 @@ t_centos:7.6.1810: FDO_DISTRIBUTION_VERSION: '7.6.1810' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:7.6.1810@prep" - when: manual + - "tier3:centos:7.6.1810@prep" t_centos:7.7.1908: extends: @@ -1019,8 +1023,7 @@ t_centos:7.7.1908: FDO_DISTRIBUTION_VERSION: '7.7.1908' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:7.7.1908@prep" - when: manual + - "tier3:centos:7.7.1908@prep" t_centos:7.8.2003: extends: @@ -1032,8 +1035,7 @@ t_centos:7.8.2003: FDO_DISTRIBUTION_VERSION: '7.8.2003' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:7.8.2003@prep" - when: manual + - "tier3:centos:7.8.2003@prep" t_centos:8.2.2004: extends: @@ -1045,8 +1047,7 @@ t_centos:8.2.2004: FDO_DISTRIBUTION_VERSION: '8.2.2004' FDO_DISTRIBUTION_TAG: $CENTOS_TAG needs: - - "centos:8.2.2004@prep" - when: manual + - "tier3:centos:8.2.2004@prep" t_alpine:latest: extends: @@ -1058,8 +1059,7 @@ t_alpine:latest: FDO_DISTRIBUTION_VERSION: 'latest' FDO_DISTRIBUTION_TAG: $ALPINE_TAG needs: - - "alpine:latest@prep" - when: manual + - "tier2:alpine:latest@prep" ################################################################# # # @@ -1074,7 +1074,7 @@ check-patch: FDO_DISTRIBUTION_VERSION: '37' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:37@prep" + - "tier1:fedora:37@prep" stage: tier1 script: - date '+%Y%m%d-%H%M%S'; NM_CHECKPATCH_FETCH_UPSTREAM=1 contrib/scripts/checkpatch-feature-branch.sh @@ -1087,7 +1087,7 @@ check-tree: FDO_DISTRIBUTION_VERSION: '37' FDO_DISTRIBUTION_TAG: $FEDORA_TAG needs: - - "fedora:37@prep" + - "tier1:fedora:37@prep" stage: tier1 script: - date '+%Y%m%d-%H%M%S'; contrib/scripts/nm-python-black-format.sh --check diff --git a/.gitlab-ci/ci.template b/.gitlab-ci/ci.template index da7a51f36..a62e9e58e 100644 --- a/.gitlab-ci/ci.template +++ b/.gitlab-ci/ci.template @@ -104,7 +104,7 @@ variables: {% for distro in distributions %} {% for version in distro.versions %} -{{distro.name}}:{{version}}@prep: +tier{{distro.tier}}:{{distro.name}}:{{version}}@prep: extends: - .fdo.container-build@{{distro.name}} stage: prep @@ -113,6 +113,9 @@ variables: FDO_DISTRIBUTION_VERSION: '{{version}}' FDO_DISTRIBUTION_TAG: ${{distro.name.upper()}}_TAG FDO_DISTRIBUTION_EXEC: ${{distro.name.upper()}}_EXEC +{% if distro.tier > 1 %} + when: manual +{% endif %} {% endfor %} {% endfor %} @@ -222,10 +225,7 @@ t_{{distro.name}}:{{version}}: {% endfor %} {% endif %} needs: - - "{{distro.name}}:{{version}}@prep" -{% if distro.tier > 1 %} - when: manual -{% endif %} + - "tier{{distro.tier}}:{{distro.name}}:{{version}}@prep" {% endfor %} {% endfor %} @@ -242,7 +242,7 @@ check-patch: FDO_DISTRIBUTION_VERSION: '{{default_distro.versions[0]}}' FDO_DISTRIBUTION_TAG: ${{default_distro.name.upper()}}_TAG needs: - - "{{default_distro.name}}:{{default_distro.versions[0]}}@prep" + - "tier{{default_distro.tier}}:{{default_distro.name}}:{{default_distro.versions[0]}}@prep" stage: tier1 script: - date '+%Y%m%d-%H%M%S'; NM_CHECKPATCH_FETCH_UPSTREAM=1 contrib/scripts/checkpatch-feature-branch.sh @@ -255,7 +255,7 @@ check-tree: FDO_DISTRIBUTION_VERSION: '{{default_distro.versions[0]}}' FDO_DISTRIBUTION_TAG: ${{default_distro.name.upper()}}_TAG needs: - - "{{default_distro.name}}:{{default_distro.versions[0]}}@prep" + - "tier{{default_distro.tier}}:{{default_distro.name}}:{{default_distro.versions[0]}}@prep" stage: tier1 script: - date '+%Y%m%d-%H%M%S'; contrib/scripts/nm-python-black-format.sh --check From 229163202dfc5e51b65c7c8e02b9df1880e1fb93 Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Wed, 12 Apr 2023 22:29:47 +0200 Subject: [PATCH 9/9] release.sh: adjust check for gitlab-ci for changes in pipeline - We need to fetch more entries per page. 100 is the maximum without pagination, but that is enough for us. - Previously, we checked all stages. Now, let's skip the "prep" and "tier3" stages. This change should work both with old and new pipelines. --- contrib/fedora/rpm/release.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/fedora/rpm/release.sh b/contrib/fedora/rpm/release.sh index f4ee4cb65..09e0417ce 100755 --- a/contrib/fedora/rpm/release.sh +++ b/contrib/fedora/rpm/release.sh @@ -131,7 +131,7 @@ check_gitlab_pipeline() { return 1 fi - PIPELINE_STATUSES="$(curl --no-progress-meter "https://gitlab.freedesktop.org/api/v4/projects/411/pipelines/$PIPELINE_ID/jobs" 2>/dev/null | jq '.[].status')" + PIPELINE_STATUSES="$(curl --no-progress-meter "https://gitlab.freedesktop.org/api/v4/projects/411/pipelines/$PIPELINE_ID/jobs?per_page=100" 2>/dev/null | jq '.[] | select(.stage!="prep" and .stage!="tier3") | .status')" if ! echo "$PIPELINE_STATUSES" | grep -q '^"success"$' ; then echo "Cannot find successful jobs for branch $BRANCH. Check \"https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/pipelines/$PIPELINE_ID\"" @@ -414,6 +414,7 @@ fi if [ $CHECK_GITLAB = 1 ]; then if ! check_gitlab_pipeline "$CUR_BRANCH" "$CUR_HEAD" ; then echo "Check the pipelines for branch \"$CUR_BRANCH\" at https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/pipelines?ref=$CUR_BRANCH" + echo "Wait for pipeline with \`ci-fairy wait-for-pipeline --project NetworkManager/NetworkManager --sha \"$CUR_HEAD\"\`" die "It seems not all gitlab-ci jobs were running/succeeding. Skip this check with --no-check-gitlab" fi fi