gitlab: merge branch 'th/gitlab-ci-tiers'

Obsoletes: https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/merge_requests/1595

https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/merge_requests/1601

.gitlab-ci/ci.template

@@ -17,17 +17,37 @@
 
 .templates_sha: &template_sha ffe4d1b10aab7534489f0c4bbc4c5899df17d3f2 # see https://docs.gitlab.com/ee/ci/yaml/#includefile
 
+{# Group distros by their common (name,) tuples.#}
+{% set distro_groups = [] %}
+{% for distro in distributions %}
+  {% set g = {'name':distro.name} %}
+  {% if g not in distro_groups %}
+  {%   do distro_groups.append(g) %}
+  {% endif %}
+{% endfor %}
+
+{# The "default_distro" builds our pages and is used for check-{tree,patch} tests. It is the first distro with tier 1. #}
+{% set default_distro = [] %}
+{% for distro in distributions %}
+  {% if distro.tier == 1 and default_distro|length == 0 %}
+    {% do default_distro.append(distro) %}
+  {% endif %}
+{% endfor %}
+{% set default_distro = default_distro[0] %}
+
 include:
-{% for distro in distributions|sort(attribute="name") %}
+{% for distro_group in distro_groups|sort(attribute='name') %}
-  # {{ distro.name.capitalize() }} container builder template
+  # {{ distro_group.name.capitalize() }} container builder template
   - project: 'freedesktop/ci-templates'
     ref: *template_sha
-    file: '/templates/{{distro.name}}.yml'
+    file: '/templates/{{distro_group.name}}.yml'
 {% endfor %}
 
 stages:
   - prep
-  - test
+  - tier1
+  - tier2
+  - tier3
   - deploy
   - triage
   - container_clean
@@ -41,17 +61,17 @@ variables:
   #
   # This is done by running `ci-fairy generate-template` and possibly bumping
   # ".default_tag".
-{% for distro in distributions %}
+{% for distro_group in distro_groups|sort(attribute='name') %}
-  {{"%-13s"| format(distro.name.upper() + '_TAG:')}}'{{distro.tag}}-{{
+  {{"%-13s"| format(distro_group.name.upper() + '_TAG:')}}'tag-{{
     (ci_fairy.hashfiles('./.gitlab-ci/config.yml',
                         './.gitlab-ci/ci.template',
-                        './.gitlab-ci/' + distro.base_type + '-install.sh',
+                        './.gitlab-ci/' + base_types[distro_group.name] + '-install.sh',
-                        './contrib/' + distro.base_type + '/REQUIRED_PACKAGES'))[0:12]
+                        './contrib/' + base_types[distro_group.name] + '/REQUIRED_PACKAGES'))[0:12]
   }}'
 {% endfor %}
 
-{% for distro in distributions %}
+{% for distro_group in distro_groups|sort(attribute='name') %}
-  {{"%-13s"| format(distro.name.upper() + '_EXEC:')}}'bash .gitlab-ci/{{distro.base_type}}-install.sh'
+  {{"%-13s"| format(distro_group.name.upper() + '_EXEC:')}}'bash .gitlab-ci/{{base_types[distro_group.name]}}-install.sh'
 {% endfor %}
 
 .nm_artifacts:
@@ -84,7 +104,7 @@ variables:
 {% for distro in distributions %}
 {% for version in distro.versions %}
 
-{{distro.name}}:{{version}}@container-prep:
+tier{{distro.tier}}:{{distro.name}}:{{version}}@prep:
   extends:
     - .fdo.container-build@{{distro.name}}
   stage: prep
@@ -93,6 +113,9 @@ variables:
     FDO_DISTRIBUTION_VERSION: '{{version}}'
     FDO_DISTRIBUTION_TAG: ${{distro.name.upper()}}_TAG
     FDO_DISTRIBUTION_EXEC: ${{distro.name.upper()}}_EXEC
+{% if distro.tier > 1 %}
+  when: manual
+{% endif %}
 {% endfor %}
 {% endfor %}
 
@@ -151,7 +174,6 @@ variables:
 #################################################################
 
 .build@template:
-  stage: test
   script:
     - env
     - r=0
@@ -174,11 +196,24 @@ t_{{distro.name}}:{{version}}:
   extends:
     - .build@template
     - .fdo.distribution-image@{{distro.name}}
-{% if distro.name == pages_build.name and
+{% if distro == default_distro %}
-      version == pages_build.version %}
     - .nm_artifacts
 {% else %}
     - .nm_artifacts_debug
+{% endif %}
+  stage: tier{{distro.tier}}
+{% if distro.tier <= 1 %}
+  parallel:
+    matrix:
+      - NM_TEST_SELECT_RUN:
+        - autotools+gcc+docs+valgrind
+        - meson+gcc+docs+valgrind
+        - autotools+clang
+        - meson+clang
+        - rpm+autotools
+        - rpm+meson
+        - tarball
+        - subtree
 {% endif %}
   variables:
     FDO_DISTRIBUTION_VERSION: '{{version}}'
@@ -190,10 +225,7 @@ t_{{distro.name}}:{{version}}:
     {% endfor %}
     {% endif %}
   needs:
-    - "{{distro.name}}:{{version}}@container-prep"
+    - "tier{{distro.tier}}:{{distro.name}}:{{version}}@prep"
-{% if not version in distro.get('always', []) and (distro.name != pages_build.name or version != pages_build.version) %}
-  when: manual
-{% endif %}
 {% endfor %}
 {% endfor %}
 
@@ -205,26 +237,26 @@ t_{{distro.name}}:{{version}}:
 
 check-patch:
   extends:
-    - .fdo.distribution-image@{{pages_build.name}}
+    - .fdo.distribution-image@{{default_distro.name}}
   variables:
-    FDO_DISTRIBUTION_VERSION: '{{pages_build.version}}'
+    FDO_DISTRIBUTION_VERSION: '{{default_distro.versions[0]}}'
-    FDO_DISTRIBUTION_TAG: ${{pages_build.name.upper()}}_TAG
+    FDO_DISTRIBUTION_TAG: ${{default_distro.name.upper()}}_TAG
   needs:
-    - "{{pages_build.name}}:{{pages_build.version}}@container-prep"
+    - "tier{{default_distro.tier}}:{{default_distro.name}}:{{default_distro.versions[0]}}@prep"
-  stage: test
+  stage: tier1
   script:
     - date '+%Y%m%d-%H%M%S'; NM_CHECKPATCH_FETCH_UPSTREAM=1 contrib/scripts/checkpatch-feature-branch.sh
   allow_failure: true
 
 check-tree:
   extends:
-    - .fdo.distribution-image@{{pages_build.name}}
+    - .fdo.distribution-image@{{default_distro.name}}
   variables:
-    FDO_DISTRIBUTION_VERSION: '{{pages_build.version}}'
+    FDO_DISTRIBUTION_VERSION: '{{default_distro.versions[0]}}'
-    FDO_DISTRIBUTION_TAG: ${{pages_build.name.upper()}}_TAG
+    FDO_DISTRIBUTION_TAG: ${{default_distro.name.upper()}}_TAG
   needs:
-    - "{{pages_build.name}}:{{pages_build.version}}@container-prep"
+    - "tier{{default_distro.tier}}:{{default_distro.name}}:{{default_distro.versions[0]}}@prep"
-  stage: test
+  stage: tier1
   script:
     - date '+%Y%m%d-%H%M%S'; contrib/scripts/nm-python-black-format.sh --check
     - date '+%Y%m%d-%H%M%S'; git ls-files -z -- 'po/*.po' | xargs -0 -n1 msgfmt -vc
@@ -242,9 +274,9 @@ pages:
   only:
     - main
   dependencies:
-    - t_{{pages_build.name}}:{{pages_build.version}}
+    - "t_{{default_distro.name}}:{{default_distro.versions[0]}}: [autotools+gcc+docs+valgrind]"
   needs:
-    - t_{{pages_build.name}}:{{pages_build.version}}
+    - "t_{{default_distro.name}}:{{default_distro.versions[0]}}: [autotools+gcc+docs+valgrind]"
 
 triage:issues:
   stage: triage

.gitlab-ci/config.yml

@@ -7,16 +7,29 @@
 # https://gitlab.freedesktop.org/freedesktop/ci-templates
 #
 
-# We're happy to rebuild all containers when one changes.
+# Some distros are fairly similar, and we reuse similar scripts.
-.default_tag: &default_tag '2023-01-18.0'
+# The base type maps the distro name to their base.
-
+base_types:
+  fedora: fedora
+  centos: fedora
+  debian: debian
+  ubuntu: debian
+  alpine: alpine
 
 # The list of all distributions we want to create job for.
-# The template generates manual jobs for all these.
 distributions:
   - name: fedora
-    tag: *default_tag
+    # The first tier:1 in the list is used to build the pages and check-{tree,patch}
-    base_type: fedora
+    tier: 1
+    versions:
+      - '37'
+  - name: fedora
+    tier: 2
+    versions:
+      - '36'
+      - '38'
+  - name: fedora
+    tier: 3
     versions:
       - '30'
       - '31'
@@ -24,12 +37,8 @@ distributions:
       - '33'
       - '34'
       - '35'
-      - '36'
-      - '37'
-      - '38'
   - name: ubuntu
-    tag: *default_tag
+    tier: 2
-    base_type: debian
     versions:
       - '18.04'
       - '20.04'
@@ -37,33 +46,31 @@ distributions:
       - 'devel'
       - 'rolling'
   - name: debian
-    tag: *default_tag
+    tier: 2
-    base_type: debian
     versions:
-      - '9'
       - '10'
       - '11'
-      - 'testing'
       - 'sid'
+  - name: debian
+    tier: 3
+    versions:
+      - '9'
+      - 'testing'
   - name: centos
-    tag: *default_tag
+    tier: 2
-    base_type: fedora
     versions:
       - '7.5.1804'
+      - '7.9.2009'
+      - '8.1.1911'
+      - '8.3.2011'
+  - name: centos
+    tier: 3
+    versions:
       - '7.6.1810'
       - '7.7.1908'
       - '7.8.2003'
-      - '7.9.2009'
-      - '8.1.1911'
       - '8.2.2004'
-      - '8.3.2011'
   - name: alpine
-    tag: *default_tag
+    tier: 2
-    base_type: alpine
     versions:
       - 'latest'
-
-# specifies which of the above distros is used as source for pages
-pages_build:
-  name: fedora
-  version: '37'

.gitlab-ci/run-test.sh

@@ -2,6 +2,11 @@
 
 set -ex
 
+die() {
+    printf "%s\n" "$*" >&2
+    exit 1
+}
+
 export PAGER=cat
 export OMP_NUM_THREADS=1
 
@@ -44,18 +49,58 @@ meson --version
 # to run that test as part of the build. Disable it.
 export NMTST_SKIP_CHECK_GITLAB_CI=1
 
+# Assert that "$1" is one of the valid values for NM_TEST_SELECT_RUN. die() otherwise.
+check_run_assert() {
+    { set +x; } 2>/dev/null
+    local run="$1"
+    local a
+
+    # These are the supported $NM_TEST_SELECT_RUN values.
+    local _CHECK_RUN_LIST=(
+        autotools+gcc+docs+valgrind
+        meson+gcc+docs+valgrind
+        autotools+clang
+        meson+clang
+        autotools+gcc+docs+el7+py2
+        rpm+autotools
+        rpm+meson
+        tarball
+        subtree
+
+        all
+        none
+    )
+
+    if [ "$run" = all ] ; then
+        set -x
+        return 0
+    fi
+
+    for a in "${_CHECK_RUN_LIST[@]}" ; do
+        if [ "$a" = "$run" ] ; then
+            set -x
+            return 0
+        fi
+    done
+    die "invalid NM_TEST_SELECT_RUN value \"$1\""
+}
+
+[ -z "$NM_TEST_SELECT_RUN" ] && NM_TEST_SELECT_RUN=all
+check_run_assert "$NM_TEST_SELECT_RUN"
+
 check_run() {
     local test_no="$1"
 
+    check_run_assert "$test_no"
+
     # Usually, we run the build several times. However, for testing
     # the build script manually, it can be useful to explicitly select
     # one step to run. For example, if step 3 is known to fail, you
-    # can still manually run step 4 by setting NM_TEST_SELECT_RUN=4.
+    # can still manually run step A by setting NM_TEST_SELECT_RUN=A.
 
-    test -z "$NM_TEST_SELECT_RUN" -o "$NM_TEST_SELECT_RUN" = "$test_no"
+    test "$NM_TEST_SELECT_RUN" = all -o "$NM_TEST_SELECT_RUN" = "$test_no"
 }
 
-
 check_run_clean() {
     if ! check_run "$1" ; then
         return 1
@@ -64,21 +109,21 @@ check_run_clean() {
     return 0
 }
 
-if check_run_clean 1 ; then
+if check_run_clean autotools+gcc+docs+valgrind ; then
     BUILD_TYPE=autotools CC=gcc WITH_DOCS=1 WITH_VALGRIND=1 contrib/scripts/nm-ci-run.sh
     mv build/INST/share/gtk-doc/html "$ARTIFACT_DIR/docs-html"
 fi
 
-check_run_clean 2 && BUILD_TYPE=meson     CC=gcc   WITH_DOCS=1 WITH_VALGRIND=1 contrib/scripts/nm-ci-run.sh
+check_run_clean meson+gcc+docs+valgrind && BUILD_TYPE=meson     CC=gcc   WITH_DOCS=1 WITH_VALGRIND=1 contrib/scripts/nm-ci-run.sh
-check_run_clean 3 && BUILD_TYPE=autotools CC=clang WITH_DOCS=0                 contrib/scripts/nm-ci-run.sh
+check_run_clean autotools+clang         && BUILD_TYPE=autotools CC=clang WITH_DOCS=0                 contrib/scripts/nm-ci-run.sh
-check_run_clean 4 && BUILD_TYPE=meson     CC=clang WITH_DOCS=0                 contrib/scripts/nm-ci-run.sh
+check_run_clean meson+clang             && BUILD_TYPE=meson     CC=clang WITH_DOCS=0                 contrib/scripts/nm-ci-run.sh
 
-check_run_clean 5 && test $IS_CENTOS_7 = 1 && PYTHON=python2 BUILD_TYPE=autotools CC=gcc WITH_DOCS=1 contrib/scripts/nm-ci-run.sh
+check_run_clean autotools+gcc+docs+el7+py2 && test $IS_CENTOS_7 = 1 && PYTHON=python2 BUILD_TYPE=autotools CC=gcc WITH_DOCS=1 contrib/scripts/nm-ci-run.sh
 
-check_run_clean 6 && test $IS_FEDORA = 1 -o $IS_CENTOS = 1 && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -W meson
+check_run_clean rpm+autotools && test $IS_FEDORA = 1 -o $IS_CENTOS = 1 && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -W meson
-check_run_clean 7 && test $IS_FEDORA = 1                   && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -w meson
+check_run_clean rpm+meson     && test $IS_FEDORA = 1                   && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -w meson
 
-if check_run_clean 8 && [ "$NM_BUILD_TARBALL" = 1 ]; then
+if check_run_clean tarball && [ "$NM_BUILD_TARBALL" = 1 ]; then
     SIGN_SOURCE=0 ./contrib/fedora/rpm/build_clean.sh -r
     mv ./NetworkManager-1*.tar.xz "$ARTIFACT_DIR/"
     mv ./contrib/fedora/rpm/latest/SRPMS/NetworkManager-1*.src.rpm "$ARTIFACT_DIR/"
@@ -110,7 +155,7 @@ test_subtree() {
     popd
 }
 
-if check_run_clean 10; then
+if check_run_clean subtree; then
     for d in c-list c-rbtree c-siphash c-stdaux n-acd n-dhcp4 ; do
         for cc in gcc clang; do
             test_subtree "$d" "$cc"
@@ -120,18 +165,16 @@ fi
 
 ###############################################################################
 
-if [ -z "$NM_TEST_SELECT_RUN" ] ; then
+if [ "$NM_BUILD_TARBALL" = 1 ]; then
     do_clean
-    if [ "$NM_BUILD_TARBALL" = 1 ]; then
+    if check_run autotools+gcc+docs+valgrind ; then
-        if check_run 1 ; then
+        mv "$ARTIFACT_DIR/docs-html/" ./
-            mv "$ARTIFACT_DIR/docs-html/" ./
+    fi
-        fi
+    if check_run tarball ; then
-        if check_run 8 ; then
+        mv \
-            mv \
+           "$ARTIFACT_DIR"/NetworkManager-1*.tar.xz \
-               "$ARTIFACT_DIR"/NetworkManager-1*.tar.xz \
+           "$ARTIFACT_DIR"/NetworkManager-1*.src.rpm \
-               "$ARTIFACT_DIR"/NetworkManager-1*.src.rpm \
+           ./
-               ./
-        fi
     fi
 fi
 

contrib/fedora/rpm/release.sh

@@ -131,7 +131,7 @@ check_gitlab_pipeline() {
         return 1
     fi
 
-    PIPELINE_STATUSES="$(curl --no-progress-meter "https://gitlab.freedesktop.org/api/v4/projects/411/pipelines/$PIPELINE_ID/jobs" 2>/dev/null | jq '.[].status')"
+    PIPELINE_STATUSES="$(curl --no-progress-meter "https://gitlab.freedesktop.org/api/v4/projects/411/pipelines/$PIPELINE_ID/jobs?per_page=100" 2>/dev/null | jq '.[] | select(.stage!="prep" and .stage!="tier3") | .status')"
 
     if ! echo "$PIPELINE_STATUSES" | grep -q '^"success"$' ; then
         echo "Cannot find successful jobs for branch $BRANCH. Check \"https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/pipelines/$PIPELINE_ID\""
@@ -414,6 +414,7 @@ fi
 if [ $CHECK_GITLAB = 1 ]; then
     if ! check_gitlab_pipeline "$CUR_BRANCH" "$CUR_HEAD" ; then
         echo "Check the pipelines for branch \"$CUR_BRANCH\" at https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/pipelines?ref=$CUR_BRANCH"
+        echo "Wait for pipeline with \`ci-fairy wait-for-pipeline --project NetworkManager/NetworkManager --sha \"$CUR_HEAD\"\`"
         die "It seems not all gitlab-ci jobs were running/succeeding. Skip this check with --no-check-gitlab"
     fi
 fi