colin/NetworkManager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

gitlab: merge branch 'th/gitlab-ci-tiers'

Browse Source 
Obsoletes: https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/merge_requests/1595

https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/merge_requests/1601
This commit is contained in:
Thomas Haller
2023-04-13 14:30:55 +02:00
parent 513c7f0280 229163202d
commit 31b5bd72eb
5 changed files with 617 additions and 492 deletions
Download Patch File Download Diff File Expand all files Collapse all files

864
.gitlab-ci.yml
View File

File diff suppressed because it is too large Load Diff

92
.gitlab-ci/ci.template
View File

@@ -17,17 +17,37 @@
.templates_sha: &template_sha ffe4d1b10aab7534489f0c4bbc4c5899df17d3f2 # see https://docs.gitlab.com/ee/ci/yaml/#includefile
{# Group distros by their common (name,) tuples.#}
{% set distro_groups = [] %}
{% for distro in distributions %}
{% set g = {'name':distro.name} %}
{% if g not in distro_groups %}
{% do distro_groups.append(g) %}
{% endif %}
{% endfor %}
{# The "default_distro" builds our pages and is used for check-{tree,patch} tests. It is the first distro with tier 1. #}
{% set default_distro = [] %}
{% for distro in distributions %}
{% if distro.tier == 1 and default_distro|length == 0 %}
{% do default_distro.append(distro) %}
{% endif %}
{% endfor %}
{% set default_distro = default_distro[0] %}
include:
{% for distro in distributions|sort(attribute="name") %}
# {{ distro.name.capitalize() }} container builder template
{% for distro_group in distro_groups|sort(attribute='name') %}
# {{ distro_group.name.capitalize() }} container builder template
- project: 'freedesktop/ci-templates'
ref: *template_sha
file: '/templates/{{distro.name}}.yml'
file: '/templates/{{distro_group.name}}.yml'
{% endfor %}
stages:
- prep
- test
- tier1
- tier2
- tier3
- deploy
- triage
- container_clean
@@ -41,17 +61,17 @@ variables:
#
# This is done by running `ci-fairy generate-template` and possibly bumping
# ".default_tag".
{% for distro in distributions %}
{{"%-13s"| format(distro.name.upper() + '_TAG:')}}'{{distro.tag}}-{{
{% for distro_group in distro_groups|sort(attribute='name') %}
{{"%-13s"| format(distro_group.name.upper() + '_TAG:')}}'tag-{{
(ci_fairy.hashfiles('./.gitlab-ci/config.yml',
'./.gitlab-ci/ci.template',
'./.gitlab-ci/' + distro.base_type + '-install.sh',
'./contrib/' + distro.base_type + '/REQUIRED_PACKAGES'))[0:12]
'./.gitlab-ci/' + base_types[distro_group.name] + '-install.sh',
'./contrib/' + base_types[distro_group.name] + '/REQUIRED_PACKAGES'))[0:12]
}}'
{% endfor %}
{% for distro in distributions %}
{{"%-13s"| format(distro.name.upper() + '_EXEC:')}}'bash .gitlab-ci/{{distro.base_type}}-install.sh'
{% for distro_group in distro_groups|sort(attribute='name') %}
{{"%-13s"| format(distro_group.name.upper() + '_EXEC:')}}'bash .gitlab-ci/{{base_types[distro_group.name]}}-install.sh'
{% endfor %}
.nm_artifacts:
@@ -84,7 +104,7 @@ variables:
{% for distro in distributions %}
{% for version in distro.versions %}
{{distro.name}}:{{version}}@container-prep:
tier{{distro.tier}}:{{distro.name}}:{{version}}@prep:
extends:
- .fdo.container-build@{{distro.name}}
stage: prep
@@ -93,6 +113,9 @@ variables:
FDO_DISTRIBUTION_VERSION: '{{version}}'
FDO_DISTRIBUTION_TAG: ${{distro.name.upper()}}_TAG
FDO_DISTRIBUTION_EXEC: ${{distro.name.upper()}}_EXEC
{% if distro.tier > 1 %}
when: manual
{% endif %}
{% endfor %}
{% endfor %}
@@ -151,7 +174,6 @@ variables:
#################################################################
.build@template:
stage: test
script:
- env
- r=0
@@ -174,11 +196,24 @@ t_{{distro.name}}:{{version}}:
extends:
- .build@template
- .fdo.distribution-image@{{distro.name}}
{% if distro.name == pages_build.name and
version == pages_build.version %}
{% if distro == default_distro %}
- .nm_artifacts
{% else %}
- .nm_artifacts_debug
{% endif %}
stage: tier{{distro.tier}}
{% if distro.tier <= 1 %}
parallel:
matrix:
- NM_TEST_SELECT_RUN:
- autotools+gcc+docs+valgrind
- meson+gcc+docs+valgrind
- autotools+clang
- meson+clang
- rpm+autotools
- rpm+meson
- tarball
- subtree
{% endif %}
variables:
FDO_DISTRIBUTION_VERSION: '{{version}}'
@@ -190,10 +225,7 @@ t_{{distro.name}}:{{version}}:
{% endfor %}
{% endif %}
needs:
- "{{distro.name}}:{{version}}@container-prep"
{% if not version in distro.get('always', []) and (distro.name != pages_build.name or version != pages_build.version) %}
when: manual
{% endif %}
- "tier{{distro.tier}}:{{distro.name}}:{{version}}@prep"
{% endfor %}
{% endfor %}
@@ -205,26 +237,26 @@ t_{{distro.name}}:{{version}}:
check-patch:
extends:
- .fdo.distribution-image@{{pages_build.name}}
- .fdo.distribution-image@{{default_distro.name}}
variables:
FDO_DISTRIBUTION_VERSION: '{{pages_build.version}}'
FDO_DISTRIBUTION_TAG: ${{pages_build.name.upper()}}_TAG
FDO_DISTRIBUTION_VERSION: '{{default_distro.versions[0]}}'
FDO_DISTRIBUTION_TAG: ${{default_distro.name.upper()}}_TAG
needs:
- "{{pages_build.name}}:{{pages_build.version}}@container-prep"
stage: test
- "tier{{default_distro.tier}}:{{default_distro.name}}:{{default_distro.versions[0]}}@prep"
stage: tier1
script:
- date '+%Y%m%d-%H%M%S'; NM_CHECKPATCH_FETCH_UPSTREAM=1 contrib/scripts/checkpatch-feature-branch.sh
allow_failure: true
check-tree:
extends:
- .fdo.distribution-image@{{pages_build.name}}
- .fdo.distribution-image@{{default_distro.name}}
variables:
FDO_DISTRIBUTION_VERSION: '{{pages_build.version}}'
FDO_DISTRIBUTION_TAG: ${{pages_build.name.upper()}}_TAG
FDO_DISTRIBUTION_VERSION: '{{default_distro.versions[0]}}'
FDO_DISTRIBUTION_TAG: ${{default_distro.name.upper()}}_TAG
needs:
- "{{pages_build.name}}:{{pages_build.version}}@container-prep"
stage: test
- "tier{{default_distro.tier}}:{{default_distro.name}}:{{default_distro.versions[0]}}@prep"
stage: tier1
script:
- date '+%Y%m%d-%H%M%S'; contrib/scripts/nm-python-black-format.sh --check
- date '+%Y%m%d-%H%M%S'; git ls-files -z -- 'po/*.po' | xargs -0 -n1 msgfmt -vc
@@ -242,9 +274,9 @@ pages:
only:
- main
dependencies:
- t_{{pages_build.name}}:{{pages_build.version}}
- "t_{{default_distro.name}}:{{default_distro.versions[0]}}: [autotools+gcc+docs+valgrind]"
needs:
- t_{{pages_build.name}}:{{pages_build.version}}
- "t_{{default_distro.name}}:{{default_distro.versions[0]}}: [autotools+gcc+docs+valgrind]"
triage:issues:
stage: triage

61
.gitlab-ci/config.yml
View File

@@ -7,16 +7,29 @@
# https://gitlab.freedesktop.org/freedesktop/ci-templates
#
# We're happy to rebuild all containers when one changes.
.default_tag: &default_tag '2023-01-18.0'
# Some distros are fairly similar, and we reuse similar scripts.
# The base type maps the distro name to their base.
base_types:
fedora: fedora
centos: fedora
debian: debian
ubuntu: debian
alpine: alpine
# The list of all distributions we want to create job for.
# The template generates manual jobs for all these.
distributions:
- name: fedora
tag: *default_tag
base_type: fedora
# The first tier:1 in the list is used to build the pages and check-{tree,patch}
tier: 1
versions:
- '37'
- name: fedora
tier: 2
versions:
- '36'
- '38'
- name: fedora
tier: 3
versions:
- '30'
- '31'
@@ -24,12 +37,8 @@ distributions:
- '33'
- '34'
- '35'
- '36'
- '37'
- '38'
- name: ubuntu
tag: *default_tag
base_type: debian
tier: 2
versions:
- '18.04'
- '20.04'
@@ -37,33 +46,31 @@ distributions:
- 'devel'
- 'rolling'
- name: debian
tag: *default_tag
base_type: debian
tier: 2
versions:
- '9'
- '10'
- '11'
- 'testing'
- 'sid'
- name: debian
tier: 3
versions:
- '9'
- 'testing'
- name: centos
tag: *default_tag
base_type: fedora
tier: 2
versions:
- '7.5.1804'
- '7.9.2009'
- '8.1.1911'
- '8.3.2011'
- name: centos
tier: 3
versions:
- '7.6.1810'
- '7.7.1908'
- '7.8.2003'
- '7.9.2009'
- '8.1.1911'
- '8.2.2004'
- '8.3.2011'
- name: alpine
tag: *default_tag
base_type: alpine
tier: 2
versions:
- 'latest'
# specifies which of the above distros is used as source for pages
pages_build:
name: fedora
version: '37'

77
.gitlab-ci/run-test.sh
View File

@@ -2,6 +2,11 @@
set -ex
die() {
printf "%s\n" "$*" >&2
exit 1
}
export PAGER=cat
export OMP_NUM_THREADS=1
@@ -44,18 +49,58 @@ meson --version
# to run that test as part of the build. Disable it.
export NMTST_SKIP_CHECK_GITLAB_CI=1
# Assert that "$1" is one of the valid values for NM_TEST_SELECT_RUN. die() otherwise.
check_run_assert() {
{ set +x; } 2>/dev/null
local run="$1"
local a
# These are the supported $NM_TEST_SELECT_RUN values.
local _CHECK_RUN_LIST=(
autotools+gcc+docs+valgrind
meson+gcc+docs+valgrind
autotools+clang
meson+clang
autotools+gcc+docs+el7+py2
rpm+autotools
rpm+meson
tarball
subtree
all
none
)
if [ "$run" = all ] ; then
set -x
return 0
fi
for a in "${_CHECK_RUN_LIST[@]}" ; do
if [ "$a" = "$run" ] ; then
set -x
return 0
fi
done
die "invalid NM_TEST_SELECT_RUN value \"$1\""
}
[ -z "$NM_TEST_SELECT_RUN" ] && NM_TEST_SELECT_RUN=all
check_run_assert "$NM_TEST_SELECT_RUN"
check_run() {
local test_no="$1"
check_run_assert "$test_no"
# Usually, we run the build several times. However, for testing
# the build script manually, it can be useful to explicitly select
# one step to run. For example, if step 3 is known to fail, you
# can still manually run step 4 by setting NM_TEST_SELECT_RUN=4.
# can still manually run step A by setting NM_TEST_SELECT_RUN=A.
test -z "$NM_TEST_SELECT_RUN" -o "$NM_TEST_SELECT_RUN" = "$test_no"
test "$NM_TEST_SELECT_RUN" = all -o "$NM_TEST_SELECT_RUN" = "$test_no"
}
check_run_clean() {
if ! check_run "$1" ; then
return 1
@@ -64,21 +109,21 @@ check_run_clean() {
return 0
}
if check_run_clean 1 ; then
if check_run_clean autotools+gcc+docs+valgrind ; then
BUILD_TYPE=autotools CC=gcc WITH_DOCS=1 WITH_VALGRIND=1 contrib/scripts/nm-ci-run.sh
mv build/INST/share/gtk-doc/html "$ARTIFACT_DIR/docs-html"
fi
check_run_clean 2 && BUILD_TYPE=meson CC=gcc WITH_DOCS=1 WITH_VALGRIND=1 contrib/scripts/nm-ci-run.sh
check_run_clean 3 && BUILD_TYPE=autotools CC=clang WITH_DOCS=0 contrib/scripts/nm-ci-run.sh
check_run_clean 4 && BUILD_TYPE=meson CC=clang WITH_DOCS=0 contrib/scripts/nm-ci-run.sh
check_run_clean meson+gcc+docs+valgrind && BUILD_TYPE=meson CC=gcc WITH_DOCS=1 WITH_VALGRIND=1 contrib/scripts/nm-ci-run.sh
check_run_clean autotools+clang && BUILD_TYPE=autotools CC=clang WITH_DOCS=0 contrib/scripts/nm-ci-run.sh
check_run_clean meson+clang && BUILD_TYPE=meson CC=clang WITH_DOCS=0 contrib/scripts/nm-ci-run.sh
check_run_clean 5 && test $IS_CENTOS_7 = 1 && PYTHON=python2 BUILD_TYPE=autotools CC=gcc WITH_DOCS=1 contrib/scripts/nm-ci-run.sh
check_run_clean autotools+gcc+docs+el7+py2 && test $IS_CENTOS_7 = 1 && PYTHON=python2 BUILD_TYPE=autotools CC=gcc WITH_DOCS=1 contrib/scripts/nm-ci-run.sh
check_run_clean 6 && test $IS_FEDORA = 1 -o $IS_CENTOS = 1 && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -W meson
check_run_clean 7 && test $IS_FEDORA = 1 && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -w meson
check_run_clean rpm+autotools && test $IS_FEDORA = 1 -o $IS_CENTOS = 1 && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -W meson
check_run_clean rpm+meson && test $IS_FEDORA = 1 && ./contrib/fedora/rpm/build_clean.sh -g -w crypto_gnutls -w debug -w iwd -w test -w meson
if check_run_clean 8 && [ "$NM_BUILD_TARBALL" = 1 ]; then
if check_run_clean tarball && [ "$NM_BUILD_TARBALL" = 1 ]; then
SIGN_SOURCE=0 ./contrib/fedora/rpm/build_clean.sh -r
mv ./NetworkManager-1*.tar.xz "$ARTIFACT_DIR/"
mv ./contrib/fedora/rpm/latest/SRPMS/NetworkManager-1*.src.rpm "$ARTIFACT_DIR/"
@@ -110,7 +155,7 @@ test_subtree() {
popd
}
if check_run_clean 10; then
if check_run_clean subtree; then
for d in c-list c-rbtree c-siphash c-stdaux n-acd n-dhcp4 ; do
for cc in gcc clang; do
test_subtree "$d" "$cc"
@@ -120,19 +165,17 @@ fi
###############################################################################
if [ -z "$NM_TEST_SELECT_RUN" ] ; then
do_clean
if [ "$NM_BUILD_TARBALL" = 1 ]; then
if check_run 1 ; then
do_clean
if check_run autotools+gcc+docs+valgrind ; then
mv "$ARTIFACT_DIR/docs-html/" ./
fi
if check_run 8 ; then
if check_run tarball ; then
mv \
"$ARTIFACT_DIR"/NetworkManager-1*.tar.xz \
"$ARTIFACT_DIR"/NetworkManager-1*.src.rpm \
./
fi
fi
fi
echo "BUILD SUCCESSFUL!!"

3
contrib/fedora/rpm/release.sh
View File

@@ -131,7 +131,7 @@ check_gitlab_pipeline() {
return 1
fi
PIPELINE_STATUSES="$(curl --no-progress-meter "https://gitlab.freedesktop.org/api/v4/projects/411/pipelines/$PIPELINE_ID/jobs" 2>/dev/null | jq '.[].status')"
PIPELINE_STATUSES="$(curl --no-progress-meter "https://gitlab.freedesktop.org/api/v4/projects/411/pipelines/$PIPELINE_ID/jobs?per_page=100" 2>/dev/null | jq '.[] | select(.stage!="prep" and .stage!="tier3") | .status')"
if ! echo "$PIPELINE_STATUSES" | grep -q '^"success"$' ; then
echo "Cannot find successful jobs for branch $BRANCH. Check \"https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/pipelines/$PIPELINE_ID\""
@@ -414,6 +414,7 @@ fi
if [ $CHECK_GITLAB = 1 ]; then
if ! check_gitlab_pipeline "$CUR_BRANCH" "$CUR_HEAD" ; then
echo "Check the pipelines for branch \"$CUR_BRANCH\" at https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/pipelines?ref=$CUR_BRANCH"
echo "Wait for pipeline with \`ci-fairy wait-for-pipeline --project NetworkManager/NetworkManager --sha \"$CUR_HEAD\"\`"
die "It seems not all gitlab-ci jobs were running/succeeding. Skip this check with --no-check-gitlab"
fi
fi