Files
NetworkManager/contrib/scripts/find-backports
Thomas Haller e9340c792c contrib: fail "find-backports" script if we have no "refs/notes/bugs" notes
"find-backports" script parses the commit messages to figure out which
patches to backport. We use "refs/notes/bugs" notes to extend the
meta data after the commit was merged. If you don't setup the
notes, the output is likely incomplete or wrong.

Yes, this is annoying. It requires you to setup the notes as described
in "CONTRIBUTING.md". Also because the "release.sh" script runs "find-backports",
so that means you cannot do releases without setting up the notes
(unless you manually disable running "find-backports"). But you really shouldn't
make a release based on incomplete information.
2022-04-06 13:23:37 +02:00

364 lines
9.7 KiB
Python
Executable File

#!/usr/bin/env python3
import subprocess
import collections
import os
import sys
import re
import pprint
FNULL = open(os.devnull, "w")
pp = pprint.PrettyPrinter(indent=4)
def print_err(s):
print(s, file=sys.stderr)
def die(s):
print_err(s)
sys.exit(1)
def memoize(f):
memo = {}
def helper(x):
if x not in memo:
memo[x] = f(x)
return memo[x]
return helper
def re_bin(r):
return r.encode("utf8")
def _keys_to_dict(itr):
d = collections.OrderedDict()
for c in itr:
d[c] = None
return d
@memoize
def git_ref_exists_plain(ref):
try:
subprocess.check_output(["git", "show-ref", "-q", "--verify", str(ref)])
except subprocess.CalledProcessError:
return False
return True
@memoize
def git_ref_exists(ref):
try:
out = subprocess.check_output(
["git", "rev-parse", "--verify", str(ref) + "^{commit}"], stderr=FNULL
)
except subprocess.CalledProcessError:
return None
try:
o = out.decode("ascii").strip()
if len(o) == 40:
return o
except Exception:
pass
raise Exception("git-rev-parse for '%s' returned unexpected output %s" % (ref, out))
@memoize
def git_get_head_name(ref):
out = subprocess.check_output(
["git", "rev-parse", "--symbolic-full-name", str(ref)], stderr=FNULL
)
return out.decode("utf-8").strip()
def git_merge_base(a, b):
out = subprocess.check_output(["git", "merge-base", str(a), str(b)], stderr=FNULL)
out = out.decode("ascii").strip()
assert git_ref_exists(out)
return out
def git_all_commits_grep(rnge, grep=None):
if grep:
grep = [("--grep=%s" % g) for g in grep]
notes = ["-c", "notes.displayref=refs/notes/bugs"]
else:
grep = []
notes = []
out = subprocess.check_output(
["git"]
+ notes
+ ["log", "--pretty=%H", "--notes", "--reverse"]
+ grep
+ [str(rnge)],
stderr=FNULL,
)
return list([x for x in out.decode("ascii").split("\n") if x])
def git_logg(commits):
commits = list(commits)
if not commits:
return ""
out = subprocess.check_output(
[
"git",
"log",
"--no-show-signature",
"--no-walk",
"--pretty=format:%Cred%h%Creset - %Cgreen(%ci)%Creset [%C(yellow)%an%Creset] %s%C(yellow)%d%Creset",
"--abbrev-commit",
"--date=local",
]
+ list([str(c) for c in commits]),
stderr=FNULL,
)
return out.decode("utf-8").strip()
@memoize
def git_all_commits(rnge):
return git_all_commits_grep(rnge)
def git_commit_sorted(commits):
commits = list(commits)
if not commits:
return []
out = subprocess.check_output(
["git", "log", "--no-walk", "--pretty=%H", "--reverse"]
+ list([str(x) for x in commits]),
stderr=FNULL,
)
out = out.decode("ascii")
return list([x for x in out.split("\n") if x])
@memoize
def git_ref_commit_body(ref):
return subprocess.check_output(
[
"git",
"-c",
"notes.displayref=refs/notes/bugs",
"log",
"-n1",
"--pretty=%B%n%N",
str(ref),
],
stderr=FNULL,
)
@memoize
def git_ref_commit_body_get_fixes(ref):
body = git_ref_commit_body(ref)
result = []
for mo in re.finditer(re_bin("\\b[fF]ixes: *([0-9a-z]+)\\b"), body):
c = mo.group(1).decode("ascii")
h = git_ref_exists(c)
if h:
result.append(h)
if result:
# The commit that contains a "Fixes:" line, can also contain an "Ignore-Fixes:" line
# to disable it. This only makes sense with refs/notes/bugs notes, to fix up a wrong
# annotation.
for mo in re.finditer(re_bin("\\bIgnore-[fF]ixes: *([0-9a-z]+)\\b"), body):
c = mo.group(1).decode("ascii")
h = git_ref_exists(c)
try:
result.remove(h)
except ValueError:
pass
return result
@memoize
def git_ref_commit_body_get_cherry_picked_one(ref):
ref = git_ref_exists(ref)
if not ref:
return None
body = git_ref_commit_body(ref)
result = None
for r in [
re_bin("\(cherry picked from commit ([0-9a-z]+)\)"),
re_bin("\\bIgnore-Backport: *([0-9a-z]+)\\b"),
]:
for mo in re.finditer(r, body):
c = mo.group(1).decode("ascii")
h = git_ref_exists(c)
if h:
if not result:
result = [h]
else:
result.append(h)
return result
@memoize
def git_ref_commit_body_get_cherry_picked_recurse(ref):
ref = git_ref_exists(ref)
if not ref:
return None
def do_recurse(result, ref):
result2 = git_ref_commit_body_get_cherry_picked_one(ref)
if result2:
extra = list([h2 for h2 in result2 if h2 not in result])
if extra:
result.extend(extra)
for h2 in extra:
do_recurse(result, h2)
result = []
do_recurse(result, ref)
return result
def git_commits_annotate_fixes(rnge):
commits = git_all_commits(rnge)
c_dict = _keys_to_dict(commits)
for c in git_all_commits_grep(rnge, grep=["[Ff]ixes:"]):
ff = git_ref_commit_body_get_fixes(c)
if ff:
c_dict[c] = ff
return c_dict
def git_commits_annotate_cherry_picked(rnge):
commits = git_all_commits(rnge)
c_dict = _keys_to_dict(commits)
for c in git_all_commits_grep(
ref_head, grep=["cherry picked from commit", "Ignore-Backport:"]
):
ff = git_ref_commit_body_get_cherry_picked_recurse(c)
if ff:
c_dict[c] = ff
return c_dict
if __name__ == "__main__":
if len(sys.argv) <= 1:
ref_head0 = "HEAD"
else:
ref_head0 = sys.argv[1]
ref_head = git_ref_exists(ref_head0)
if not ref_head:
die('Ref "%s" does not exist' % (ref_head0))
if not git_ref_exists_plain("refs/notes/bugs"):
die(
"Notes refs/notes/bugs not found. Read CONTRIBUTING.md file for how to setup the notes"
)
ref_upstreams = []
if len(sys.argv) <= 2:
head_name = git_get_head_name(ref_head0)
match = False
if head_name:
match = re.match("^refs/(heads|remotes/[^/]*)/nm-1-([0-9]+)$", head_name)
if match:
i = int(match.group(2))
while True:
i += 2
r = "nm-1-" + str(i)
if not git_ref_exists(r):
r = "refs/remotes/origin/nm-1-" + str(i)
if not git_ref_exists(r):
break
ref_upstreams.append(r)
ref_upstreams.append("main")
if not ref_upstreams:
if len(sys.argv) <= 2:
ref_upstreams = ["main"]
else:
ref_upstreams = list(sys.argv[2:])
for h in ref_upstreams:
if not git_ref_exists(h):
die('Upstream ref "%s" does not exist' % (h))
print_err("Check %s (%s)" % (ref_head0, ref_head))
print_err("Upstream refs: %s" % (ref_upstreams))
print_err('Check patches of "%s"...' % (ref_head))
own_commits_list = git_all_commits(ref_head)
own_commits_cherry_picked = git_commits_annotate_cherry_picked(ref_head)
cherry_picks_all = collections.OrderedDict()
for c, cherry_picked in own_commits_cherry_picked.items():
if cherry_picked:
for c2 in cherry_picked:
l = cherry_picks_all.get(c2)
if not l:
cherry_picks_all[c2] = [c]
else:
l.append(c)
own_commits_cherry_picked_flat = set()
for c, p in own_commits_cherry_picked.items():
own_commits_cherry_picked_flat.add(c)
if p:
own_commits_cherry_picked_flat.update(p)
# print(">>> own_commits_cherry_picked")
# pp.pprint(own_commits_cherry_picked)
# print(">>> cherry_picks_all")
# pp.pprint(cherry_picks_all)
# find all commits on the upstream branches that fix another commit.
fixing_commits = {}
for ref_upstream in ref_upstreams:
ref_str = ref_head + ".." + ref_upstream
print_err('Check upstream patches "%s"...' % (ref_str))
for c, fixes in git_commits_annotate_fixes(ref_str).items():
# print(">>> test %s ==> %s" % (c, fixes))
if not fixes:
# print(">>> test %s ==> SKIP" % (c))
continue
if c in cherry_picks_all:
# commit 'c' is already backported. Skip it.
# print(">>> in cherry_picks_all")
continue
for f in fixes:
if f not in own_commits_cherry_picked_flat:
# commit "c" fixes commit "f", but this is not one of our own commits
# and not interesting.
# print(">>> fixes %s not in own_commits_cherry_picked" % (f))
continue
# print(">>> take %s (fixes %s)" % (c, fixes))
fixing_commits[c] = fixes
break
extra = collections.OrderedDict(
[(c, git_ref_commit_body_get_cherry_picked_recurse(c)) for c in fixing_commits]
)
extra2 = []
for c in extra:
is_back = False
for e_v in extra.values():
if c in e_v:
is_back = True
break
if not is_back:
extra2.append(c)
commits_good = extra2
commits_good = git_commit_sorted(commits_good)
print_err(git_logg(commits_good))
for c in reversed(commits_good):
print("%s" % (c))