colin/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sourcehut: fix Unix socket support for Redis

Browse Source
This commit is contained in:
Julien Moutinho 2022-05-13 00:53:16 +02:00 committed by tomberek
parent 2103e323e8
commit cd6808ba9f
15 changed files with 47 additions and 271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
nixos/modules/services/misc/sourcehut/default.nix
View File

@ -83,7 +83,7 @@ let
python = pkgs.sourcehut.python.withPackages (ps: with ps; [
gunicorn
eventlet
# For monitoring Celery: sudo -u listssrht celery --app listssrht.process -b redis+socket:///run/redis-sourcehut/redis.sock?virtual_host=5 flower
# For monitoring Celery: sudo -u listssrht celery --app listssrht.process -b redis+socket:///run/redis-sourcehut/redis.sock?virtual_host=1 flower
flower
# Sourcehut services
srht

2
nixos/modules/services/misc/sourcehut/service.nix
View File

@ -148,7 +148,7 @@ in
redis = {
host = mkOption {
type = types.str;
default = "unix:/run/redis-sourcehut-${srvsrht}/redis.sock?db=0";
default = "unix:///run/redis-sourcehut-${srvsrht}/redis.sock?db=0";
example = "redis://shared.wireguard:6379/0";
description = ''
The redis host URL. This is used for caching and temporary storage, and must

1
nixos/tests/sourcehut.nix
View File

@ -195,6 +195,7 @@ in
# Testing metasrht
machine.wait_for_unit("metasrht-api.service")
machine.wait_for_unit("metasrht.service")
machine.wait_for_unit("metasrht-webhooks.service")
machine.wait_for_open_port(5000)
machine.succeed("curl -sL http://localhost:5000 | grep meta.${domain}")
machine.succeed("curl -sL http://meta.${domain} | grep meta.${domain}")

4
pkgs/applications/version-management/sourcehut/builds.nix
View File

@ -39,10 +39,6 @@ buildPythonPackage rec {
inherit src version;
pname = "buildsrht";
patches = [
# Revert change breaking Unix socket support for Redis
patches/redis-socket/build/0001-Revert-Add-build-submission-and-queue-monitoring.patch
];
postPatch = ''
substituteInPlace Makefile \
--replace "all: api worker" ""

2
pkgs/applications/version-management/sourcehut/core.nix
View File

@ -48,6 +48,8 @@ buildPythonPackage rec {
patches = [
# Disable check for npm
./disable-npm-install.patch
# Fix Unix socket support in RedisQueueCollector
patches/redis-socket/core/0001-Fix-Unix-socket-support-in-RedisQueueCollector.patch
];
nativeBuildInputs = [

4
pkgs/applications/version-management/sourcehut/git.nix
View File

@ -58,10 +58,6 @@ buildPythonPackage rec {
inherit src version;
pname = "gitsrht";
patches = [
# Revert change breaking Unix socket support for Redis
patches/redis-socket/git/0001-Revert-Add-webhook-queue-monitoring.patch
];
postPatch = ''
substituteInPlace Makefile \
--replace "all: api gitsrht-dispatch gitsrht-keys gitsrht-shell gitsrht-update-hook" ""

4
pkgs/applications/version-management/sourcehut/lists.nix
View File

@ -30,10 +30,6 @@ buildPythonPackage rec {
vendorSha256 = "sha256-xnmMkRSokbhWD+kz0XQ9AinYdm6/50FRBISURPvlzD0=";
} // import ./fix-gqlgen-trimpath.nix {inherit unzip;});
patches = [
# Revert change breaking Unix socket support for Redis
patches/redis-socket/lists/0001-Revert-Add-webhook-queue-monitoring.patch
];
postPatch = ''
substituteInPlace Makefile \
--replace "all: api" ""

4
pkgs/applications/version-management/sourcehut/meta.nix
View File

@ -40,10 +40,6 @@ buildPythonPackage rec {
pname = "metasrht";
inherit version src;
patches = [
# Revert change breaking Unix socket support for Redis
patches/redis-socket/meta/0001-Revert-Add-webhook-queue-monitoring.patch
];
postPatch = ''
substituteInPlace Makefile \
--replace "all: api" ""

69
pkgs/applications/version-management/sourcehut/patches/redis-socket/build/0001-Revert-Add-build-submission-and-queue-monitoring.patch
View File

@ -1,69 +0,0 @@
From 069b03f85847ed4a9223183b62ee53f420838911 Mon Sep 17 00:00:00 2001
From: Julien Moutinho <julm+srht@sourcephile.fr>
Date: Thu, 16 Dec 2021 04:54:24 +0100
Subject: [PATCH builds.sr.ht] Revert "Add build submission and queue
monitoring"
This reverts commit 690f1aa16c77e418dc40109cd5e8fdf4a7ed947a.
This has broken Unix socket support for Redis
See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E
---
buildsrht/app.py | 3 ---
buildsrht/runner.py | 9 +--------
2 files changed, 1 insertion(+), 11 deletions(-)
diff --git a/buildsrht/app.py b/buildsrht/app.py
index e5321a2..7c9977c 100644
--- a/buildsrht/app.py
+++ b/buildsrht/app.py
@@ -36,9 +36,6 @@ class BuildApp(SrhtFlask):
self.register_blueprint(secrets)
self.register_blueprint(gql_blueprint)
- from buildsrht.runner import builds_queue_metrics_collector
- self.metrics_registry.register(builds_queue_metrics_collector)
-
@self.context_processor
def inject():
return {
diff --git a/buildsrht/runner.py b/buildsrht/runner.py
index 7773452..0389c8e 100644
--- a/buildsrht/runner.py
+++ b/buildsrht/runner.py
@@ -5,13 +5,10 @@ from srht.config import cfg
from srht.database import db
from srht.email import send_email
from srht.oauth import UserType
-from srht.metrics import RedisQueueCollector
-from prometheus_client import Counter
allow_free = cfg("builds.sr.ht", "allow-free", default="no") == "yes"
-builds_broker = cfg("builds.sr.ht", "redis")
-runner = Celery('builds', broker=builds_broker, config_source={
+runner = Celery('builds', broker=cfg("builds.sr.ht", "redis"), config_source={
"CELERY_TASK_SERIALIZER": "json",
"CELERY_ACCEPT_CONTENT": ["json"],
"CELERY_RESULT_SERIALIZER": "json",
@@ -19,9 +16,6 @@ runner = Celery('builds', broker=builds_broker, config_source={
"CELERY_TASK_PROTOCOL": 1
})
-builds_queue_metrics_collector = RedisQueueCollector(builds_broker, "buildsrht_builds", "Number of builds currently in queue")
-builds_submitted = Counter("buildsrht_builds_submited", "Number of builds submitted")
-
def queue_build(job, manifest):
from buildsrht.types import JobStatus
job.status = JobStatus.queued
@@ -34,7 +28,6 @@ def queue_build(job, manifest):
cfg("sr.ht", "owner-email"),
"Cryptocurrency mining attempt on builds.sr.ht")
else:
- builds_submitted.inc()
run_build.delay(job.id, manifest.to_dict())
def requires_payment(user):
--
2.34.0

42
pkgs/applications/version-management/sourcehut/patches/redis-socket/core/0001-Fix-Unix-socket-support-in-RedisQueueCollector.patch Normal file
View File

@ -0,0 +1,42 @@
From 76dd636151735671be74ba9d55f773e190e22827 Mon Sep 17 00:00:00 2001
From: Julien Moutinho <julm+srht@sourcephile.fr>
Date: Fri, 13 May 2022 22:40:46 +0200
Subject: [PATCH core.sr.ht] Fix Unix socket support in RedisQueueCollector
The broker URL is not necessarily in the format expected by Redis.from_url
Especially, Redis.from_url supports this format for Unix sockets:
unix:///run/redis-sourcehut-metasrht/redis.sock?db=0
See https://redis-py.readthedocs.io/en/stable/#redis.ConnectionPool.from_url
Whereas Celery+Kombu support Redis but also other transports
and thus expect another scheme:
redis+socket:///run/redis-sourcehut-metasrht/redis.sock?virtual_host=1
See https://docs.celeryproject.org/en/stable/userguide/configuration.html#redis-backend-settings
and https://github.com/celery/celery/blob/e5d99801e4b56a02af4a2e183879c767228d2817/celery/backends/redis.py#L299-L352
and https://github.com/celery/kombu/blob/master/kombu/utils/url.py
---
srht/metrics.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/srht/metrics.py b/srht/metrics.py
index 68caf8e..2df5777 100644
--- a/srht/metrics.py
+++ b/srht/metrics.py
@@ -1,11 +1,12 @@
import time
+from celery import Celery
from prometheus_client.metrics_core import GaugeMetricFamily
from redis import Redis, ResponseError
class RedisQueueCollector:
def __init__(self, broker, name, documentation, queue_name="celery"):
- self.redis = Redis.from_url(broker)
+ self.redis = Celery("collector", broker=broker).connection_for_read().channel().client
self.queue_name = queue_name
self.name = name
self.documentation = documentation
--
2.35.1

33
pkgs/applications/version-management/sourcehut/patches/redis-socket/git/0001-Revert-Add-webhook-queue-monitoring.patch
View File

@ -1,33 +0,0 @@
diff --git a/gitsrht/app.py b/gitsrht/app.py
index eaae395..731bb03 100644
--- a/gitsrht/app.py
+++ b/gitsrht/app.py
@@ -55,9 +55,6 @@ class GitApp(SrhtFlask):
self.add_template_filter(url_quote)
self.add_template_filter(commit_links)
- from gitsrht.webhooks import webhook_metrics_collector
- self.metrics_registry.register(webhook_metrics_collector)
-
@self.context_processor
def inject():
notice = session.get("notice")
diff --git a/gitsrht/webhooks.py b/gitsrht/webhooks.py
index 36486d0..0109428 100644
--- a/gitsrht/webhooks.py
+++ b/gitsrht/webhooks.py
@@ -7,12 +7,10 @@ if not hasattr(db, "session"):
db.init()
from srht.webhook import Event
from srht.webhook.celery import CeleryWebhook, make_worker
-from srht.metrics import RedisQueueCollector
+from scmsrht.webhooks import UserWebhook
import sqlalchemy as sa
-webhook_broker = cfg("git.sr.ht", "webhooks")
-worker = make_worker(broker=webhook_broker)
-webhook_metrics_collector = RedisQueueCollector(webhook_broker, "srht_webhooks", "Webhook queue length")
+worker = make_worker(broker=cfg("git.sr.ht", "webhooks"))
class UserWebhook(CeleryWebhook):
events = [

48
pkgs/applications/version-management/sourcehut/patches/redis-socket/lists/0001-Revert-Add-webhook-queue-monitoring.patch
View File

@ -1,48 +0,0 @@
From 730e090f31b150d42be4b4722751f8e4610835b0 Mon Sep 17 00:00:00 2001
From: Julien Moutinho <julm+srht@sourcephile.fr>
Date: Sat, 12 Feb 2022 00:38:12 +0100
Subject: [PATCH lists.sr.ht] Revert "Add webhook queue monitoring"
This reverts commit e74e344808e8d523a9786cefcbf64c9a247d7a0e.
Which has broken Unix socket support for Redis.
See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E
---
listssrht/app.py | 3 ---
listssrht/webhooks.py | 5 +----
2 files changed, 1 insertion(+), 7 deletions(-)
diff --git a/listssrht/app.py b/listssrht/app.py
index aec59f3..83a355d 100644
--- a/listssrht/app.py
+++ b/listssrht/app.py
@@ -29,9 +29,6 @@ class ListsApp(SrhtFlask):
self.register_blueprint(user)
self.register_blueprint(gql_blueprint)
- from listssrht.webhooks import webhook_metrics_collector
- self.metrics_registry.register(webhook_metrics_collector)
-
@self.context_processor
def inject():
from listssrht.types import ListAccess
diff --git a/listssrht/webhooks.py b/listssrht/webhooks.py
index ae5b1cb..86421ba 100644
--- a/listssrht/webhooks.py
+++ b/listssrht/webhooks.py
@@ -8,11 +8,8 @@ if not hasattr(db, "session"):
db.init()
from srht.webhook import Event
from srht.webhook.celery import CeleryWebhook, make_worker
-from srht.metrics import RedisQueueCollector
-webhook_broker = cfg("lists.sr.ht", "webhooks")
-worker = make_worker(broker=webhook_broker)
-webhook_metrics_collector = RedisQueueCollector(webhook_broker, "srht_webhooks", "Webhook queue length")
+worker = make_worker(broker=cfg("lists.sr.ht", "webhooks"))
class ListWebhook(CeleryWebhook):
events = [
--
2.34.1

48
pkgs/applications/version-management/sourcehut/patches/redis-socket/meta/0001-Revert-Add-webhook-queue-monitoring.patch
View File

@ -1,48 +0,0 @@
From d88bee195797c6c294320617ff14798da94cd0f3 Mon Sep 17 00:00:00 2001
From: Julien Moutinho <julm+srht@sourcephile.fr>
Date: Thu, 16 Dec 2021 04:52:08 +0100
Subject: [PATCH meta.sr.ht] Revert "Add webhook queue monitoring"
This reverts commit 9931df3c23094af5179df9ef019ca732b8125dac.
This has broken Unix socket support for Redis.
See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E
---
metasrht/app.py | 3 ---
metasrht/webhooks.py | 5 +----
2 files changed, 1 insertion(+), 7 deletions(-)
diff --git a/metasrht/app.py b/metasrht/app.py
index b190875..89c59bc 100644
--- a/metasrht/app.py
+++ b/metasrht/app.py
@@ -49,9 +49,6 @@ class MetaApp(SrhtFlask):
from metasrht.blueprints.billing import billing
self.register_blueprint(billing)
- from metasrht.webhooks import webhook_metrics_collector
- self.metrics_registry.register(webhook_metrics_collector)
-
@self.context_processor
def inject():
return {
diff --git a/metasrht/webhooks.py b/metasrht/webhooks.py
index 3e1149e..3f0ba01 100644
--- a/metasrht/webhooks.py
+++ b/metasrht/webhooks.py
@@ -7,11 +7,8 @@ if not hasattr(db, "session"):
db.init()
from srht.webhook import Event
from srht.webhook.celery import CeleryWebhook, make_worker
-from srht.metrics import RedisQueueCollector
-webhook_broker = cfg("meta.sr.ht", "webhooks", "redis://")
-worker = make_worker(broker=webhook_broker)
-webhook_metrics_collector = RedisQueueCollector(webhook_broker, "srht_webhooks", "Webhook queue length")
+worker = make_worker(broker=cfg("meta.sr.ht", "webhooks", "redis://"))
class UserWebhook(CeleryWebhook):
events = [
--
2.34.0

50
pkgs/applications/version-management/sourcehut/patches/redis-socket/todo/0001-Revert-Add-webhook-queue-monitoring.patch
View File

@ -1,50 +0,0 @@
From 42a27ea60d8454552d54e1f51f1b976d1067fc32 Mon Sep 17 00:00:00 2001
From: Julien Moutinho <julm+srht@sourcephile.fr>
Date: Sat, 12 Feb 2022 00:30:29 +0100
Subject: [PATCH todo.sr.ht] Revert "Add webhook queue monitoring"
This reverts commit 320a5e8f7cd16ca43928c36f0320593f84d986fa.
Which has broken Unix socket support for Redis.
See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E
---
todosrht/flask.py | 3 ---
todosrht/webhooks.py | 6 +-----
2 files changed, 1 insertion(+), 8 deletions(-)
diff --git a/todosrht/flask.py b/todosrht/flask.py
index 5e8ac66..9d0fd27 100644
--- a/todosrht/flask.py
+++ b/todosrht/flask.py
@@ -43,9 +43,6 @@ class TodoApp(SrhtFlask):
self.add_template_filter(urls.tracker_url)
self.add_template_filter(urls.user_url)
- from todosrht.webhooks import webhook_metrics_collector
- self.metrics_registry.register(webhook_metrics_collector)
-
@self.context_processor
def inject():
return {
diff --git a/todosrht/webhooks.py b/todosrht/webhooks.py
index eb8e08a..950047f 100644
--- a/todosrht/webhooks.py
+++ b/todosrht/webhooks.py
@@ -7,13 +7,9 @@ if not hasattr(db, "session"):
db.init()
from srht.webhook import Event
from srht.webhook.celery import CeleryWebhook, make_worker
-from srht.metrics import RedisQueueCollector
import sqlalchemy as sa
-
-webhooks_broker = cfg("todo.sr.ht", "webhooks")
-worker = make_worker(broker=webhooks_broker)
-webhook_metrics_collector = RedisQueueCollector(webhooks_broker, "srht_webhooks", "Webhook queue length")
+worker = make_worker(broker=cfg("todo.sr.ht", "webhooks"))
import todosrht.tracker_import
--
2.34.1

5
pkgs/applications/version-management/sourcehut/todo.nix
View File

@ -23,16 +23,11 @@ buildPythonPackage rec {
sha256 = "sha256-m7FY+jXpwPnK1+b1iQiDGe8JPfAFQp65BzGH6WvNwhM=";
};
patches = [
# Revert change breaking Unix socket support for Redis
patches/redis-socket/todo/0001-Revert-Add-webhook-queue-monitoring.patch
];
postPatch = ''
substituteInPlace Makefile \
--replace "all: api" ""
'';
todosrht-api = buildGoModule ({
inherit src version;
pname = "todosrht-api";