sourcehut: fix Unix socket support for Redis
This commit is contained in:
parent
2103e323e8
commit
cd6808ba9f
@ -83,7 +83,7 @@ let
|
||||
python = pkgs.sourcehut.python.withPackages (ps: with ps; [
|
||||
gunicorn
|
||||
eventlet
|
||||
# For monitoring Celery: sudo -u listssrht celery --app listssrht.process -b redis+socket:///run/redis-sourcehut/redis.sock?virtual_host=5 flower
|
||||
# For monitoring Celery: sudo -u listssrht celery --app listssrht.process -b redis+socket:///run/redis-sourcehut/redis.sock?virtual_host=1 flower
|
||||
flower
|
||||
# Sourcehut services
|
||||
srht
|
||||
|
@ -148,7 +148,7 @@ in
|
||||
redis = {
|
||||
host = mkOption {
|
||||
type = types.str;
|
||||
default = "unix:/run/redis-sourcehut-${srvsrht}/redis.sock?db=0";
|
||||
default = "unix:///run/redis-sourcehut-${srvsrht}/redis.sock?db=0";
|
||||
example = "redis://shared.wireguard:6379/0";
|
||||
description = ''
|
||||
The redis host URL. This is used for caching and temporary storage, and must
|
||||
|
@ -195,6 +195,7 @@ in
|
||||
# Testing metasrht
|
||||
machine.wait_for_unit("metasrht-api.service")
|
||||
machine.wait_for_unit("metasrht.service")
|
||||
machine.wait_for_unit("metasrht-webhooks.service")
|
||||
machine.wait_for_open_port(5000)
|
||||
machine.succeed("curl -sL http://localhost:5000 | grep meta.${domain}")
|
||||
machine.succeed("curl -sL http://meta.${domain} | grep meta.${domain}")
|
||||
|
@ -39,10 +39,6 @@ buildPythonPackage rec {
|
||||
inherit src version;
|
||||
pname = "buildsrht";
|
||||
|
||||
patches = [
|
||||
# Revert change breaking Unix socket support for Redis
|
||||
patches/redis-socket/build/0001-Revert-Add-build-submission-and-queue-monitoring.patch
|
||||
];
|
||||
postPatch = ''
|
||||
substituteInPlace Makefile \
|
||||
--replace "all: api worker" ""
|
||||
|
@ -48,6 +48,8 @@ buildPythonPackage rec {
|
||||
patches = [
|
||||
# Disable check for npm
|
||||
./disable-npm-install.patch
|
||||
# Fix Unix socket support in RedisQueueCollector
|
||||
patches/redis-socket/core/0001-Fix-Unix-socket-support-in-RedisQueueCollector.patch
|
||||
];
|
||||
|
||||
nativeBuildInputs = [
|
||||
|
@ -58,10 +58,6 @@ buildPythonPackage rec {
|
||||
inherit src version;
|
||||
pname = "gitsrht";
|
||||
|
||||
patches = [
|
||||
# Revert change breaking Unix socket support for Redis
|
||||
patches/redis-socket/git/0001-Revert-Add-webhook-queue-monitoring.patch
|
||||
];
|
||||
postPatch = ''
|
||||
substituteInPlace Makefile \
|
||||
--replace "all: api gitsrht-dispatch gitsrht-keys gitsrht-shell gitsrht-update-hook" ""
|
||||
|
@ -30,10 +30,6 @@ buildPythonPackage rec {
|
||||
vendorSha256 = "sha256-xnmMkRSokbhWD+kz0XQ9AinYdm6/50FRBISURPvlzD0=";
|
||||
} // import ./fix-gqlgen-trimpath.nix {inherit unzip;});
|
||||
|
||||
patches = [
|
||||
# Revert change breaking Unix socket support for Redis
|
||||
patches/redis-socket/lists/0001-Revert-Add-webhook-queue-monitoring.patch
|
||||
];
|
||||
postPatch = ''
|
||||
substituteInPlace Makefile \
|
||||
--replace "all: api" ""
|
||||
|
@ -40,10 +40,6 @@ buildPythonPackage rec {
|
||||
pname = "metasrht";
|
||||
inherit version src;
|
||||
|
||||
patches = [
|
||||
# Revert change breaking Unix socket support for Redis
|
||||
patches/redis-socket/meta/0001-Revert-Add-webhook-queue-monitoring.patch
|
||||
];
|
||||
postPatch = ''
|
||||
substituteInPlace Makefile \
|
||||
--replace "all: api" ""
|
||||
|
@ -1,69 +0,0 @@
|
||||
From 069b03f85847ed4a9223183b62ee53f420838911 Mon Sep 17 00:00:00 2001
|
||||
From: Julien Moutinho <julm+srht@sourcephile.fr>
|
||||
Date: Thu, 16 Dec 2021 04:54:24 +0100
|
||||
Subject: [PATCH builds.sr.ht] Revert "Add build submission and queue
|
||||
monitoring"
|
||||
|
||||
This reverts commit 690f1aa16c77e418dc40109cd5e8fdf4a7ed947a.
|
||||
|
||||
This has broken Unix socket support for Redis
|
||||
See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E
|
||||
---
|
||||
buildsrht/app.py | 3 ---
|
||||
buildsrht/runner.py | 9 +--------
|
||||
2 files changed, 1 insertion(+), 11 deletions(-)
|
||||
|
||||
diff --git a/buildsrht/app.py b/buildsrht/app.py
|
||||
index e5321a2..7c9977c 100644
|
||||
--- a/buildsrht/app.py
|
||||
+++ b/buildsrht/app.py
|
||||
@@ -36,9 +36,6 @@ class BuildApp(SrhtFlask):
|
||||
self.register_blueprint(secrets)
|
||||
self.register_blueprint(gql_blueprint)
|
||||
|
||||
- from buildsrht.runner import builds_queue_metrics_collector
|
||||
- self.metrics_registry.register(builds_queue_metrics_collector)
|
||||
-
|
||||
@self.context_processor
|
||||
def inject():
|
||||
return {
|
||||
diff --git a/buildsrht/runner.py b/buildsrht/runner.py
|
||||
index 7773452..0389c8e 100644
|
||||
--- a/buildsrht/runner.py
|
||||
+++ b/buildsrht/runner.py
|
||||
@@ -5,13 +5,10 @@ from srht.config import cfg
|
||||
from srht.database import db
|
||||
from srht.email import send_email
|
||||
from srht.oauth import UserType
|
||||
-from srht.metrics import RedisQueueCollector
|
||||
-from prometheus_client import Counter
|
||||
|
||||
allow_free = cfg("builds.sr.ht", "allow-free", default="no") == "yes"
|
||||
|
||||
-builds_broker = cfg("builds.sr.ht", "redis")
|
||||
-runner = Celery('builds', broker=builds_broker, config_source={
|
||||
+runner = Celery('builds', broker=cfg("builds.sr.ht", "redis"), config_source={
|
||||
"CELERY_TASK_SERIALIZER": "json",
|
||||
"CELERY_ACCEPT_CONTENT": ["json"],
|
||||
"CELERY_RESULT_SERIALIZER": "json",
|
||||
@@ -19,9 +16,6 @@ runner = Celery('builds', broker=builds_broker, config_source={
|
||||
"CELERY_TASK_PROTOCOL": 1
|
||||
})
|
||||
|
||||
-builds_queue_metrics_collector = RedisQueueCollector(builds_broker, "buildsrht_builds", "Number of builds currently in queue")
|
||||
-builds_submitted = Counter("buildsrht_builds_submited", "Number of builds submitted")
|
||||
-
|
||||
def queue_build(job, manifest):
|
||||
from buildsrht.types import JobStatus
|
||||
job.status = JobStatus.queued
|
||||
@@ -34,7 +28,6 @@ def queue_build(job, manifest):
|
||||
cfg("sr.ht", "owner-email"),
|
||||
"Cryptocurrency mining attempt on builds.sr.ht")
|
||||
else:
|
||||
- builds_submitted.inc()
|
||||
run_build.delay(job.id, manifest.to_dict())
|
||||
|
||||
def requires_payment(user):
|
||||
--
|
||||
2.34.0
|
||||
|
@ -0,0 +1,42 @@
|
||||
From 76dd636151735671be74ba9d55f773e190e22827 Mon Sep 17 00:00:00 2001
|
||||
From: Julien Moutinho <julm+srht@sourcephile.fr>
|
||||
Date: Fri, 13 May 2022 22:40:46 +0200
|
||||
Subject: [PATCH core.sr.ht] Fix Unix socket support in RedisQueueCollector
|
||||
|
||||
The broker URL is not necessarily in the format expected by Redis.from_url
|
||||
|
||||
Especially, Redis.from_url supports this format for Unix sockets:
|
||||
unix:///run/redis-sourcehut-metasrht/redis.sock?db=0
|
||||
See https://redis-py.readthedocs.io/en/stable/#redis.ConnectionPool.from_url
|
||||
|
||||
Whereas Celery+Kombu support Redis but also other transports
|
||||
and thus expect another scheme:
|
||||
redis+socket:///run/redis-sourcehut-metasrht/redis.sock?virtual_host=1
|
||||
See https://docs.celeryproject.org/en/stable/userguide/configuration.html#redis-backend-settings
|
||||
and https://github.com/celery/celery/blob/e5d99801e4b56a02af4a2e183879c767228d2817/celery/backends/redis.py#L299-L352
|
||||
and https://github.com/celery/kombu/blob/master/kombu/utils/url.py
|
||||
---
|
||||
srht/metrics.py | 3 ++-
|
||||
1 file changed, 2 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/srht/metrics.py b/srht/metrics.py
|
||||
index 68caf8e..2df5777 100644
|
||||
--- a/srht/metrics.py
|
||||
+++ b/srht/metrics.py
|
||||
@@ -1,11 +1,12 @@
|
||||
import time
|
||||
+from celery import Celery
|
||||
from prometheus_client.metrics_core import GaugeMetricFamily
|
||||
from redis import Redis, ResponseError
|
||||
|
||||
|
||||
class RedisQueueCollector:
|
||||
def __init__(self, broker, name, documentation, queue_name="celery"):
|
||||
- self.redis = Redis.from_url(broker)
|
||||
+ self.redis = Celery("collector", broker=broker).connection_for_read().channel().client
|
||||
self.queue_name = queue_name
|
||||
self.name = name
|
||||
self.documentation = documentation
|
||||
--
|
||||
2.35.1
|
||||
|
@ -1,33 +0,0 @@
|
||||
diff --git a/gitsrht/app.py b/gitsrht/app.py
|
||||
index eaae395..731bb03 100644
|
||||
--- a/gitsrht/app.py
|
||||
+++ b/gitsrht/app.py
|
||||
@@ -55,9 +55,6 @@ class GitApp(SrhtFlask):
|
||||
self.add_template_filter(url_quote)
|
||||
self.add_template_filter(commit_links)
|
||||
|
||||
- from gitsrht.webhooks import webhook_metrics_collector
|
||||
- self.metrics_registry.register(webhook_metrics_collector)
|
||||
-
|
||||
@self.context_processor
|
||||
def inject():
|
||||
notice = session.get("notice")
|
||||
diff --git a/gitsrht/webhooks.py b/gitsrht/webhooks.py
|
||||
index 36486d0..0109428 100644
|
||||
--- a/gitsrht/webhooks.py
|
||||
+++ b/gitsrht/webhooks.py
|
||||
@@ -7,12 +7,10 @@ if not hasattr(db, "session"):
|
||||
db.init()
|
||||
from srht.webhook import Event
|
||||
from srht.webhook.celery import CeleryWebhook, make_worker
|
||||
-from srht.metrics import RedisQueueCollector
|
||||
+from scmsrht.webhooks import UserWebhook
|
||||
import sqlalchemy as sa
|
||||
|
||||
-webhook_broker = cfg("git.sr.ht", "webhooks")
|
||||
-worker = make_worker(broker=webhook_broker)
|
||||
-webhook_metrics_collector = RedisQueueCollector(webhook_broker, "srht_webhooks", "Webhook queue length")
|
||||
+worker = make_worker(broker=cfg("git.sr.ht", "webhooks"))
|
||||
|
||||
class UserWebhook(CeleryWebhook):
|
||||
events = [
|
@ -1,48 +0,0 @@
|
||||
From 730e090f31b150d42be4b4722751f8e4610835b0 Mon Sep 17 00:00:00 2001
|
||||
From: Julien Moutinho <julm+srht@sourcephile.fr>
|
||||
Date: Sat, 12 Feb 2022 00:38:12 +0100
|
||||
Subject: [PATCH lists.sr.ht] Revert "Add webhook queue monitoring"
|
||||
|
||||
This reverts commit e74e344808e8d523a9786cefcbf64c9a247d7a0e.
|
||||
|
||||
Which has broken Unix socket support for Redis.
|
||||
See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E
|
||||
---
|
||||
listssrht/app.py | 3 ---
|
||||
listssrht/webhooks.py | 5 +----
|
||||
2 files changed, 1 insertion(+), 7 deletions(-)
|
||||
|
||||
diff --git a/listssrht/app.py b/listssrht/app.py
|
||||
index aec59f3..83a355d 100644
|
||||
--- a/listssrht/app.py
|
||||
+++ b/listssrht/app.py
|
||||
@@ -29,9 +29,6 @@ class ListsApp(SrhtFlask):
|
||||
self.register_blueprint(user)
|
||||
self.register_blueprint(gql_blueprint)
|
||||
|
||||
- from listssrht.webhooks import webhook_metrics_collector
|
||||
- self.metrics_registry.register(webhook_metrics_collector)
|
||||
-
|
||||
@self.context_processor
|
||||
def inject():
|
||||
from listssrht.types import ListAccess
|
||||
diff --git a/listssrht/webhooks.py b/listssrht/webhooks.py
|
||||
index ae5b1cb..86421ba 100644
|
||||
--- a/listssrht/webhooks.py
|
||||
+++ b/listssrht/webhooks.py
|
||||
@@ -8,11 +8,8 @@ if not hasattr(db, "session"):
|
||||
db.init()
|
||||
from srht.webhook import Event
|
||||
from srht.webhook.celery import CeleryWebhook, make_worker
|
||||
-from srht.metrics import RedisQueueCollector
|
||||
|
||||
-webhook_broker = cfg("lists.sr.ht", "webhooks")
|
||||
-worker = make_worker(broker=webhook_broker)
|
||||
-webhook_metrics_collector = RedisQueueCollector(webhook_broker, "srht_webhooks", "Webhook queue length")
|
||||
+worker = make_worker(broker=cfg("lists.sr.ht", "webhooks"))
|
||||
|
||||
class ListWebhook(CeleryWebhook):
|
||||
events = [
|
||||
--
|
||||
2.34.1
|
||||
|
@ -1,48 +0,0 @@
|
||||
From d88bee195797c6c294320617ff14798da94cd0f3 Mon Sep 17 00:00:00 2001
|
||||
From: Julien Moutinho <julm+srht@sourcephile.fr>
|
||||
Date: Thu, 16 Dec 2021 04:52:08 +0100
|
||||
Subject: [PATCH meta.sr.ht] Revert "Add webhook queue monitoring"
|
||||
|
||||
This reverts commit 9931df3c23094af5179df9ef019ca732b8125dac.
|
||||
|
||||
This has broken Unix socket support for Redis.
|
||||
See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E
|
||||
---
|
||||
metasrht/app.py | 3 ---
|
||||
metasrht/webhooks.py | 5 +----
|
||||
2 files changed, 1 insertion(+), 7 deletions(-)
|
||||
|
||||
diff --git a/metasrht/app.py b/metasrht/app.py
|
||||
index b190875..89c59bc 100644
|
||||
--- a/metasrht/app.py
|
||||
+++ b/metasrht/app.py
|
||||
@@ -49,9 +49,6 @@ class MetaApp(SrhtFlask):
|
||||
from metasrht.blueprints.billing import billing
|
||||
self.register_blueprint(billing)
|
||||
|
||||
- from metasrht.webhooks import webhook_metrics_collector
|
||||
- self.metrics_registry.register(webhook_metrics_collector)
|
||||
-
|
||||
@self.context_processor
|
||||
def inject():
|
||||
return {
|
||||
diff --git a/metasrht/webhooks.py b/metasrht/webhooks.py
|
||||
index 3e1149e..3f0ba01 100644
|
||||
--- a/metasrht/webhooks.py
|
||||
+++ b/metasrht/webhooks.py
|
||||
@@ -7,11 +7,8 @@ if not hasattr(db, "session"):
|
||||
db.init()
|
||||
from srht.webhook import Event
|
||||
from srht.webhook.celery import CeleryWebhook, make_worker
|
||||
-from srht.metrics import RedisQueueCollector
|
||||
|
||||
-webhook_broker = cfg("meta.sr.ht", "webhooks", "redis://")
|
||||
-worker = make_worker(broker=webhook_broker)
|
||||
-webhook_metrics_collector = RedisQueueCollector(webhook_broker, "srht_webhooks", "Webhook queue length")
|
||||
+worker = make_worker(broker=cfg("meta.sr.ht", "webhooks", "redis://"))
|
||||
|
||||
class UserWebhook(CeleryWebhook):
|
||||
events = [
|
||||
--
|
||||
2.34.0
|
||||
|
@ -1,50 +0,0 @@
|
||||
From 42a27ea60d8454552d54e1f51f1b976d1067fc32 Mon Sep 17 00:00:00 2001
|
||||
From: Julien Moutinho <julm+srht@sourcephile.fr>
|
||||
Date: Sat, 12 Feb 2022 00:30:29 +0100
|
||||
Subject: [PATCH todo.sr.ht] Revert "Add webhook queue monitoring"
|
||||
|
||||
This reverts commit 320a5e8f7cd16ca43928c36f0320593f84d986fa.
|
||||
|
||||
Which has broken Unix socket support for Redis.
|
||||
See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E
|
||||
---
|
||||
todosrht/flask.py | 3 ---
|
||||
todosrht/webhooks.py | 6 +-----
|
||||
2 files changed, 1 insertion(+), 8 deletions(-)
|
||||
|
||||
diff --git a/todosrht/flask.py b/todosrht/flask.py
|
||||
index 5e8ac66..9d0fd27 100644
|
||||
--- a/todosrht/flask.py
|
||||
+++ b/todosrht/flask.py
|
||||
@@ -43,9 +43,6 @@ class TodoApp(SrhtFlask):
|
||||
self.add_template_filter(urls.tracker_url)
|
||||
self.add_template_filter(urls.user_url)
|
||||
|
||||
- from todosrht.webhooks import webhook_metrics_collector
|
||||
- self.metrics_registry.register(webhook_metrics_collector)
|
||||
-
|
||||
@self.context_processor
|
||||
def inject():
|
||||
return {
|
||||
diff --git a/todosrht/webhooks.py b/todosrht/webhooks.py
|
||||
index eb8e08a..950047f 100644
|
||||
--- a/todosrht/webhooks.py
|
||||
+++ b/todosrht/webhooks.py
|
||||
@@ -7,13 +7,9 @@ if not hasattr(db, "session"):
|
||||
db.init()
|
||||
from srht.webhook import Event
|
||||
from srht.webhook.celery import CeleryWebhook, make_worker
|
||||
-from srht.metrics import RedisQueueCollector
|
||||
import sqlalchemy as sa
|
||||
|
||||
-
|
||||
-webhooks_broker = cfg("todo.sr.ht", "webhooks")
|
||||
-worker = make_worker(broker=webhooks_broker)
|
||||
-webhook_metrics_collector = RedisQueueCollector(webhooks_broker, "srht_webhooks", "Webhook queue length")
|
||||
+worker = make_worker(broker=cfg("todo.sr.ht", "webhooks"))
|
||||
|
||||
import todosrht.tracker_import
|
||||
|
||||
--
|
||||
2.34.1
|
||||
|
@ -23,16 +23,11 @@ buildPythonPackage rec {
|
||||
sha256 = "sha256-m7FY+jXpwPnK1+b1iQiDGe8JPfAFQp65BzGH6WvNwhM=";
|
||||
};
|
||||
|
||||
patches = [
|
||||
# Revert change breaking Unix socket support for Redis
|
||||
patches/redis-socket/todo/0001-Revert-Add-webhook-queue-monitoring.patch
|
||||
];
|
||||
postPatch = ''
|
||||
substituteInPlace Makefile \
|
||||
--replace "all: api" ""
|
||||
'';
|
||||
|
||||
|
||||
todosrht-api = buildGoModule ({
|
||||
inherit src version;
|
||||
pname = "todosrht-api";
|
||||
|
Loading…
Reference in New Issue
Block a user