diff --git a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml index 2c3b4755894c..6347804e6856 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml @@ -312,6 +312,14 @@ services.endlessh-go. + + + Garage, + a simple object storage server for geodistributed deployments, + alternative to MinIO. Available as + services.garage. + + netbird, a zero diff --git a/nixos/doc/manual/release-notes/rl-2211.section.md b/nixos/doc/manual/release-notes/rl-2211.section.md index 6c07bf065190..6e95431b330a 100644 --- a/nixos/doc/manual/release-notes/rl-2211.section.md +++ b/nixos/doc/manual/release-notes/rl-2211.section.md @@ -108,6 +108,8 @@ In addition to numerous new and upgraded packages, this release has the followin - [endlessh-go](https://github.com/shizunge/endlessh-go), an SSH tarpit that exposes Prometheus metrics. Available as [services.endlessh-go](#opt-services.endlessh-go.enable). +- [Garage](https://garagehq.deuxfleurs.fr/), a simple object storage server for geodistributed deployments, alternative to MinIO. Available as [services.garage](#opt-services.garage.enable). + - [netbird](https://netbird.io), a zero configuration VPN. Available as [services.netbird](options.html#opt-services.netbird.enable). diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index f821a95c3bc8..ad0679d6165e 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1147,6 +1147,7 @@ ./services/web-servers/caddy/default.nix ./services/web-servers/darkhttpd.nix ./services/web-servers/fcgiwrap.nix + ./services/web-servers/garage.nix ./services/web-servers/hitch/default.nix ./services/web-servers/hydron.nix ./services/web-servers/jboss/default.nix diff --git a/nixos/modules/services/web-servers/garage.nix b/nixos/modules/services/web-servers/garage.nix new file mode 100644 index 000000000000..76ab273483eb --- /dev/null +++ b/nixos/modules/services/web-servers/garage.nix @@ -0,0 +1,91 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.garage; + toml = pkgs.formats.toml {}; + configFile = toml.generate "garage.toml" cfg.settings; +in +{ + meta.maintainers = [ maintainers.raitobezarius ]; + + options.services.garage = { + enable = mkEnableOption (lib.mdDoc "Garage Object Storage (S3 compatible)"); + + extraEnvironment = mkOption { + type = types.attrsOf types.str; + description = lib.mdDoc "Extra environment variables to pass to the Garage server."; + default = {}; + example = { RUST_BACKTRACE="yes"; }; + }; + + logLevel = mkOption { + type = types.enum (["info" "debug" "trace"]); + default = "info"; + example = "debug"; + description = lib.mdDoc "Garage log level, see for examples."; + }; + + settings = mkOption { + type = types.submodule { + freeformType = toml.type; + + options = { + metadata_dir = mkOption { + default = "/var/lib/garage/meta"; + type = types.path; + description = lib.mdDoc "The metadata directory, put this on a fast disk (e.g. SSD) if possible."; + }; + + data_dir = mkOption { + default = "/var/lib/garage/data"; + type = types.path; + description = lib.mdDoc "The main data storage, put this on your large storage (e.g. high capacity HDD)"; + }; + + replication_mode = mkOption { + default = "none"; + type = types.enum ([ "none" "1" "2" "3" 1 2 3 ]); + apply = v: toString v; + description = lib.mdDoc "Garage replication mode, defaults to none, see: for reference."; + }; + }; + }; + description = lib.mdDoc "Garage configuration, see for reference."; + }; + + package = mkOption { + default = pkgs.garage; + defaultText = literalExpression "pkgs.garage"; + type = types.package; + description = lib.mdDoc "Garage package to use."; + }; + }; + + config = mkIf cfg.enable { + environment.etc."garage.toml" = { + source = configFile; + }; + + environment.systemPackages = [ cfg.package ]; # For administration + + systemd.services.garage = { + description = "Garage Object Storage (S3 compatible)"; + after = [ "network.target" "network-online.target" ]; + wants = [ "network.target" "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + ExecStart = "${cfg.package}/bin/garage server"; + + StateDirectory = mkIf (hasPrefix "/var/lib/garage" cfg.settings.data_dir && hasPrefix "/var/lib/garage" cfg.settings.metadata_dir) "garage"; + DynamicUser = lib.mkDefault true; + ProtectHome = true; + NoNewPrivileges = true; + }; + environment = { + RUST_LOG = lib.mkDefault "garage=${cfg.logLevel}"; + } // cfg.extraEnvironment; + }; + }; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 8f404b2d1d65..df1304d94c69 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -214,6 +214,7 @@ in { fsck = handleTest ./fsck.nix {}; ft2-clone = handleTest ./ft2-clone.nix {}; mimir = handleTest ./mimir.nix {}; + garage = handleTest ./garage.nix {}; gerrit = handleTest ./gerrit.nix {}; geth = handleTest ./geth.nix {}; ghostunnel = handleTest ./ghostunnel.nix {}; diff --git a/nixos/tests/garage.nix b/nixos/tests/garage.nix new file mode 100644 index 000000000000..dc1f83e7f8f3 --- /dev/null +++ b/nixos/tests/garage.nix @@ -0,0 +1,169 @@ +import ./make-test-python.nix ({ pkgs, ...} : +let + mkNode = { replicationMode, publicV6Address ? "::1" }: { pkgs, ... }: { + networking.interfaces.eth1.ipv6.addresses = [{ + address = publicV6Address; + prefixLength = 64; + }]; + + networking.firewall.allowedTCPPorts = [ 3901 3902 ]; + + services.garage = { + enable = true; + settings = { + replication_mode = replicationMode; + + rpc_bind_addr = "[::]:3901"; + rpc_public_addr = "[${publicV6Address}]:3901"; + rpc_secret = "5c1915fa04d0b6739675c61bf5907eb0fe3d9c69850c83820f51b4d25d13868c"; + + s3_api = { + s3_region = "garage"; + api_bind_addr = "[::]:3900"; + root_domain = ".s3.garage"; + }; + + s3_web = { + bind_addr = "[::]:3902"; + root_domain = ".web.garage"; + index = "index.html"; + }; + }; + }; + environment.systemPackages = [ pkgs.minio-client ]; + + # Garage requires at least 1GiB of free disk space to run. + virtualisation.diskSize = 2 * 1024; + }; + + +in { + name = "garage"; + meta = { + maintainers = with pkgs.lib.maintainers; [ raitobezarius ]; + }; + + nodes = { + single_node = mkNode { replicationMode = "none"; }; + node1 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::1"; }; + node2 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::2"; }; + node3 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::3"; }; + node4 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::4"; }; + }; + + testScript = '' + from typing import List + from dataclasses import dataclass + import re + start_all() + + cur_version_regex = re.compile('Current cluster layout version: (?P\d*)') + key_creation_regex = re.compile('Key name: (?P.*)\nKey ID: (?P.*)\nSecret key: (?P.*)') + + @dataclass + class S3Key: + key_name: str + key_id: str + secret_key: str + + @dataclass + class GarageNode: + node_id: str + host: str + + def get_node_fqn(machine: Machine) -> GarageNode: + node_id, host = machine.succeed("garage node id").split('@') + return GarageNode(node_id=node_id, host=host) + + def get_node_id(machine: Machine) -> str: + return get_node_fqn(machine).node_id + + def get_layout_version(machine: Machine) -> int: + version_data = machine.succeed("garage layout show") + m = cur_version_regex.search(version_data) + if m and m.group('ver') is not None: + return int(m.group('ver')) + 1 + else: + raise ValueError('Cannot find current layout version') + + def apply_garage_layout(machine: Machine, layouts: List[str]): + for layout in layouts: + machine.succeed(f"garage layout assign {layout}") + version = get_layout_version(machine) + machine.succeed(f"garage layout apply --version {version}") + + def create_api_key(machine: Machine, key_name: str) -> S3Key: + output = machine.succeed(f"garage key new --name {key_name}") + m = key_creation_regex.match(output) + if not m or not m.group('key_id') or not m.group('secret_key'): + raise ValueError('Cannot parse API key data') + return S3Key(key_name=key_name, key_id=m.group('key_id'), secret_key=m.group('secret_key')) + + def get_api_key(machine: Machine, key_pattern: str) -> S3Key: + output = machine.succeed(f"garage key info {key_pattern}") + m = key_creation_regex.match(output) + if not m or not m.group('key_name') or not m.group('key_id') or not m.group('secret_key'): + raise ValueError('Cannot parse API key data') + return S3Key(key_name=m.group('key_name'), key_id=m.group('key_id'), secret_key=m.group('secret_key')) + + def test_bucket_writes(node): + node.succeed("garage bucket create test-bucket") + s3_key = create_api_key(node, "test-api-key") + node.succeed("garage bucket allow --read --write test-bucket --key test-api-key") + other_s3_key = get_api_key(node, 'test-api-key') + assert other_s3_key.secret_key == other_s3_key.secret_key + node.succeed( + f"mc alias set test-garage http://[::1]:3900 {s3_key.key_id} {s3_key.secret_key} --api S3v4" + ) + node.succeed("echo test | mc pipe test-garage/test-bucket/test.txt") + assert node.succeed("mc cat test-garage/test-bucket/test.txt").strip() == "test" + + def test_bucket_over_http(node, bucket='test-bucket', url=None): + if url is None: + url = f"{bucket}.web.garage" + + node.succeed(f'garage bucket website --allow {bucket}') + node.succeed(f'echo hello world | mc pipe test-garage/{bucket}/index.html') + assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world' + + with subtest("Garage works as a single-node S3 storage"): + single_node.wait_for_unit("garage.service") + single_node.wait_for_open_port(3900) + # Now Garage is initialized. + single_node_id = get_node_id(single_node) + apply_garage_layout(single_node, [f'-z qemutest -c 1 "{single_node_id}"']) + # Now Garage is operational. + test_bucket_writes(single_node) + test_bucket_over_http(single_node) + + with subtest("Garage works as a multi-node S3 storage"): + nodes = ('node1', 'node2', 'node3', 'node4') + rev_machines = {m.name: m for m in machines} + def get_machine(key): return rev_machines[key] + for key in nodes: + node = get_machine(key) + node.wait_for_unit("garage.service") + node.wait_for_open_port(3900) + + # Garage is initialized on all nodes. + node_ids = {key: get_node_fqn(get_machine(key)) for key in nodes} + + for key in nodes: + for other_key in nodes: + if other_key != key: + other_id = node_ids[other_key] + get_machine(key).succeed(f"garage node connect {other_id.node_id}@{other_id.host}") + + # Provide multiple zones for the nodes. + zones = ["nixcon", "nixcon", "paris_meetup", "fosdem"] + apply_garage_layout(node1, + [ + f'{ndata.node_id} -z {zones[index]} -c 1' + for index, ndata in enumerate(node_ids.values()) + ]) + # Now Garage is operational. + test_bucket_writes(node1) + for node in nodes: + test_bucket_over_http(get_machine(node)) + ''; +})