From 2554eba2ca05a1c3bbd7aaaa443b50b2a7ae4430 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Thu, 2 Nov 2023 04:17:03 +0100 Subject: [PATCH 1/6] formats.hocon: init --- pkgs/pkgs-lib/formats.nix | 2 + pkgs/pkgs-lib/formats/hocon/default.nix | 149 +++++++++++++ pkgs/pkgs-lib/formats/hocon/src/.gitignore | 1 + pkgs/pkgs-lib/formats/hocon/src/Cargo.lock | 89 ++++++++ pkgs/pkgs-lib/formats/hocon/src/Cargo.toml | 10 + pkgs/pkgs-lib/formats/hocon/src/src/main.rs | 226 ++++++++++++++++++++ pkgs/pkgs-lib/formats/hocon/update.sh | 4 + 7 files changed, 481 insertions(+) create mode 100644 pkgs/pkgs-lib/formats/hocon/default.nix create mode 100644 pkgs/pkgs-lib/formats/hocon/src/.gitignore create mode 100644 pkgs/pkgs-lib/formats/hocon/src/Cargo.lock create mode 100644 pkgs/pkgs-lib/formats/hocon/src/Cargo.toml create mode 100644 pkgs/pkgs-lib/formats/hocon/src/src/main.rs create mode 100755 pkgs/pkgs-lib/formats/hocon/update.sh diff --git a/pkgs/pkgs-lib/formats.nix b/pkgs/pkgs-lib/formats.nix index 950547c4f001..c78bd82e01ef 100644 --- a/pkgs/pkgs-lib/formats.nix +++ b/pkgs/pkgs-lib/formats.nix @@ -41,6 +41,8 @@ rec { libconfig = (import ./formats/libconfig/default.nix { inherit lib pkgs; }).format; + hocon = (import ./formats/hocon/default.nix { inherit lib pkgs; }).format; + json = {}: { type = with lib.types; let diff --git a/pkgs/pkgs-lib/formats/hocon/default.nix b/pkgs/pkgs-lib/formats/hocon/default.nix new file mode 100644 index 000000000000..d5b6308dea60 --- /dev/null +++ b/pkgs/pkgs-lib/formats/hocon/default.nix @@ -0,0 +1,149 @@ +{ lib +, pkgs +}: +let + inherit (pkgs) buildPackages callPackage; + + hocon-generator = buildPackages.rustPlatform.buildRustPackage { + name = "hocon-generator"; + version = "0.1.0"; + src = ./src; + + passthru.updateScript = ./update.sh; + + cargoLock.lockFile = ./src/Cargo.lock; + }; + + hocon-validator = pkgs.writers.writePython3Bin "hocon-validator" { + libraries = [ pkgs.python3Packages.pyhocon ]; + } '' + from sys import argv + from pyhocon import ConfigFactory + + if not len(argv) == 2: + print("USAGE: hocon-validator ") + + ConfigFactory.parse_file(argv[1]) + ''; +in +{ + # https://github.com/lightbend/config/blob/main/HOCON.md + format = { + generator ? hocon-generator + , validator ? hocon-validator + # `include classpath("")` is not implemented in pyhocon. + # In the case that you need this functionality, + # you will have to disable pyhocon validation. + , doCheck ? true + }: { + type = let + type' = with lib.types; let + atomType = nullOr (oneOf [ + bool + float + int + path + str + ]); + in (oneOf [ + atomType + (listOf atomType) + (attrsOf type') + ]) // { + description = "HOCON value"; + }; + in type'; + + lib = { + mkInclude = value: let + includeStatement = if lib.isAttrs value && !(lib.isDerivation value) then { + required = false; + type = null; + _type = "include"; + } // value else { + value = toString value; + required = false; + type = null; + _type = "include"; + }; + in + assert lib.assertMsg (lib.elem includeStatement.type [ "file" "url" "classpath" null ]) '' + Type of HOCON mkInclude is not of type 'file', 'url' or 'classpath': + ${(lib.generators.toPretty {}) includeStatement} + ''; + includeStatement; + + mkAppend = value: { + inherit value; + _type = "append"; + }; + + mkSubstitution = value: + if lib.isString value + then + { + inherit value; + optional = false; + _type = "substitution"; + } + else + assert lib.assertMsg (lib.isAttrs value) '' + Value of invalid type provided to `hocon.lib.mkSubstition`: ${lib.typeOf value} + ''; + assert lib.assertMsg (value ? "value") '' + Argument to `hocon.lib.mkSubstition` is missing a `value`: + ${builtins.toJSON value} + ''; + { + value = value.value; + optional = value.optional or false; + _type = "substitution"; + }; + }; + + generate = name: value: + callPackage + ({ + stdenvNoCC + , hocon-generator + , hocon-validator + , writeText + }: + stdenvNoCC.mkDerivation rec { + inherit name; + + dontUnpack = true; + + json = builtins.toJSON value; + passAsFile = [ "json" ]; + + strictDeps = true; + nativeBuildInputs = [ hocon-generator ]; + buildPhase = '' + runHook preBuild + hocon-generator < $jsonPath > output.conf + runHook postBuild + ''; + + inherit doCheck; + nativeCheckInputs = [ hocon-validator ]; + checkPhase = '' + runHook preCheck + hocon-validator output.conf + runHook postCheck + ''; + + installPhase = '' + runHook preInstall + mv output.conf $out + runHook postInstall + ''; + + passthru.json = writeText "${name}.json" json; + }) + { + hocon-generator = generator; + hocon-validator = validator; + }; + }; +} diff --git a/pkgs/pkgs-lib/formats/hocon/src/.gitignore b/pkgs/pkgs-lib/formats/hocon/src/.gitignore new file mode 100644 index 000000000000..eb5a316cbd19 --- /dev/null +++ b/pkgs/pkgs-lib/formats/hocon/src/.gitignore @@ -0,0 +1 @@ +target diff --git a/pkgs/pkgs-lib/formats/hocon/src/Cargo.lock b/pkgs/pkgs-lib/formats/hocon/src/Cargo.lock new file mode 100644 index 000000000000..735461cd5f0e --- /dev/null +++ b/pkgs/pkgs-lib/formats/hocon/src/Cargo.lock @@ -0,0 +1,89 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "hocon-generator" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "proc-macro2" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "serde" +version = "1.0.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "2.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" diff --git a/pkgs/pkgs-lib/formats/hocon/src/Cargo.toml b/pkgs/pkgs-lib/formats/hocon/src/Cargo.toml new file mode 100644 index 000000000000..e39e636a9f50 --- /dev/null +++ b/pkgs/pkgs-lib/formats/hocon/src/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "hocon-generator" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = "1.0.178" +serde_json = "1.0.104" diff --git a/pkgs/pkgs-lib/formats/hocon/src/src/main.rs b/pkgs/pkgs-lib/formats/hocon/src/src/main.rs new file mode 100644 index 000000000000..a564fc7dccdb --- /dev/null +++ b/pkgs/pkgs-lib/formats/hocon/src/src/main.rs @@ -0,0 +1,226 @@ +use serde_json::{value, Map, Value}; + +#[derive(Debug)] +enum HOCONValue { + Null, + Append(Box), + Bool(bool), + Number(value::Number), + String(String), + List(Vec), + Substitution(String, bool), + Object(Vec, Vec<(String, HOCONValue)>), +} + +#[derive(Debug)] +enum HOCONInclude { + Heuristic(String, bool), + Url(String, bool), + File(String, bool), + ClassPath(String, bool), +} + +impl HOCONInclude { + fn map_fst(&self, f: &dyn Fn(&String) -> String) -> HOCONInclude { + match self { + HOCONInclude::Heuristic(s, r) => HOCONInclude::Heuristic(f(s), *r), + HOCONInclude::Url(s, r) => HOCONInclude::Url(f(s), *r), + HOCONInclude::File(s, r) => HOCONInclude::File(f(s), *r), + HOCONInclude::ClassPath(s, r) => HOCONInclude::ClassPath(f(s), *r), + } + } +} + +fn parse_include(o: &Map) -> HOCONInclude { + let value = o + .get("value") + .expect("Missing field 'value' for include statement") + .as_str() + .expect("Field 'value' is not a string in include statement") + .to_string(); + let required = o + .get("required") + .expect("Missing field 'required' for include statement") + .as_bool() + .expect("Field 'required'is not a bool in include statement"); + let include_type = match o + .get("type") + .expect("Missing field 'type' for include statement") + { + Value::Null => None, + Value::String(s) => Some(s.as_str()), + t => panic!("Field 'type' is not a string in include statement: {:?}", t), + }; + + // Assert that this was an intentional include + debug_assert!(o.get("_type").and_then(|t| t.as_str()) == Some("include")); + + match include_type { + None => HOCONInclude::Heuristic(value, required), + Some("url") => HOCONInclude::Url(value, required), + Some("file") => HOCONInclude::File(value, required), + Some("classpath") => HOCONInclude::ClassPath(value, required), + _ => panic!( + "Could not recognize type for include statement: {}", + include_type.unwrap() + ), + } +} + +fn parse_special_types(o: &Map) -> Option { + o.get("_type") + .and_then(|r#type| r#type.as_str()) + .map(|r#type| match r#type { + "substitution" => { + let value = o + .get("value") + .expect("Missing value for substitution") + .as_str() + .unwrap_or_else(|| panic!("Substition value is not a string: {:?}", o)); + let required = o + .get("required") + .unwrap_or(&Value::Bool(false)) + .as_bool() + .unwrap_or_else(|| panic!("Substition value is not a string: {:?}", o)); + + debug_assert!(!value.contains('}')); + + HOCONValue::Substitution(value.to_string(), required) + } + "append" => { + let value = o.get("value").expect("Missing value for append"); + + HOCONValue::Append(Box::new(json_to_hocon(value))) + } + _ => panic!( + "\ + Attribute set contained special element '_type',\ + but its value is not recognized:\n{}", + r#type + ), + }) +} + +fn json_to_hocon(v: &Value) -> HOCONValue { + match v { + Value::Null => HOCONValue::Null, + Value::Bool(b) => HOCONValue::Bool(*b), + Value::Number(n) => HOCONValue::Number(n.clone()), + Value::String(s) => HOCONValue::String(s.clone()), + Value::Array(a) => { + let items = a.iter().map(json_to_hocon).collect::>(); + HOCONValue::List(items) + } + Value::Object(o) => { + if let Some(result) = parse_special_types(o) { + return result; + } + + let mut items = o + .iter() + .filter(|(key, _)| key.as_str() != "_includes") + .map(|(key, value)| (key.clone(), json_to_hocon(value))) + .collect::>(); + + items.sort_by(|(a, _), (b, _)| a.partial_cmp(b).unwrap()); + + let includes = o + .get("_includes") + .map(|x| { + x.as_array() + .expect("_includes is not an array") + .iter() + .map(|x| { + x.as_object() + .unwrap_or_else(|| panic!("Include is not an object: {}", x)) + }) + .map(parse_include) + .collect::>() + }) + .unwrap_or(vec![]); + + HOCONValue::Object(includes, items) + } + } +} + +impl ToString for HOCONValue { + fn to_string(&self) -> String { + match self { + HOCONValue::Null => "null".to_string(), + HOCONValue::Bool(b) => b.to_string(), + HOCONValue::Number(n) => n.to_string(), + HOCONValue::String(s) => serde_json::to_string(&Value::String(s.clone())).unwrap(), + HOCONValue::Substitution(v, required) => { + format!("${{{}{}}}", if *required { "" } else { "?" }, v) + } + HOCONValue::List(l) => { + let items = l + .iter() + .map(|item| item.to_string()) + .collect::>() + .join(",\n") + .split('\n') + .map(|s| " ".to_owned() + s) + .collect::>() + .join("\n"); + format!("[\n{}\n]", items) + } + HOCONValue::Object(i, o) => { + let includes = i + .iter() + .map(|x| { + x.map_fst(&|s| serde_json::to_string(&Value::String(s.clone())).unwrap()) + }) + .map(|x| match x { + HOCONInclude::Heuristic(s, r) => (s.to_string(), r), + HOCONInclude::Url(s, r) => (format!("url({})", s), r), + HOCONInclude::File(s, r) => (format!("file({})", s), r), + HOCONInclude::ClassPath(s, r) => (format!("classpath({})", s), r), + }) + .map(|(i, r)| if r { format!("required({})", i) } else { i }) + .map(|s| format!("include {}", s)) + .collect::>() + .join("\n"); + let items = o + .iter() + .map(|(key, value)| { + ( + serde_json::to_string(&Value::String(key.clone())).unwrap(), + value, + ) + }) + .map(|(key, value)| match value { + HOCONValue::Append(v) => format!("{} += {}", key, v.to_string()), + v => format!("{} = {}", key, v.to_string()), + }) + .collect::>() + .join("\n"); + + let content = (if includes.is_empty() { + items + } else { + format!("{}{}", includes, items) + }) + .split('\n') + .map(|s| format!(" {}", s)) + .collect::>() + .join("\n"); + + format!("{{\n{}\n}}", content) + } + HOCONValue::Append(_) => panic!("Append should not be present at this point"), + } + } +} + +fn main() { + let stdin = std::io::stdin().lock(); + let json = serde_json::Deserializer::from_reader(stdin) + .into_iter::() + .next() + .expect("Could not read content from stdin") + .expect("Could not parse JSON from stdin"); + + print!("{}\n\n", json_to_hocon(&json).to_string()); +} diff --git a/pkgs/pkgs-lib/formats/hocon/update.sh b/pkgs/pkgs-lib/formats/hocon/update.sh new file mode 100755 index 000000000000..ffc5ad3917f7 --- /dev/null +++ b/pkgs/pkgs-lib/formats/hocon/update.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env nix-shell +#!nix-shell -p cargo -i bash +cd "$(dirname "$0")" +cargo update From b6cdfec16ce7ce7c0d837b05ed3ad99aa6223647 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Thu, 2 Nov 2023 04:18:13 +0100 Subject: [PATCH 2/6] formats.hocon: add tests --- .../hocon/test/comprehensive/default.nix | 83 +++++++++++++++++++ .../hocon/test/comprehensive/expected.txt | 47 +++++++++++ pkgs/pkgs-lib/formats/hocon/test/default.nix | 4 + pkgs/pkgs-lib/tests/default.nix | 3 + 4 files changed, 137 insertions(+) create mode 100644 pkgs/pkgs-lib/formats/hocon/test/comprehensive/default.nix create mode 100644 pkgs/pkgs-lib/formats/hocon/test/comprehensive/expected.txt create mode 100644 pkgs/pkgs-lib/formats/hocon/test/default.nix diff --git a/pkgs/pkgs-lib/formats/hocon/test/comprehensive/default.nix b/pkgs/pkgs-lib/formats/hocon/test/comprehensive/default.nix new file mode 100644 index 000000000000..ae4fae443d41 --- /dev/null +++ b/pkgs/pkgs-lib/formats/hocon/test/comprehensive/default.nix @@ -0,0 +1,83 @@ +{ lib, formats, stdenvNoCC, writeText, ... }: +let + hocon = formats.hocon { }; + + include_file = (writeText "hocon-test-include.conf" '' + "val" = 1 + '').overrideAttrs (_: _: { + outputHashAlgo = "sha256"; + outputHashMode = "flat"; + outputHash = "sha256-UhkJLhT3bD6znq+IdDjs/ahP19mLzrLCy/R14pVrfew="; + }); + + expression = { + simple_top_level_attr = "1.0"; + nested.attrset.has.a.integer.value = 100; + some_floaty = 29.95; + + array2d = [ + [ 1 2 "a" ] + [ 2 1 "b" ] + ]; + nasty_string = "\"@\n\\\t^*\b\f\n\0\";'''$"; + + "misc attrs" = { + x = 1; + y = hocon.lib.mkAppend { a = 1; }; + }; + + "cursed \" .attrs \" " = { + "a" = 1; + "a b" = hocon.lib.mkSubstitution "a"; + "a b c" = hocon.lib.mkSubstitution { + value = "a b"; + required = false; + }; + }; + + to_include = { + _includes = [ + (hocon.lib.mkInclude include_file) + (hocon.lib.mkInclude "https://example.com") + (hocon.lib.mkInclude { + required = true; + type = "file"; + value = include_file; + }) + (hocon.lib.mkInclude { value = include_file; }) + (hocon.lib.mkInclude { + value = "https://example.com"; + type = "url"; + }) + ]; + }; + }; + + hocon-test-conf = hocon.generate "hocon-test.conf" expression; +in + stdenvNoCC.mkDerivation { + name = "pkgs.formats.hocon-test-comprehensive"; + + dontUnpack = true; + dontBuild = true; + + doCheck = true; + checkPhase = '' + runHook preCheck + + diff -U3 ${./expected.txt} ${hocon-test-conf} + + runHook postCheck + ''; + + installPhase = '' + runHook preInstall + + mkdir $out + cp ${./expected.txt} $out/expected.txt + cp ${hocon-test-conf} $out/hocon-test.conf + cp ${hocon-test-conf.passthru.json} $out/hocon-test.json + + runHook postInstall + ''; + } diff --git a/pkgs/pkgs-lib/formats/hocon/test/comprehensive/expected.txt b/pkgs/pkgs-lib/formats/hocon/test/comprehensive/expected.txt new file mode 100644 index 000000000000..ec196be4f686 --- /dev/null +++ b/pkgs/pkgs-lib/formats/hocon/test/comprehensive/expected.txt @@ -0,0 +1,47 @@ +{ + "array2d" = [ + [ + 1, + 2, + "a" + ], + [ + 2, + 1, + "b" + ] + ] + "cursed \" .attrs \" " = { + "a" = 1 + "a b" = ${?a} + "a b c" = ${?a b} + } + "misc attrs" = { + "x" = 1 + "y" += { + "a" = 1 + } + } + "nasty_string" = "\"@\n\\\t^*bf\n0\";'''$" + "nested" = { + "attrset" = { + "has" = { + "a" = { + "integer" = { + "value" = 100 + } + } + } + } + } + "simple_top_level_attr" = "1.0" + "some_floaty" = 29.95 + "to_include" = { + include "/nix/store/ccnzr53dpipdacxgci3ii3bqacvb5hxm-hocon-test-include.conf" + include "https://example.com" + include required(file("/nix/store/ccnzr53dpipdacxgci3ii3bqacvb5hxm-hocon-test-include.conf")) + include "/nix/store/ccnzr53dpipdacxgci3ii3bqacvb5hxm-hocon-test-include.conf" + include url("https://example.com") + } +} + diff --git a/pkgs/pkgs-lib/formats/hocon/test/default.nix b/pkgs/pkgs-lib/formats/hocon/test/default.nix new file mode 100644 index 000000000000..6cd03fe4854f --- /dev/null +++ b/pkgs/pkgs-lib/formats/hocon/test/default.nix @@ -0,0 +1,4 @@ +{ pkgs, ... }: +{ + comprehensive = pkgs.callPackage ./comprehensive { }; +} diff --git a/pkgs/pkgs-lib/tests/default.nix b/pkgs/pkgs-lib/tests/default.nix index 289780f57650..8e5e24301a29 100644 --- a/pkgs/pkgs-lib/tests/default.nix +++ b/pkgs/pkgs-lib/tests/default.nix @@ -17,7 +17,10 @@ let jdk11 = pkgs.callPackage ../formats/java-properties/test { jdk = pkgs.jdk11_headless; }; jdk17 = pkgs.callPackage ../formats/java-properties/test { jdk = pkgs.jdk17_headless; }; }; + libconfig = recurseIntoAttrs (import ../formats/libconfig/test { inherit pkgs; }); + + hocon = recurseIntoAttrs (import ../formats/hocon/test { inherit pkgs; }); }; flatten = prefix: as: From 39a779e269b012c721b34eee74f76afca3d03d7d Mon Sep 17 00:00:00 2001 From: h7x4 Date: Thu, 2 Nov 2023 04:48:06 +0100 Subject: [PATCH 3/6] treewide: use `formats.hocon` --- .../services/networking/jibri/default.nix | 15 ++--- nixos/modules/services/networking/jicofo.nix | 13 ++--- .../services/networking/jitsi-videobridge.nix | 15 +---- .../services/web-apps/suwayomi-server.nix | 55 ++----------------- 4 files changed, 17 insertions(+), 81 deletions(-) diff --git a/nixos/modules/services/networking/jibri/default.nix b/nixos/modules/services/networking/jibri/default.nix index a931831fc281..db2a17bd5590 100644 --- a/nixos/modules/services/networking/jibri/default.nix +++ b/nixos/modules/services/networking/jibri/default.nix @@ -5,12 +5,7 @@ with lib; let cfg = config.services.jibri; - # Copied from the jitsi-videobridge.nix file. - toHOCON = x: - if isAttrs x && x ? __hocon_envvar then ("\${" + x.__hocon_envvar + "}") - else if isAttrs x then "{${ concatStringsSep "," (mapAttrsToList (k: v: ''"${k}":${toHOCON v}'') x) }}" - else if isList x then "[${ concatMapStringsSep "," toHOCON x }]" - else builtins.toJSON x; + format = pkgs.formats.hocon { }; # We're passing passwords in environment variables that have names generated # from an attribute name, which may not be a valid bash identifier. @@ -38,13 +33,13 @@ let control-login = { domain = env.control.login.domain; username = env.control.login.username; - password.__hocon_envvar = toVarName "${name}_control"; + password = format.lib.mkSubstitution (toVarName "${name}_control"); }; call-login = { domain = env.call.login.domain; username = env.call.login.username; - password.__hocon_envvar = toVarName "${name}_call"; + password = format.lib.mkSubstitution (toVarName "${name}_call"); }; strip-from-room-domain = env.stripFromRoomDomain; @@ -85,13 +80,13 @@ let }; # Allow overriding leaves of the default config despite types.attrs not doing any merging. jibriConfig = recursiveUpdate defaultJibriConfig cfg.config; - configFile = pkgs.writeText "jibri.conf" (toHOCON { jibri = jibriConfig; }); + configFile = format.generate "jibri.conf" { jibri = jibriConfig; }; in { options.services.jibri = with types; { enable = mkEnableOption (lib.mdDoc "Jitsi BRoadcasting Infrastructure. Currently Jibri must be run on a host that is also running {option}`services.jitsi-meet.enable`, so for most use cases it will be simpler to run {option}`services.jitsi-meet.jibri.enable`"); config = mkOption { - type = attrs; + type = format.type; default = { }; description = lib.mdDoc '' Jibri configuration. diff --git a/nixos/modules/services/networking/jicofo.nix b/nixos/modules/services/networking/jicofo.nix index 0886bbe004c4..380344c8eaa1 100644 --- a/nixos/modules/services/networking/jicofo.nix +++ b/nixos/modules/services/networking/jicofo.nix @@ -5,14 +5,9 @@ with lib; let cfg = config.services.jicofo; - # HOCON is a JSON superset that some jitsi-meet components use for configuration - toHOCON = x: if isAttrs x && x ? __hocon_envvar then ("\${" + x.__hocon_envvar + "}") - else if isAttrs x && x ? __hocon_unquoted_string then x.__hocon_unquoted_string - else if isAttrs x then "{${ concatStringsSep "," (mapAttrsToList (k: v: ''"${k}":${toHOCON v}'') x) }}" - else if isList x then "[${ concatMapStringsSep "," toHOCON x }]" - else builtins.toJSON x; + format = pkgs.formats.hocon { }; - configFile = pkgs.writeText "jicofo.conf" (toHOCON cfg.config); + configFile = format.generate "jicofo.conf" cfg.config; in { options.services.jicofo = with types; { @@ -77,7 +72,7 @@ in }; config = mkOption { - type = (pkgs.formats.json {}).type; + type = format.type; default = { }; example = literalExpression '' { @@ -99,7 +94,7 @@ in hostname = cfg.xmppHost; username = cfg.userName; domain = cfg.userDomain; - password = { __hocon_envvar = "JICOFO_AUTH_PASS"; }; + password = format.lib.mkSubstitution "JICOFO_AUTH_PASS"; xmpp-domain = if cfg.xmppDomain == null then cfg.xmppHost else cfg.xmppDomain; }; service = client; diff --git a/nixos/modules/services/networking/jitsi-videobridge.nix b/nixos/modules/services/networking/jitsi-videobridge.nix index 37b0b1e5bf50..00ea5b9da546 100644 --- a/nixos/modules/services/networking/jitsi-videobridge.nix +++ b/nixos/modules/services/networking/jitsi-videobridge.nix @@ -6,16 +6,7 @@ let cfg = config.services.jitsi-videobridge; attrsToArgs = a: concatStringsSep " " (mapAttrsToList (k: v: "${k}=${toString v}") a); - # HOCON is a JSON superset that videobridge2 uses for configuration. - # It can substitute environment variables which we use for passwords here. - # https://github.com/lightbend/config/blob/master/README.md - # - # Substitution for environment variable FOO is represented as attribute set - # { __hocon_envvar = "FOO"; } - toHOCON = x: if isAttrs x && x ? __hocon_envvar then ("\${" + x.__hocon_envvar + "}") - else if isAttrs x then "{${ concatStringsSep "," (mapAttrsToList (k: v: ''"${k}":${toHOCON v}'') x) }}" - else if isList x then "[${ concatMapStringsSep "," toHOCON x }]" - else builtins.toJSON x; + format = pkgs.formats.hocon { }; # We're passing passwords in environment variables that have names generated # from an attribute name, which may not be a valid bash identifier. @@ -38,7 +29,7 @@ let hostname = xmppConfig.hostName; domain = xmppConfig.domain; username = xmppConfig.userName; - password = { __hocon_envvar = toVarName name; }; + password = format.lib.mkSubstitution (toVarName name); muc_jids = xmppConfig.mucJids; muc_nickname = xmppConfig.mucNickname; disable_certificate_verification = xmppConfig.disableCertificateVerification; @@ -221,7 +212,7 @@ in "-Dnet.java.sip.communicator.SC_HOME_DIR_LOCATION" = "/etc/jitsi"; "-Dnet.java.sip.communicator.SC_HOME_DIR_NAME" = "videobridge"; "-Djava.util.logging.config.file" = "/etc/jitsi/videobridge/logging.properties"; - "-Dconfig.file" = pkgs.writeText "jvb.conf" (toHOCON jvbConfig); + "-Dconfig.file" = format.generate "jvb.conf" jvbConfig; # Mitigate CVE-2021-44228 "-Dlog4j2.formatMsgNoLookups" = true; } // (mapAttrs' (k: v: nameValuePair "-D${k}" v) cfg.extraProperties); diff --git a/nixos/modules/services/web-apps/suwayomi-server.nix b/nixos/modules/services/web-apps/suwayomi-server.nix index c4c1540edbee..94dbe6f99356 100644 --- a/nixos/modules/services/web-apps/suwayomi-server.nix +++ b/nixos/modules/services/web-apps/suwayomi-server.nix @@ -3,6 +3,8 @@ let cfg = config.services.suwayomi-server; inherit (lib) mkOption mdDoc mkEnableOption mkIf types; + + format = pkgs.formats.hocon { }; in { options = { @@ -48,19 +50,7 @@ in settings = mkOption { type = types.submodule { - freeformType = - let - recursiveAttrsType = with types; attrsOf (nullOr (oneOf [ - str - path - int - float - bool - (listOf str) - (recursiveAttrsType // { description = "instances of this type recursively"; }) - ])); - in - recursiveAttrsType; + freeformType = format.type; options = { server = { ip = mkOption { @@ -180,38 +170,7 @@ in systemd.services.suwayomi-server = let - flattenConfig = prefix: config: - lib.foldl' - lib.mergeAttrs - { } - (lib.attrValues - (lib.mapAttrs - (k: v: - if !(lib.isAttrs v) - then { "${prefix}${k}" = v; } - else flattenConfig "${prefix}${k}." v - ) - config - ) - ); - - # HOCON is a JSON superset that suwayomi-server use for configuration - toHOCON = attr: - let - attrType = builtins.typeOf attr; - in - if builtins.elem attrType [ "string" "path" "int" "float" ] - then ''"${toString attr}"'' - else if attrType == "bool" - then lib.boolToString attr - else if attrType == "list" - then "[\n${lib.concatMapStringsSep ",\n" toHOCON attr}\n]" - else # attrs, lambda, null - throw '' - [suwayomi-server]: invalid config value type '${attrType}'. - ''; - - configFile = pkgs.writeText "server.conf" (lib.pipe cfg.settings [ + configFile = format.generate "server.conf" (lib.pipe cfg.settings [ (settings: lib.recursiveUpdate settings { server.basicAuthPasswordFile = null; server.basicAuthPassword = @@ -219,12 +178,8 @@ in then "$TACHIDESK_SERVER_BASIC_AUTH_PASSWORD" else null; }) - (flattenConfig "") - (lib.filterAttrs (_: x: x != null)) - (lib.mapAttrsToList (name: value: ''${name} = ${toHOCON value}'')) - lib.concatLines + (lib.filterAttrsRecursive (_: x: x != null)) ]); - in { description = "A free and open source manga reader server that runs extensions built for Tachiyomi."; From 9ebcb6f5dbd1a091c9b073587b6906b0c0663e08 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 9 Feb 2024 18:32:23 +0100 Subject: [PATCH 4/6] pkgs-lib: Make `lib` overlays be propagated This is useful because the tests in `pkgs-lib` can mock out certain `lib` functions like this using a `lib` overlay. --- pkgs/top-level/all-packages.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 8d895297e963..e8d3f8e7b4c0 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1480,7 +1480,12 @@ with pkgs; writers = callPackage ../build-support/writers { }; # lib functions depending on pkgs - inherit (import ../pkgs-lib { inherit lib pkgs; }) formats; + inherit (import ../pkgs-lib { + # The `lib` variable in this scope doesn't include any applied lib overlays, + # `pkgs.lib` does. + inherit (pkgs) lib; + inherit pkgs; + }) formats; testers = callPackage ../build-support/testers { }; From 0e65eca7c6c724bbddeb89ae8135c1fd67f71a84 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Wed, 24 Jan 2024 05:12:30 +0100 Subject: [PATCH 5/6] formats.hocon: add backwards compatibility --- pkgs/pkgs-lib/formats/hocon/default.nix | 82 ++++++++++++++----- pkgs/pkgs-lib/formats/hocon/src/src/main.rs | 11 +++ .../test/backwards-compatibility/default.nix | 65 +++++++++++++++ .../test/backwards-compatibility/expected.txt | 22 +++++ pkgs/pkgs-lib/formats/hocon/test/default.nix | 11 +++ 5 files changed, 170 insertions(+), 21 deletions(-) create mode 100644 pkgs/pkgs-lib/formats/hocon/test/backwards-compatibility/default.nix create mode 100644 pkgs/pkgs-lib/formats/hocon/test/backwards-compatibility/expected.txt diff --git a/pkgs/pkgs-lib/formats/hocon/default.nix b/pkgs/pkgs-lib/formats/hocon/default.nix index d5b6308dea60..318ee0143320 100644 --- a/pkgs/pkgs-lib/formats/hocon/default.nix +++ b/pkgs/pkgs-lib/formats/hocon/default.nix @@ -35,26 +35,8 @@ in # In the case that you need this functionality, # you will have to disable pyhocon validation. , doCheck ? true - }: { - type = let - type' = with lib.types; let - atomType = nullOr (oneOf [ - bool - float - int - path - str - ]); - in (oneOf [ - atomType - (listOf atomType) - (attrsOf type') - ]) // { - description = "HOCON value"; - }; - in type'; - - lib = { + }: let + hoconLib = { mkInclude = value: let includeStatement = if lib.isAttrs value && !(lib.isDerivation value) then { required = false; @@ -101,7 +83,65 @@ in }; }; + in { + type = let + type' = with lib.types; let + atomType = nullOr (oneOf [ + bool + float + int + path + str + ]); + in (oneOf [ + atomType + (listOf atomType) + (attrsOf type') + ]) // { + description = "HOCON value"; + }; + in type'; + + lib = hoconLib; + generate = name: value: + let + # TODO: remove in 24.11 + # Backwards compatability for generators in the following locations: + # - nixos/modules/services/networking/jibri/default.nix (__hocon_envvar) + # - nixos/modules/services/networking/jicofo.nix (__hocon_envvar, __hocon_unquoted_string) + # - nixos/modules/services/networking/jitsi-videobridge.nix (__hocon_envvar) + replaceOldIndicators = value: + if lib.isAttrs value then + (if value ? "__hocon_envvar" + then + lib.warn '' + Use of `__hocon_envvar` has been deprecated, and will + be removed in the future. + + Please use `(pkgs.formats.hocon {}).lib.mkSubstitution` instead. + '' + (hoconLib.mkSubstitution value.__hocon_envvar) + else if value ? "__hocon_unquoted_string" + then + lib.warn '' + Use of `__hocon_unquoted_string` has been deprecated, and will + be removed in the future. + + Please make use of the freeform options of + `(pkgs.formats.hocon {}).format` instead. + '' + { + value = value.__hocon_unquoted_string; + _type = "unquoted_string"; + } + else lib.mapAttrs (_: replaceOldIndicators) value) + else if lib.isList value + then map replaceOldIndicators value + else value; + + finalValue = replaceOldIndicators value; + in callPackage ({ stdenvNoCC @@ -114,7 +154,7 @@ in dontUnpack = true; - json = builtins.toJSON value; + json = builtins.toJSON finalValue; passAsFile = [ "json" ]; strictDeps = true; diff --git a/pkgs/pkgs-lib/formats/hocon/src/src/main.rs b/pkgs/pkgs-lib/formats/hocon/src/src/main.rs index a564fc7dccdb..2e53f3fd5659 100644 --- a/pkgs/pkgs-lib/formats/hocon/src/src/main.rs +++ b/pkgs/pkgs-lib/formats/hocon/src/src/main.rs @@ -10,6 +10,7 @@ enum HOCONValue { List(Vec), Substitution(String, bool), Object(Vec, Vec<(String, HOCONValue)>), + Literal(String), } #[derive(Debug)] @@ -92,6 +93,15 @@ fn parse_special_types(o: &Map) -> Option { HOCONValue::Append(Box::new(json_to_hocon(value))) } + "unquoted_string" => { + let value = o + .get("value") + .expect("Missing value for unquoted_string") + .as_str() + .unwrap_or_else(|| panic!("Unquoted string value is not a string: {:?}", o)); + + HOCONValue::Literal(value.to_string()) + } _ => panic!( "\ Attribute set contained special element '_type',\ @@ -210,6 +220,7 @@ impl ToString for HOCONValue { format!("{{\n{}\n}}", content) } HOCONValue::Append(_) => panic!("Append should not be present at this point"), + Self::Literal(s) => s.to_string(), } } } diff --git a/pkgs/pkgs-lib/formats/hocon/test/backwards-compatibility/default.nix b/pkgs/pkgs-lib/formats/hocon/test/backwards-compatibility/default.nix new file mode 100644 index 000000000000..5f0b3d12a2d0 --- /dev/null +++ b/pkgs/pkgs-lib/formats/hocon/test/backwards-compatibility/default.nix @@ -0,0 +1,65 @@ +{ lib, formats, stdenvNoCC, writeText, ... }: +let + hocon = formats.hocon { }; + + expression = { + substitution = { __hocon_envvar = "PATH"; }; + literal = { + __hocon_unquoted_string = '' + [ + 1, + "a", + ]''; + }; + + nested = { + substitution = { __hocon_envvar = "PATH"; }; + literal = { + __hocon_unquoted_string = '' + [ + 1, + "a", + ]''; + }; + }; + + nested_in_array = [ + { __hocon_envvar = "PATH"; } + { + __hocon_unquoted_string = '' + [ + 1, + "a", + ]''; + } + ]; + }; + + hocon-test-conf = hocon.generate "hocon-test.conf" expression; +in + stdenvNoCC.mkDerivation { + name = "pkgs.formats.hocon-test-backwards-compatibility"; + + dontUnpack = true; + dontBuild = true; + + doCheck = true; + checkPhase = '' + runHook preCheck + + diff -U3 ${./expected.txt} ${hocon-test-conf} + + runHook postCheck + ''; + + installPhase = '' + runHook preInstall + + mkdir $out + cp ${./expected.txt} $out/expected.txt + cp ${hocon-test-conf} $out/hocon-test.conf + cp ${hocon-test-conf.passthru.json} $out/hocon-test.json + + runHook postInstall + ''; + } diff --git a/pkgs/pkgs-lib/formats/hocon/test/backwards-compatibility/expected.txt b/pkgs/pkgs-lib/formats/hocon/test/backwards-compatibility/expected.txt new file mode 100644 index 000000000000..2835a3c6ca39 --- /dev/null +++ b/pkgs/pkgs-lib/formats/hocon/test/backwards-compatibility/expected.txt @@ -0,0 +1,22 @@ +{ + "literal" = [ + 1, + "a", + ] + "nested" = { + "literal" = [ + 1, + "a", + ] + "substitution" = ${?PATH} + } + "nested_in_array" = [ + ${?PATH}, + [ + 1, + "a", + ] + ] + "substitution" = ${?PATH} +} + diff --git a/pkgs/pkgs-lib/formats/hocon/test/default.nix b/pkgs/pkgs-lib/formats/hocon/test/default.nix index 6cd03fe4854f..19928703b95e 100644 --- a/pkgs/pkgs-lib/formats/hocon/test/default.nix +++ b/pkgs/pkgs-lib/formats/hocon/test/default.nix @@ -1,4 +1,15 @@ { pkgs, ... }: { comprehensive = pkgs.callPackage ./comprehensive { }; + backwards-compatibility = + let + pkgsNoWarn = pkgs.extend (final: prev: { + lib = prev.lib.extend (libFinal: libPrev: { + warn = msg: v: v; + trivial = libPrev.trivial // { + warn = msg: v: v; + }; + }); + }); + in pkgsNoWarn.callPackage ./backwards-compatibility { }; } From 7065951e177847b3d2325568071b7c0ece9957ca Mon Sep 17 00:00:00 2001 From: h7x4 Date: Thu, 25 Jan 2024 08:20:38 +0100 Subject: [PATCH 6/6] CODEOWNERS: add h7x4 to pkgs.formats.hocon --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3ef3d178fe5d..e58c00f6a5dc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -52,6 +52,7 @@ /pkgs/pkgs-lib @infinisil ## Format generators/serializers /pkgs/pkgs-lib/formats/libconfig @ckiee @h7x4 +/pkgs/pkgs-lib/formats/hocon @h7x4 # pkgs/by-name /pkgs/test/nixpkgs-check-by-name @infinisil