diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index 701507178380..19d4496eef51 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -285,11 +285,11 @@ specifying an interpreter version), like this: ```nix { lib -, python3 +, python3Packages , fetchPypi }: -python3.pkgs.buildPythonApplication rec { +python3Packages.buildPythonApplication rec { pname = "luigi"; version = "2.7.9"; pyproject = true; @@ -300,13 +300,13 @@ python3.pkgs.buildPythonApplication rec { }; nativeBuildInputs = [ - python3.pkgs.setuptools - python3.pkgs.wheel + python3Packages.setuptools + python3Packages.wheel ]; - propagatedBuildInputs = with python3.pkgs; [ - tornado - python-daemon + propagatedBuildInputs = [ + python3Packages.tornado + python3Packages.python-daemon ]; meta = with lib; { diff --git a/lib/fileset/README.md b/lib/fileset/README.md index 16ab58e2f266..8518d88a7d64 100644 --- a/lib/fileset/README.md +++ b/lib/fileset/README.md @@ -244,5 +244,4 @@ Here's a list of places in the library that need to be updated in the future: - > The file set library is currently somewhat limited but is being expanded to include more functions over time. in [the manual](../../doc/functions/fileset.section.md) -- If/Once a function to convert `lib.sources` values into file sets exists, the `_coerce` and `toSource` functions should be updated to mention that function in the error when such a value is passed - If/Once a function exists that can optionally include a path depending on whether it exists, the error message for the path not existing in `_coerce` should mention the new function diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index fe7b304ba698..372d445269f5 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -3,8 +3,10 @@ let inherit (import ./internal.nix { inherit lib; }) _coerce + _singleton _coerceMany _toSourceFilter + _fromSourceFilter _unionMany _fileFilter _printFileset @@ -152,7 +154,12 @@ in { sourceFilter = _toSourceFilter fileset; in if ! isPath root then - if isStringLike root then + if root ? _isLibCleanSourceWith then + throw '' + lib.fileset.toSource: `root` is a `lib.sources`-based value, but it should be a path instead. + To use a `lib.sources`-based value, convert it to a file set using `lib.fileset.fromSource` and pass it as `fileset`. + Note that this only works for sources created from paths.'' + else if isStringLike root then throw '' lib.fileset.toSource: `root` (${toString root}) is a string-like value, but it should be a path instead. Paths in strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.'' @@ -188,6 +195,75 @@ in { filter = sourceFilter; }; + /* + Create a file set with the same files as a `lib.sources`-based value. + This does not import any of the files into the store. + + This can be used to gradually migrate from `lib.sources`-based filtering to `lib.fileset`. + + A file set can be turned back into a source using [`toSource`](#function-library-lib.fileset.toSource). + + :::{.note} + File sets cannot represent empty directories. + Turning the result of this function back into a source using `toSource` will therefore not preserve empty directories. + ::: + + Type: + fromSource :: SourceLike -> FileSet + + Example: + # There's no cleanSource-like function for file sets yet, + # but we can just convert cleanSource to a file set and use it that way + toSource { + root = ./.; + fileset = fromSource (lib.sources.cleanSource ./.); + } + + # Keeping a previous sourceByRegex (which could be migrated to `lib.fileset.unions`), + # but removing a subdirectory using file set functions + difference + (fromSource (lib.sources.sourceByRegex ./. [ + "^README\.md$" + # This regex includes everything in ./doc + "^doc(/.*)?$" + ]) + ./doc/generated + + # Use cleanSource, but limit it to only include ./Makefile and files under ./src + intersection + (fromSource (lib.sources.cleanSource ./.)) + (unions [ + ./Makefile + ./src + ]); + */ + fromSource = source: + let + # This function uses `._isLibCleanSourceWith`, `.origSrc` and `.filter`, + # which are technically internal to lib.sources, + # but we'll allow this since both libraries are in the same code base + # and this function is a bridge between them. + isFiltered = source ? _isLibCleanSourceWith; + path = if isFiltered then source.origSrc else source; + in + # We can only support sources created from paths + if ! isPath path then + if isStringLike path then + throw '' + lib.fileset.fromSource: The source origin of the argument is a string-like value ("${toString path}"), but it should be a path instead. + Sources created from paths in strings cannot be turned into file sets, use `lib.sources` or derivations instead.'' + else + throw '' + lib.fileset.fromSource: The source origin of the argument is of type ${typeOf path}, but it should be a path instead.'' + else if ! pathExists path then + throw '' + lib.fileset.fromSource: The source origin (${toString path}) of the argument does not exist.'' + else if isFiltered then + _fromSourceFilter path source.filter + else + # If there's no filter, no need to run the expensive conversion, all subpaths will be included + _singleton path; + /* The file set containing all files that are in either of two given file sets. This is the same as [`unions`](#function-library-lib.fileset.unions), diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index d55c84a395e7..717253f45715 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -167,7 +167,12 @@ rec { else value else if ! isPath value then - if isStringLike value then + if value ? _isLibCleanSourceWith then + throw '' + ${context} is a `lib.sources`-based value, but it should be a file set or a path instead. + To convert a `lib.sources`-based value to a file set you can use `lib.fileset.fromSource`. + Note that this only works for sources created from paths.'' + else if isStringLike value then throw '' ${context} ("${toString value}") is a string-like value, but it should be a file set or a path instead. Paths represented as strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.'' @@ -470,6 +475,59 @@ rec { else nonEmpty; + # Turn a builtins.filterSource-based source filter on a root path into a file set + # containing only files included by the filter. + # The filter is lazily called as necessary to determine whether paths are included + # Type: Path -> (String -> String -> Bool) -> fileset + _fromSourceFilter = root: sourceFilter: + let + # During the recursion we need to track both: + # - The path value such that we can safely call `readDir` on it + # - The path string value such that we can correctly call the `filter` with it + # + # While we could just recurse with the path value, + # this would then require converting it to a path string for every path, + # which is a fairly expensive operation + + # Create a file set from a directory entry + fromDirEntry = path: pathString: type: + # The filter needs to run on the path as a string + if ! sourceFilter pathString type then + null + else if type == "directory" then + fromDir path pathString + else + type; + + # Create a file set from a directory + fromDir = path: pathString: + mapAttrs + # This looks a bit funny, but we need both the path-based and the path string-based values + (name: fromDirEntry (path + "/${name}") (pathString + "/${name}")) + # We need to readDir on the path value, because reading on a path string + # would be unspecified if there are multiple filesystem roots + (readDir path); + + rootPathType = pathType root; + + # We need to convert the path to a string to imitate what builtins.path calls the filter function with. + # We don't want to rely on `toString` for this though because it's not very well defined, see ../path/README.md + # So instead we use `lib.path.splitRoot` to safely deconstruct the path into its filesystem root and subpath + # We don't need the filesystem root though, builtins.path doesn't expose that in any way to the filter. + # So we only need the components, which we then turn into a string as one would expect. + rootString = "/" + concatStringsSep "/" (components (splitRoot root).subpath); + in + if rootPathType == "directory" then + # We imitate builtins.path not calling the filter on the root path + _create root (fromDir root rootString) + else + # Direct files are always included by builtins.path without calling the filter + # But we need to lift up the base path to its parent to satisfy the base path invariant + _create (dirOf root) + { + ${baseNameOf root} = rootPathType; + }; + # Transforms the filesetTree of a file set to a shorter base path, e.g. # _shortenTreeBase [ "foo" ] (_create /foo/bar null) # => { bar = null; } diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index c1c67800f5e2..796a03b52f0e 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash # shellcheck disable=SC2016 +# shellcheck disable=SC2317 +# shellcheck disable=SC2192 # Tests lib.fileset # Run: @@ -224,23 +226,17 @@ withFileMonitor() { fi } -# Check whether a file set includes/excludes declared paths as expected, usage: + +# Create the tree structure declared in the tree variable, usage: # # tree=( -# [a/b] =1 # Declare that file a/b should exist and expect it to be included in the store path -# [c/a] = # Declare that file c/a should exist and expect it to be excluded in the store path -# [c/d/]= # Declare that directory c/d/ should exist and expect it to be excluded in the store path +# [a/b] = # Declare that file a/b should exist +# [c/a] = # Declare that file c/a should exist +# [c/d/]= # Declare that directory c/d/ should exist # ) -# checkFileset './a' # Pass the fileset as the argument +# createTree declare -A tree -checkFileset() { - # New subshell so that we can have a separate trap handler, see `trap` below - local fileset=$1 - - # Process the tree into separate arrays for included paths, excluded paths and excluded files. - local -a included=() - local -a excluded=() - local -a excludedFiles=() +createTree() { # Track which paths need to be created local -a dirsToCreate=() local -a filesToCreate=() @@ -248,24 +244,9 @@ checkFileset() { # If keys end with a `/` we treat them as directories, otherwise files if [[ "$p" =~ /$ ]]; then dirsToCreate+=("$p") - isFile= else filesToCreate+=("$p") - isFile=1 fi - case "${tree[$p]}" in - 1) - included+=("$p") - ;; - 0) - excluded+=("$p") - if [[ -n "$isFile" ]]; then - excludedFiles+=("$p") - fi - ;; - *) - die "Unsupported tree value: ${tree[$p]}" - esac done # Create all the necessary paths. @@ -280,6 +261,43 @@ checkFileset() { mkdir -p "${parentsToCreate[@]}" touch "${filesToCreate[@]}" fi +} + +# Check whether a file set includes/excludes declared paths as expected, usage: +# +# tree=( +# [a/b] =1 # Declare that file a/b should exist and expect it to be included in the store path +# [c/a] = # Declare that file c/a should exist and expect it to be excluded in the store path +# [c/d/]= # Declare that directory c/d/ should exist and expect it to be excluded in the store path +# ) +# checkFileset './a' # Pass the fileset as the argument +checkFileset() { + # New subshell so that we can have a separate trap handler, see `trap` below + local fileset=$1 + + # Create the tree + createTree + + # Process the tree into separate arrays for included paths, excluded paths and excluded files. + local -a included=() + local -a excluded=() + local -a excludedFiles=() + for p in "${!tree[@]}"; do + case "${tree[$p]}" in + 1) + included+=("$p") + ;; + 0) + excluded+=("$p") + # If keys end with a `/` we treat them as directories, otherwise files + if [[ ! "$p" =~ /$ ]]; then + excludedFiles+=("$p") + fi + ;; + *) + die "Unsupported tree value: ${tree[$p]}" + esac + done expression="toSource { root = ./.; fileset = $fileset; }" @@ -321,6 +339,10 @@ checkFileset() { expectFailure 'toSource { root = "/nix/store/foobar"; fileset = ./.; }' 'lib.fileset.toSource: `root` \(/nix/store/foobar\) is a string-like value, but it should be a path instead. \s*Paths in strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.' +expectFailure 'toSource { root = cleanSourceWith { src = ./.; }; fileset = ./.; }' 'lib.fileset.toSource: `root` is a `lib.sources`-based value, but it should be a path instead. +\s*To use a `lib.sources`-based value, convert it to a file set using `lib.fileset.fromSource` and pass it as `fileset`. +\s*Note that this only works for sources created from paths.' + # Only paths are accepted as `root` expectFailure 'toSource { root = 10; fileset = ./.; }' 'lib.fileset.toSource: `root` is of type int, but it should be a path instead.' @@ -365,6 +387,9 @@ rm -rf -- * expectFailure 'toSource { root = ./.; fileset = 10; }' 'lib.fileset.toSource: `fileset` is of type int, but it should be a file set or a path instead.' expectFailure 'toSource { root = ./.; fileset = "/some/path"; }' 'lib.fileset.toSource: `fileset` \("/some/path"\) is a string-like value, but it should be a file set or a path instead. \s*Paths represented as strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.' +expectFailure 'toSource { root = ./.; fileset = cleanSourceWith { src = ./.; }; }' 'lib.fileset.toSource: `fileset` is a `lib.sources`-based value, but it should be a file set or a path instead. +\s*To convert a `lib.sources`-based value to a file set you can use `lib.fileset.fromSource`. +\s*Note that this only works for sources created from paths.' # Path coercion errors for non-existent paths expectFailure 'toSource { root = ./.; fileset = ./a; }' 'lib.fileset.toSource: `fileset` \('"$work"'/a\) is a path that does not exist.' @@ -995,6 +1020,217 @@ touch 0 "${filesToCreate[@]}" expectTrace 'unions (mapAttrsToList (n: _: ./. + "/${n}") (removeAttrs (builtins.readDir ./.) [ "0" ]))' "$expectedTrace" rm -rf -- * +## lib.fileset.fromSource + +# Check error messages +expectFailure 'fromSource null' 'lib.fileset.fromSource: The source origin of the argument is of type null, but it should be a path instead.' + +expectFailure 'fromSource (lib.cleanSource "")' 'lib.fileset.fromSource: The source origin of the argument is a string-like value \(""\), but it should be a path instead. +\s*Sources created from paths in strings cannot be turned into file sets, use `lib.sources` or derivations instead.' + +expectFailure 'fromSource (lib.cleanSource null)' 'lib.fileset.fromSource: The source origin of the argument is of type null, but it should be a path instead.' + +# fromSource on a path works and is the same as coercing that path +mkdir a +touch a/b c +expectEqual 'trace (fromSource ./.) null' 'trace ./. null' +rm -rf -- * + +# Check that converting to a file set doesn't read the included files +mkdir a +touch a/b +run() { + expectEqual "trace (fromSource (lib.cleanSourceWith { src = ./a; })) null" "builtins.trace \"$work/a (all files in directory)\" null" + rm a/b +} +withFileMonitor run a/b +rm -rf -- * + +# Check that converting to a file set doesn't read entries for directories that are filtered out +mkdir -p a/b +touch a/b/c +run() { + expectEqual "trace (fromSource (lib.cleanSourceWith { + src = ./a; + filter = pathString: type: false; + })) null" "builtins.trace \"(empty)\" null" + rm a/b/c + rmdir a/b +} +withFileMonitor run a/b +rm -rf -- * + +# The filter is not needed on empty directories +expectEqual 'trace (fromSource (lib.cleanSourceWith { + src = ./.; + filter = abort "filter should not be needed"; +})) null' 'trace _emptyWithoutBase null' + +# Single files also work +touch a b +expectEqual 'trace (fromSource (cleanSourceWith { src = ./a; })) null' 'trace ./a null' +rm -rf -- * + +# For a tree assigning each subpath true/false, +# check whether a source filter with those results includes the same files +# as a file set created using fromSource. Usage: +# +# tree=( +# [a]=1 # ./a is a file and the filter should return true for it +# [b/]=0 # ./b is a directory and the filter should return false for it +# ) +# checkSource +checkSource() { + createTree + + # Serialise the tree as JSON (there's only minimal savings with jq, + # and we don't need to handle escapes) + { + echo "{" + first=1 + for p in "${!tree[@]}"; do + if [[ -z "$first" ]]; then + echo "," + else + first= + fi + echo "\"$p\":" + case "${tree[$p]}" in + 1) + echo "true" + ;; + 0) + echo "false" + ;; + *) + die "Unsupported tree value: ${tree[$p]}" + esac + done + echo "}" + } > "$tmp/tree.json" + + # An expression to create a source value with a filter matching the tree + sourceExpr=' + let + tree = importJSON '"$tmp"'/tree.json; + in + cleanSourceWith { + src = ./.; + filter = + pathString: type: + let + stripped = removePrefix (toString ./. + "/") pathString; + key = stripped + optionalString (type == "directory") "/"; + in + tree.${key} or + (throw "tree key ${key} missing"); + } + ' + + filesetExpr=' + toSource { + root = ./.; + fileset = fromSource ('"$sourceExpr"'); + } + ' + + # Turn both into store paths + sourceStorePath=$(expectStorePath "$sourceExpr") + filesetStorePath=$(expectStorePath "$filesetExpr") + + # Loop through each path in the tree + while IFS= read -r -d $'\0' subpath; do + if [[ ! -e "$sourceStorePath"/"$subpath" ]]; then + # If it's not in the source store path, it's also not in the file set store path + if [[ -e "$filesetStorePath"/"$subpath" ]]; then + die "The store path $sourceStorePath created by $expr doesn't contain $subpath, but the corresponding store path $filesetStorePath created via fromSource does contain $subpath" + fi + elif [[ -z "$(find "$sourceStorePath"/"$subpath" -type f)" ]]; then + # If it's an empty directory in the source store path, it shouldn't be in the file set store path + if [[ -e "$filesetStorePath"/"$subpath" ]]; then + die "The store path $sourceStorePath created by $expr contains the path $subpath without any files, but the corresponding store path $filesetStorePath created via fromSource didn't omit it" + fi + else + # If it's non-empty directory or a file, it should be in the file set store path + if [[ ! -e "$filesetStorePath"/"$subpath" ]]; then + die "The store path $sourceStorePath created by $expr contains the non-empty path $subpath, but the corresponding store path $filesetStorePath created via fromSource doesn't include it" + fi + fi + done < <(find . -mindepth 1 -print0) + + rm -rf -- * +} + +# Check whether the filter is evaluated correctly +tree=( + [a]= + [b/]= + [b/c]= + [b/d]= + [e/]= + [e/e/]= +) +# We fill out the above tree values with all possible combinations of 0 and 1 +# Then check whether a filter based on those return values gets turned into the corresponding file set +for i in $(seq 0 $((2 ** ${#tree[@]} - 1 ))); do + for p in "${!tree[@]}"; do + tree[$p]=$(( i % 2 )) + (( i /= 2 )) || true + done + checkSource +done + +# The filter is called with the same arguments in the same order +mkdir a e +touch a/b a/c d e +expectEqual ' + trace (fromSource (cleanSourceWith { + src = ./.; + filter = pathString: type: builtins.trace "${pathString} ${toString type}" true; + })) null +' ' + builtins.seq (cleanSourceWith { + src = ./.; + filter = pathString: type: builtins.trace "${pathString} ${toString type}" true; + }).outPath + builtins.trace "'"$work"' (all files in directory)" + null +' +rm -rf -- * + +# Test that if a directory is not included, the filter isn't called on its contents +mkdir a b +touch a/c b/d +expectEqual 'trace (fromSource (cleanSourceWith { + src = ./.; + filter = pathString: type: + if pathString == toString ./a then + false + else if pathString == toString ./b then + true + else if pathString == toString ./b/d then + true + else + abort "This filter should not be called with path ${pathString}"; +})) null' 'trace (_create ./. { b = "directory"; }) null' +rm -rf -- * + +# The filter is called lazily: +# If a later say intersection removes a part of the tree, the filter won't run on it +mkdir a d +touch a/{b,c} d/e +expectEqual 'trace (intersection ./a (fromSource (lib.cleanSourceWith { + src = ./.; + filter = pathString: type: + if pathString == toString ./a || pathString == toString ./a/b then + true + else if pathString == toString ./a/c then + false + else + abort "filter should not be called on ${pathString}"; +}))) null' 'trace ./a/b null' +rm -rf -- * + # TODO: Once we have combinators and a property testing library, derive property tests from https://en.wikipedia.org/wiki/Algebra_of_sets echo >&2 tests ok diff --git a/nixos/doc/manual/development/running-nixos-tests-interactively.section.md b/nixos/doc/manual/development/running-nixos-tests-interactively.section.md index 54002941d634..a816213f37df 100644 --- a/nixos/doc/manual/development/running-nixos-tests-interactively.section.md +++ b/nixos/doc/manual/development/running-nixos-tests-interactively.section.md @@ -57,6 +57,22 @@ using: Once the connection is established, you can enter commands in the socat terminal where socat is running. +## Port forwarding to NixOS test VMs {#sec-nixos-test-port-forwarding} + +If your test has only a single VM, you may use e.g. + +```ShellSession +$ QEMU_NET_OPTS="hostfwd=tcp:127.0.0.1:2222-127.0.0.1:22" ./result/bin/nixos-test-driver +``` + +to port-forward a port in the VM (here `22`) to the host machine (here port `2222`). + +This naturally does not work when multiple machines are involved, +since a single port on the host cannot forward to multiple VMs. + +If the test defines multiple machines, you may opt to _temporarily_ set +`virtualisation.forwardPorts` in the test definition for debugging. + ## Reuse VM state {#sec-nixos-test-reuse-vm-state} You can re-use the VM states coming from a previous run by setting the diff --git a/nixos/doc/manual/installation/changing-config.chapter.md b/nixos/doc/manual/installation/changing-config.chapter.md index 11b49ccb1f67..f2ffea9088a1 100644 --- a/nixos/doc/manual/installation/changing-config.chapter.md +++ b/nixos/doc/manual/installation/changing-config.chapter.md @@ -89,7 +89,7 @@ guest. For instance, the following will forward host port 2222 to guest port 22 (SSH): ```ShellSession -$ QEMU_NET_OPTS="hostfwd=tcp::2222-:22" ./result/bin/run-*-vm +$ QEMU_NET_OPTS="hostfwd=tcp:127.0.0.1:2222-127.0.0.1:22" ./result/bin/run-*-vm ``` allowing you to log in via SSH (assuming you have set the appropriate diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index 03bd201246c5..6b4e7adbc92e 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -33,6 +33,8 @@ - All [ROCm](https://rocm.docs.amd.com/en/latest/) packages have been updated to 5.7.0. - [ROCm](https://rocm.docs.amd.com/en/latest/) package attribute sets are versioned: `rocmPackages` -> `rocmPackages_5`. +- `yarn-berry` has been updated to 4.0.1. This means that NodeJS versions less than `18.12` are no longer supported by it. More details at the [upstream changelog](https://github.com/yarnpkg/berry/blob/master/CHANGELOG.md). + - If the user has a custom shell enabled via `users.users.${USERNAME}.shell = ${CUSTOMSHELL}`, the assertion will require them to also set `programs.${CUSTOMSHELL}.enable = true`. This is generally safe behavior, but for anyone needing to opt out from @@ -373,6 +375,8 @@ - The `junicode` font package has been updated to [major version 2](https://github.com/psb1558/Junicode-font/releases/tag/v2.001), which is now a font family. In particular, plain `Junicode.ttf` no longer exists. In addition, TrueType font files are now placed in `font/truetype` instead of `font/junicode-ttf`; this change does not affect use via `fonts.packages` NixOS option. +- The `prayer` package as well as `services.prayer` have been removed because it's been unmaintained for several years and the author's website has vanished. + ## Other Notable Changes {#sec-release-23.11-notable-changes} - A new option `system.switch.enable` was added. By default, this is option is @@ -525,6 +529,8 @@ The module update takes care of the new config syntax and the data itself (user - `services.bitcoind` now properly respects the `enable` option. +- The Home Assistant module now offers support for installing custom components and lovelace modules. Available at [`services.home-assistant.customComponents`](#opt-services.home-assistant.customComponents) and [`services.home-assistant.customLovelaceModules`](#opt-services.home-assistant.customLovelaceModules). + ## Nixpkgs internals {#sec-release-23.11-nixpkgs-internals} - The use of `sourceRoot = "source";`, `sourceRoot = "source/subdir";`, and similar lines in package derivations using the default `unpackPhase` is deprecated as it requires `unpackPhase` to always produce a directory named "source". Use `sourceRoot = src.name`, `sourceRoot = "${src.name}/subdir";`, or `setSourceRoot = "sourceRoot=$(echo */subdir)";` or similar instead. diff --git a/nixos/modules/misc/ids.nix b/nixos/modules/misc/ids.nix index 5b278b5e8062..18928a6bf21b 100644 --- a/nixos/modules/misc/ids.nix +++ b/nixos/modules/misc/ids.nix @@ -86,7 +86,7 @@ in #rtkit = 45; # dynamically allocated 2021-09-03 dovecot2 = 46; dovenull2 = 47; - prayer = 49; + # prayer = 49; # dropped in 23.11 mpd = 50; clamav = 51; #fprot = 52; # unused @@ -411,7 +411,7 @@ in #rtkit = 45; # unused dovecot2 = 46; dovenull2 = 47; - prayer = 49; + # prayer = 49; # dropped in 23.11 mpd = 50; clamav = 51; #fprot = 52; # unused diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 6679e5bb7c65..00da63992951 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1041,7 +1041,6 @@ ./services/networking/powerdns.nix ./services/networking/pppd.nix ./services/networking/pptpd.nix - ./services/networking/prayer.nix ./services/networking/privoxy.nix ./services/networking/prosody.nix ./services/networking/quassel.nix diff --git a/nixos/modules/rename.nix b/nixos/modules/rename.nix index 0fbb2351f986..3fab863adb7f 100644 --- a/nixos/modules/rename.nix +++ b/nixos/modules/rename.nix @@ -111,6 +111,7 @@ in (mkRemovedOptionModule [ "services" "riak" ] "The corresponding package was removed from nixpkgs.") (mkRemovedOptionModule [ "services" "cryptpad" ] "The corresponding package was removed from nixpkgs.") (mkRemovedOptionModule [ "services" "rtsp-simple-server" ] "Package has been completely rebranded by upstream as mediamtx, and thus the service and the package were renamed in NixOS as well.") + (mkRemovedOptionModule [ "services" "prayer" ] "The corresponding package was removed from nixpkgs.") (mkRemovedOptionModule [ "i18n" "inputMethod" "fcitx" ] "The fcitx module has been removed. Please use fcitx5 instead") (mkRemovedOptionModule [ "services" "dhcpd4" ] '' diff --git a/nixos/modules/services/desktops/gnome/at-spi2-core.nix b/nixos/modules/services/desktops/gnome/at-spi2-core.nix index 10a2f1f9eca0..d0320c1e6307 100644 --- a/nixos/modules/services/desktops/gnome/at-spi2-core.nix +++ b/nixos/modules/services/desktops/gnome/at-spi2-core.nix @@ -51,7 +51,7 @@ with lib; }) (mkIf (!config.services.gnome.at-spi2-core.enable) { - environment.variables = { + environment.sessionVariables = { NO_AT_BRIDGE = "1"; GTK_A11Y = "none"; }; diff --git a/nixos/modules/services/home-automation/home-assistant.nix b/nixos/modules/services/home-automation/home-assistant.nix index 789b06af19b1..54fd3e17292f 100644 --- a/nixos/modules/services/home-automation/home-assistant.nix +++ b/nixos/modules/services/home-automation/home-assistant.nix @@ -16,7 +16,8 @@ let cp ${format.generate "configuration.yaml" filteredConfig} $out sed -i -e "s/'\!\([a-z_]\+\) \(.*\)'/\!\1 \2/;s/^\!\!/\!/;" $out ''; - lovelaceConfig = cfg.lovelaceConfig or {}; + lovelaceConfig = if (cfg.lovelaceConfig == null) then {} + else (lib.recursiveUpdate customLovelaceModulesResources cfg.lovelaceConfig); lovelaceConfigFile = format.generate "ui-lovelace.yaml" lovelaceConfig; # Components advertised by the home-assistant package @@ -62,8 +63,24 @@ let # Respect overrides that already exist in the passed package and # concat it with values passed via the module. extraComponents = oldArgs.extraComponents or [] ++ extraComponents; - extraPackages = ps: (oldArgs.extraPackages or (_: []) ps) ++ (cfg.extraPackages ps); + extraPackages = ps: (oldArgs.extraPackages or (_: []) ps) + ++ (cfg.extraPackages ps) + ++ (lib.concatMap (component: component.propagatedBuildInputs or []) cfg.customComponents); })); + + # Create a directory that holds all lovelace modules + customLovelaceModulesDir = pkgs.buildEnv { + name = "home-assistant-custom-lovelace-modules"; + paths = cfg.customLovelaceModules; + }; + + # Create parts of the lovelace config that reference lovelave modules as resources + customLovelaceModulesResources = { + lovelace.resources = map (card: { + url = "/local/nixos-lovelace-modules/${card.entrypoint or card.pname}.js?${card.version}"; + type = "module"; + }) cfg.customLovelaceModules; + }; in { imports = [ # Migrations in NixOS 22.05 @@ -137,6 +154,41 @@ in { ''; }; + customComponents = mkOption { + type = types.listOf types.package; + default = []; + example = literalExpression '' + with pkgs.home-assistant-custom-components; [ + prometheus-sensor + ]; + ''; + description = lib.mdDoc '' + List of custom component packages to install. + + Available components can be found below `pkgs.home-assistant-custom-components`. + ''; + }; + + customLovelaceModules = mkOption { + type = types.listOf types.package; + default = []; + example = literalExpression '' + with pkgs.home-assistant-custom-lovelace-modules; [ + mini-graph-card + mini-media-player + ]; + ''; + description = lib.mdDoc '' + List of custom lovelace card packages to load as lovelace resources. + + Available cards can be found below `pkgs.home-assistant-custom-lovelace-modules`. + + ::: {.note} + Automatic loading only works with lovelace in `yaml` mode. + ::: + ''; + }; + config = mkOption { type = types.nullOr (types.submodule { freeformType = format.type; @@ -408,9 +460,35 @@ in { rm -f "${cfg.configDir}/ui-lovelace.yaml" ln -s /etc/home-assistant/ui-lovelace.yaml "${cfg.configDir}/ui-lovelace.yaml" ''; + copyCustomLovelaceModules = if cfg.customLovelaceModules != [] then '' + mkdir -p "${cfg.configDir}/www" + ln -fns ${customLovelaceModulesDir} "${cfg.configDir}/www/nixos-lovelace-modules" + '' else '' + rm -f "${cfg.configDir}/www/nixos-lovelace-modules" + ''; + copyCustomComponents = '' + mkdir -p "${cfg.configDir}/custom_components" + + # remove components symlinked in from below the /nix/store + components="$(find "${cfg.configDir}/custom_components" -maxdepth 1 -type l)" + for component in "$components"; do + if [[ "$(readlink "$component")" =~ ^${escapeShellArg builtins.storeDir} ]]; then + rm "$component" + fi + done + + # recreate symlinks for desired components + declare -a components=(${escapeShellArgs cfg.customComponents}) + for component in "''${components[@]}"; do + path="$(dirname $(find "$component" -name "manifest.json"))" + ln -fns "$path" "${cfg.configDir}/custom_components/" + done + ''; in (optionalString (cfg.config != null) copyConfig) + - (optionalString (cfg.lovelaceConfig != null) copyLovelaceConfig) + (optionalString (cfg.lovelaceConfig != null) copyLovelaceConfig) + + copyCustomLovelaceModules + + copyCustomComponents ; environment.PYTHONPATH = package.pythonPath; serviceConfig = let diff --git a/nixos/modules/services/networking/prayer.nix b/nixos/modules/services/networking/prayer.nix deleted file mode 100644 index 197aa8a6f448..000000000000 --- a/nixos/modules/services/networking/prayer.nix +++ /dev/null @@ -1,90 +0,0 @@ -{ config, lib, pkgs, ... }: - -with lib; - -let - - inherit (pkgs) prayer; - - cfg = config.services.prayer; - - stateDir = "/var/lib/prayer"; - - prayerUser = "prayer"; - prayerGroup = "prayer"; - - prayerExtraCfg = pkgs.writeText "extraprayer.cf" '' - prefix = "${prayer}" - var_prefix = "${stateDir}" - prayer_user = "${prayerUser}" - prayer_group = "${prayerGroup}" - sendmail_path = "/run/wrappers/bin/sendmail" - - use_http_port ${cfg.port} - - ${cfg.extraConfig} - ''; - - prayerCfg = pkgs.runCommand "prayer.cf" { preferLocalBuild = true; } '' - # We have to remove the http_port 80, or it will start a server there - cat ${prayer}/etc/prayer.cf | grep -v http_port > $out - cat ${prayerExtraCfg} >> $out - ''; - -in - -{ - - ###### interface - - options = { - - services.prayer = { - - enable = mkEnableOption (lib.mdDoc "the prayer webmail http server"); - - port = mkOption { - default = 2080; - type = types.port; - description = lib.mdDoc '' - Port the prayer http server is listening to. - ''; - }; - - extraConfig = mkOption { - type = types.lines; - default = "" ; - description = lib.mdDoc '' - Extra configuration. Contents will be added verbatim to the configuration file. - ''; - }; - }; - - }; - - - ###### implementation - - config = mkIf config.services.prayer.enable { - environment.systemPackages = [ prayer ]; - - users.users.${prayerUser} = - { uid = config.ids.uids.prayer; - description = "Prayer daemon user"; - home = stateDir; - }; - - users.groups.${prayerGroup} = - { gid = config.ids.gids.prayer; }; - - systemd.services.prayer = { - wantedBy = [ "multi-user.target" ]; - serviceConfig.Type = "forking"; - preStart = '' - mkdir -m 0755 -p ${stateDir} - chown ${prayerUser}:${prayerGroup} ${stateDir} - ''; - script = "${prayer}/sbin/prayer --config-file=${prayerCfg}"; - }; - }; -} diff --git a/nixos/tests/home-assistant.nix b/nixos/tests/home-assistant.nix index b7deb95b2c19..e97e8a467b18 100644 --- a/nixos/tests/home-assistant.nix +++ b/nixos/tests/home-assistant.nix @@ -43,6 +43,16 @@ in { psycopg2 ]; + # test loading custom components + customComponents = with pkgs.home-assistant-custom-components; [ + prometheus-sensor + ]; + + # test loading lovelace modules + customLovelaceModules = with pkgs.home-assistant-custom-lovelace-modules; [ + mini-graph-card + ]; + config = { homeassistant = { name = "Home"; @@ -114,6 +124,14 @@ in { inheritParentConfig = true; configuration.services.home-assistant.config.backup = {}; }; + + specialisation.removeCustomThings = { + inheritParentConfig = true; + configuration.services.home-assistant = { + customComponents = lib.mkForce []; + customLovelaceModules = lib.mkForce []; + }; + }; }; testScript = { nodes, ... }: let @@ -161,6 +179,14 @@ in { hass.wait_for_open_port(8123) hass.succeed("curl --fail http://localhost:8123/lovelace") + with subtest("Check that custom components get installed"): + hass.succeed("test -f ${configDir}/custom_components/prometheus_sensor/manifest.json") + hass.wait_until_succeeds("journalctl -u home-assistant.service | grep -q 'We found a custom integration prometheus_sensor which has not been tested by Home Assistant'") + + with subtest("Check that lovelace modules are referenced and fetchable"): + hass.succeed("grep -q 'mini-graph-card-bundle.js' '${configDir}/ui-lovelace.yaml'") + hass.succeed("curl --fail http://localhost:8123/local/nixos-lovelace-modules/mini-graph-card-bundle.js") + with subtest("Check that optional dependencies are in the PYTHONPATH"): env = get_unit_property("Environment") python_path = env.split("PYTHONPATH=")[1].split()[0] @@ -200,6 +226,13 @@ in { for domain in ["backup"]: assert f"Setup of domain {domain} took" in journal, f"{domain} setup missing" + with subtest("Check custom components and custom lovelace modules get removed"): + cursor = get_journal_cursor() + hass.succeed("${system}/specialisation/removeCustomThings/bin/switch-to-configuration test") + hass.fail("grep -q 'mini-graph-card-bundle.js' '${configDir}/ui-lovelace.yaml'") + hass.fail("test -f ${configDir}/custom_components/prometheus_sensor/manifest.json") + wait_for_homeassistant(cursor) + with subtest("Check that no errors were logged"): hass.fail("journalctl -u home-assistant -o cat | grep -q ERROR") diff --git a/pkgs/applications/audio/vorbis-tools/default.nix b/pkgs/applications/audio/vorbis-tools/default.nix index 46e67c7dc33e..877f670d6861 100644 --- a/pkgs/applications/audio/vorbis-tools/default.nix +++ b/pkgs/applications/audio/vorbis-tools/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchurl, libogg, libvorbis, libao, pkg-config, curl +{ lib, stdenv, fetchurl, fetchpatch, libogg, libvorbis, libao, pkg-config, curl, libiconv , speex, flac , autoreconfHook }: @@ -11,12 +11,18 @@ stdenv.mkDerivation rec { sha256 = "1c7h4ivgfdyygz2hyh6nfibxlkz8kdk868a576qkkjgj5gn78xyv"; }; - nativeBuildInputs = [ autoreconfHook pkg-config ]; - buildInputs = [ libogg libvorbis libao curl speex flac ]; + patches = lib.optionals stdenv.cc.isClang [ + # Fixes a call to undeclared function `utf8_decode`. + # https://github.com/xiph/vorbis-tools/pull/33 + (fetchpatch { + url = "https://github.com/xiph/vorbis-tools/commit/8a645f78b45ae7e370c0dc2a52d0f2612aa6110b.patch"; + hash = "sha256-RkT9Xa0pRu/oO9E9qhDa17L0luWgYHI2yINIkPZanmI="; + }) + ]; - env = lib.optionalAttrs stdenv.cc.isClang { - NIX_CFLAGS_COMPILE = "-Wno-error=implicit-function-declaration"; - }; + nativeBuildInputs = [ autoreconfHook pkg-config ]; + buildInputs = [ libogg libvorbis libao curl speex flac ] + ++ lib.optionals stdenv.isDarwin [ libiconv ]; meta = with lib; { description = "Extra tools for Ogg-Vorbis audio codec"; diff --git a/pkgs/applications/editors/vscode/extensions/default.nix b/pkgs/applications/editors/vscode/extensions/default.nix index d55b79fdf509..ace04b883728 100644 --- a/pkgs/applications/editors/vscode/extensions/default.nix +++ b/pkgs/applications/editors/vscode/extensions/default.nix @@ -1575,8 +1575,8 @@ let mktplcRef = { publisher = "github"; name = "copilot"; - version = "1.126.493"; - sha256 = "1an7z8z3xz2piw2xz1hdrs6l5rhpyvnjmb650ff2m4k24n01svfy"; + version = "1.135.544"; + sha256 = "sha256-OeG1nkQbQAfu8NuDEA+iaWy0ioFyXPe7Qm/CZIKPiX8="; }; meta = { @@ -1592,8 +1592,8 @@ let mktplcRef = { publisher = "github"; name = "copilot-chat"; - version = "0.3.2023061502"; - sha256 = "sha256-sUoKwlPDMz+iQbmIsD2JhyDwmUQzOyCHXaXCUaizQ7k="; + version = "0.11.2023111001"; + sha256 = "sha256-sBDvqqyq0R0ZyS81G61fI9Vd860RIjhNzCqY0bdz1mg="; }; meta = { description = "GitHub Copilot Chat is a companion extension to GitHub Copilot that houses experimental chat features"; @@ -3554,8 +3554,8 @@ let mktplcRef = { name = "uiua-vscode"; publisher = "uiua-lang"; - version = "0.0.22"; - sha256 = "sha256-fJcSJwwRVofduWEEMa5f2VrSfyONKPkFl9OW+++lSRw="; + version = "0.0.23"; + sha256 = "sha256-NauXoYTAka8qXNPYlW5g7r6NNX1x8cnvDRbEGkRsMoY="; }; meta = { description = "VSCode language extension for Uiua"; diff --git a/pkgs/applications/emulators/flycast/default.nix b/pkgs/applications/emulators/flycast/default.nix index f1874d62fba4..ee02b1eee071 100644 --- a/pkgs/applications/emulators/flycast/default.nix +++ b/pkgs/applications/emulators/flycast/default.nix @@ -6,35 +6,24 @@ , makeWrapper , alsa-lib , curl -, egl-wayland , libao -, libdecor -, libevdev -, libffi -, libGL , libpulseaudio -, libX11 -, libXext -, libxkbcommon , libzip -, mesa +, lua , miniupnpc -, udev -, vulkan-headers +, SDL2 , vulkan-loader -, wayland -, zlib }: stdenv.mkDerivation rec { pname = "flycast"; - version = "2.1"; + version = "2.2"; src = fetchFromGitHub { owner = "flyinghead"; repo = "flycast"; - rev = "V${version}"; - sha256 = "sha256-PRInOqg9OpaUVLwSj1lOxDtjpVaYehkRsp0jLrVKPyY="; + rev = "v${version}"; + sha256 = "sha256-eQMKaUaZ1b0oXre4Ouli4qIyNaG64KntyRGk3/YIopc="; fetchSubmodules = true; }; @@ -47,23 +36,16 @@ stdenv.mkDerivation rec { buildInputs = [ alsa-lib curl - egl-wayland libao - libdecor - libevdev - libffi - libGL libpulseaudio - libX11 - libXext - libxkbcommon libzip - mesa # for libgbm + lua miniupnpc - udev - vulkan-headers - wayland - zlib + SDL2 + ]; + + cmakeFlags = [ + "-DUSE_HOST_SDL=ON" ]; postFixup = '' diff --git a/pkgs/applications/gis/grass/clang-integer-conversion.patch b/pkgs/applications/gis/grass/clang-integer-conversion.patch new file mode 100644 index 000000000000..85145f45c37d --- /dev/null +++ b/pkgs/applications/gis/grass/clang-integer-conversion.patch @@ -0,0 +1,21 @@ +diff -ur a/db/drivers/mysql/db.c b/db/drivers/mysql/db.c +--- a/db/drivers/mysql/db.c 1969-12-31 19:00:01.000000000 -0500 ++++ b/db/drivers/mysql/db.c 2023-11-09 23:26:25.329700495 -0500 +@@ -52,9 +52,16 @@ + + db_get_login2("mysql", name, &user, &password, &host, &port); + ++ const char* errstr; ++ unsigned int port_number = (unsigned int)strtonum(port, 0, 65536, &errstr); ++ if (errstr != NULL) { ++ db_d_append_error("%s", errstr); ++ return DB_FAILED; ++ } ++ + connection = mysql_init(NULL); + res = mysql_real_connect(connection, host, user, password, +- connpar.dbname, port, NULL, 0); ++ connpar.dbname, port_number, NULL, 0); + + if (res == NULL) { + db_d_append_error("%s\n%s", _("Connection failed."), diff --git a/pkgs/applications/gis/grass/default.nix b/pkgs/applications/gis/grass/default.nix index 0f250a80b970..cd0d6dbc9386 100644 --- a/pkgs/applications/gis/grass/default.nix +++ b/pkgs/applications/gis/grass/default.nix @@ -81,12 +81,13 @@ stdenv.mkDerivation (finalAttrs: { strictDeps = true; - # On Darwin the installer tries to symlink the help files into a system - # directory - patches = [ ./no_symbolic_links.patch ]; + patches = lib.optionals stdenv.isDarwin [ + # Fix conversion of const char* to unsigned int. + ./clang-integer-conversion.patch + ]; # Correct mysql_config query - patchPhase = '' + postPatch = '' substituteInPlace configure --replace "--libmysqld-libs" "--libs" ''; diff --git a/pkgs/applications/gis/grass/no_symbolic_links.patch b/pkgs/applications/gis/grass/no_symbolic_links.patch deleted file mode 100644 index ef09b97b7037..000000000000 --- a/pkgs/applications/gis/grass/no_symbolic_links.patch +++ /dev/null @@ -1,37 +0,0 @@ -diff --git a/include/Make/Install.make b/include/Make/Install.make -index 0aba138..8ba74bc 100644 ---- a/include/Make/Install.make -+++ b/include/Make/Install.make -@@ -116,11 +116,6 @@ real-install: | $(INST_DIR) $(UNIX_BIN) - -$(INSTALL) config.status $(INST_DIR)/config.status - -$(CHMOD) -R a+rX $(INST_DIR) 2>/dev/null - --ifneq ($(findstring darwin,$(ARCH)),) -- @# enable OSX Help Viewer -- @/bin/ln -sfh "$(INST_DIR)/docs/html" /Library/Documentation/Help/GRASS-$(GRASS_VERSION_MAJOR).$(GRASS_VERSION_MINOR) --endif -- - $(INST_DIR) $(UNIX_BIN): - $(MAKE_DIR_CMD) $@ - -diff --git a/macosx/app/build_html_user_index.sh b/macosx/app/build_html_user_index.sh -index 04e63eb..c9d9c2c 100755 ---- a/macosx/app/build_html_user_index.sh -+++ b/macosx/app/build_html_user_index.sh -@@ -140,7 +140,6 @@ else - # echo "