{ lib ? import ../. }: let inherit (builtins) isAttrs isPath isString pathExists readDir seq split trace typeOf ; inherit (lib.attrsets) attrNames attrValues mapAttrs setAttrByPath zipAttrsWith ; inherit (lib.filesystem) pathType ; inherit (lib.lists) all commonPrefix drop elemAt filter findFirst findFirstIndex foldl' head length sublist tail ; inherit (lib.path) append splitRoot ; inherit (lib.path.subpath) components join ; inherit (lib.strings) isStringLike concatStringsSep substring stringLength ; in # Rare case of justified usage of rec: # - This file is internal, so the return value doesn't matter, no need to make things overridable # - The functions depend on each other # - We want to expose all of these functions for easy testing rec { # If you change the internal representation, make sure to: # - Increment this version # - Add an additional migration function below # - Update the description of the internal representation in ./README.md _currentVersion = 3; # Migrations between versions. The 0th element converts from v0 to v1, and so on migrations = [ # Convert v0 into v1: Add the _internalBase{Root,Components} attributes ( filesetV0: let parts = splitRoot filesetV0._internalBase; in filesetV0 // { _internalVersion = 1; _internalBaseRoot = parts.root; _internalBaseComponents = components parts.subpath; } ) # Convert v1 into v2: filesetTree's can now also omit attributes to signal paths not being included ( filesetV1: # This change is backwards compatible (but not forwards compatible, so we still need a new version) filesetV1 // { _internalVersion = 2; } ) # Convert v2 into v3: filesetTree's now have a representation for an empty file set without a base path ( filesetV2: filesetV2 // { # All v1 file sets are not the new empty file set _internalIsEmptyWithoutBase = false; _internalVersion = 3; } ) ]; _noEvalMessage = '' lib.fileset: Directly evaluating a file set is not supported. To turn it into a usable source, use `lib.fileset.toSource`. To pretty-print the contents, use `lib.fileset.trace` or `lib.fileset.traceVal`.''; # The empty file set without a base path _emptyWithoutBase = { _type = "fileset"; _internalVersion = _currentVersion; # The one and only! _internalIsEmptyWithoutBase = true; # Due to alphabetical ordering, this is evaluated last, # which makes the nix repl output nicer than if it would be ordered first. # It also allows evaluating it strictly up to this error, which could be useful _noEval = throw _noEvalMessage; }; # Create a fileset, see ./README.md#fileset # Type: path -> filesetTree -> fileset _create = base: tree: let # Decompose the base into its components # See ../path/README.md for why we're not just using `toString` parts = splitRoot base; in { _type = "fileset"; _internalVersion = _currentVersion; _internalIsEmptyWithoutBase = false; _internalBase = base; _internalBaseRoot = parts.root; _internalBaseComponents = components parts.subpath; _internalTree = tree; # Due to alphabetical ordering, this is evaluated last, # which makes the nix repl output nicer than if it would be ordered first. # It also allows evaluating it strictly up to this error, which could be useful _noEval = throw _noEvalMessage; }; # Coerce a value to a fileset, erroring when the value cannot be coerced. # The string gives the context for error messages. # Type: String -> (fileset | Path) -> fileset _coerce = context: value: if value._type or "" == "fileset" then if value._internalVersion > _currentVersion then throw '' ${context} is a file set created from a future version of the file set library with a different internal representation: - Internal version of the file set: ${toString value._internalVersion} - Internal version of the library: ${toString _currentVersion} Make sure to update your Nixpkgs to have a newer version of `lib.fileset`.'' else if value._internalVersion < _currentVersion then let # Get all the migration functions necessary to convert from the old to the current version migrationsToApply = sublist value._internalVersion (_currentVersion - value._internalVersion) migrations; in foldl' (value: migration: migration value) value migrationsToApply else value else if ! isPath value then if isStringLike value then throw '' ${context} ("${toString value}") is a string-like value, but it should be a file set or a path instead. Paths represented as strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.'' else throw '' ${context} is of type ${typeOf value}, but it should be a file set or a path instead.'' else if ! pathExists value then throw '' ${context} (${toString value}) does not exist.'' else _singleton value; # Coerce many values to filesets, erroring when any value cannot be coerced, # or if the filesystem root of the values doesn't match. # Type: String -> [ { context :: String, value :: fileset | Path } ] -> [ fileset ] _coerceMany = functionContext: list: let filesets = map ({ context, value }: _coerce "${functionContext}: ${context}" value ) list; # Find the first value with a base, there may be none! firstWithBase = findFirst (fileset: ! fileset._internalIsEmptyWithoutBase) null filesets; # This value is only accessed if first != null firstBaseRoot = firstWithBase._internalBaseRoot; # Finds the first element with a filesystem root different than the first element, if any differentIndex = findFirstIndex (fileset: # The empty value without a base doesn't have a base path ! fileset._internalIsEmptyWithoutBase && firstBaseRoot != fileset._internalBaseRoot ) null filesets; in # Only evaluates `differentIndex` if there are any elements with a base if firstWithBase != null && differentIndex != null then throw '' ${functionContext}: Filesystem roots are not the same: ${(head list).context}: root "${toString firstBaseRoot}" ${(elemAt list differentIndex).context}: root "${toString (elemAt filesets differentIndex)._internalBaseRoot}" Different roots are not supported.'' else filesets; # Create a file set from a path. # Type: Path -> fileset _singleton = path: let type = pathType path; in if type == "directory" then _create path type else # This turns a file path ./default.nix into a fileset with # - _internalBase: ./. # - _internalTree: { # "default.nix" = ; # } # See ./README.md#single-files _create (dirOf path) { ${baseNameOf path} = type; }; # Expand a directory representation to an equivalent one in attribute set form. # All directory entries are included in the result. # Type: Path -> filesetTree -> { = filesetTree; } _directoryEntries = path: value: if value == "directory" then readDir path else # Set all entries not present to null mapAttrs (name: value: null) (readDir path) // value; /* A normalisation of a filesetTree suitable filtering with `builtins.path`: - Replace all directories that have no files with `null`. This removes directories that would be empty - Replace all directories with all files with `"directory"`. This speeds up the source filter function Note that this function is strict, it evaluates the entire tree Type: Path -> filesetTree -> filesetTree */ _normaliseTreeFilter = path: tree: if tree == "directory" || isAttrs tree then let entries = _directoryEntries path tree; normalisedSubtrees = mapAttrs (name: _normaliseTreeFilter (path + "/${name}")) entries; subtreeValues = attrValues normalisedSubtrees; in # This triggers either when all files in a directory are filtered out # Or when the directory doesn't contain any files at all if all isNull subtreeValues then null # Triggers when we have the same as a `readDir path`, so we can turn it back into an equivalent "directory". else if all isString subtreeValues then "directory" else normalisedSubtrees else tree; /* A minimal normalisation of a filesetTree, intended for pretty-printing: - If all children of a path are recursively included or empty directories, the path itself is also recursively included - If all children of a path are fully excluded or empty directories, the path itself is an empty directory - Other empty directories are represented with the special "emptyDir" string While these could be replaced with `null`, that would take another mapAttrs Note that this function is partially lazy. Type: Path -> filesetTree -> filesetTree (with "emptyDir"'s) */ _normaliseTreeMinimal = path: tree: if tree == "directory" || isAttrs tree then let entries = _directoryEntries path tree; normalisedSubtrees = mapAttrs (name: _normaliseTreeMinimal (path + "/${name}")) entries; subtreeValues = attrValues normalisedSubtrees; in # If there are no entries, or all entries are empty directories, return "emptyDir". # After this branch we know that there's at least one file if all (value: value == "emptyDir") subtreeValues then "emptyDir" # If all subtrees are fully included or empty directories # (both of which are coincidentally represented as strings), return "directory". # This takes advantage of the fact that empty directories can be represented as included directories. # Note that the tree == "directory" check allows avoiding recursion else if tree == "directory" || all (value: isString value) subtreeValues then "directory" # If all subtrees are fully excluded or empty directories, return null. # This takes advantage of the fact that empty directories can be represented as excluded directories else if all (value: isNull value || value == "emptyDir") subtreeValues then null # Mix of included and excluded entries else normalisedSubtrees else tree; # Trace a filesetTree in a pretty way when the resulting value is evaluated. # This can handle both normal filesetTree's, and ones returned from _normaliseTreeMinimal # Type: Path -> filesetTree (with "emptyDir"'s) -> Null _printMinimalTree = base: tree: let treeSuffix = tree: if isAttrs tree then "" else if tree == "directory" then " (all files in directory)" else # This does "leak" the file type strings of the internal representation, # but this is the main reason these file type strings even are in the representation! # TODO: Consider removing that information from the internal representation for performance. # The file types can still be printed by querying them only during tracing " (${tree})"; # Only for attribute set trees traceTreeAttrs = prevLine: indent: tree: foldl' (prevLine: name: let subtree = tree.${name}; # Evaluating this prints the line for this subtree thisLine = trace "${indent}- ${name}${treeSuffix subtree}" prevLine; in if subtree == null || subtree == "emptyDir" then # Don't print anything at all if this subtree is empty prevLine else if isAttrs subtree then # A directory with explicit entries # Do print this node, but also recurse traceTreeAttrs thisLine "${indent} " subtree else # Either a file, or a recursively included directory # Do print this node but no further recursion needed thisLine ) prevLine (attrNames tree); # Evaluating this will print the first line firstLine = if tree == null || tree == "emptyDir" then trace "(empty)" null else trace "${toString base}${treeSuffix tree}" null; in if isAttrs tree then traceTreeAttrs firstLine "" tree else firstLine; # Pretty-print a file set in a pretty way when the resulting value is evaluated # Type: fileset -> Null _printFileset = fileset: if fileset._internalIsEmptyWithoutBase then trace "(empty)" null else _printMinimalTree fileset._internalBase (_normaliseTreeMinimal fileset._internalBase fileset._internalTree); # Turn a fileset into a source filter function suitable for `builtins.path` # Only directories recursively containing at least one files are recursed into # Type: Path -> fileset -> (String -> String -> Bool) _toSourceFilter = fileset: let # Simplify the tree, necessary to make sure all empty directories are null # which has the effect that they aren't included in the result tree = _normaliseTreeFilter fileset._internalBase fileset._internalTree; # The base path as a string with a single trailing slash baseString = if fileset._internalBaseComponents == [] then # Need to handle the filesystem root specially "/" else "/" + concatStringsSep "/" fileset._internalBaseComponents + "/"; baseLength = stringLength baseString; # Check whether a list of path components under the base path exists in the tree. # This function is called often, so it should be fast. # Type: [ String ] -> Bool inTree = components: let recurse = index: localTree: if isAttrs localTree then # We have an attribute set, meaning this is a directory with at least one file if index >= length components then # The path may have no more components though, meaning the filter is running on the directory itself, # so we always include it, again because there's at least one file in it. true else # If we do have more components, the filter runs on some entry inside this directory, so we need to recurse # We do +2 because builtins.split is an interleaved list of the inbetweens and the matches recurse (index + 2) localTree.${elemAt components index} else # If it's not an attribute set it can only be either null (in which case it's not included) # or a string ("directory" or "regular", etc.) in which case it's included localTree != null; in recurse 0 tree; # Filter suited when there's no files empty = _: _: false; # Filter suited when there's some files # This can't be used for when there's no files, because the base directory is always included nonEmpty = path: _: let # Add a slash to the path string, turning "/foo" to "/foo/", # making sure to not have any false prefix matches below. # Note that this would produce "//" for "/", # but builtins.path doesn't call the filter function on the `path` argument itself, # meaning this function can never receive "/" as an argument pathSlash = path + "/"; in # Same as `hasPrefix pathSlash baseString`, but more efficient. # With base /foo/bar we need to include /foo: # hasPrefix "/foo/" "/foo/bar/" if substring 0 (stringLength pathSlash) baseString == pathSlash then true # Same as `! hasPrefix baseString pathSlash`, but more efficient. # With base /foo/bar we need to exclude /baz # ! hasPrefix "/baz/" "/foo/bar/" else if substring 0 baseLength pathSlash != baseString then false else # Same as `removePrefix baseString path`, but more efficient. # From the above code we know that hasPrefix baseString pathSlash holds, so this is safe. # We don't use pathSlash here because we only needed the trailing slash for the prefix matching. # With base /foo and path /foo/bar/baz this gives # inTree (split "/" (removePrefix "/foo/" "/foo/bar/baz")) # == inTree (split "/" "bar/baz") # == inTree [ "bar" "baz" ] inTree (split "/" (substring baseLength (-1) path)); in # Special case because the code below assumes that the _internalBase is always included in the result # which shouldn't be done when we have no files at all in the base # This also forces the tree before returning the filter, leads to earlier error messages if fileset._internalIsEmptyWithoutBase || tree == null then empty else nonEmpty; # Transforms the filesetTree of a file set to a shorter base path, e.g. # _shortenTreeBase [ "foo" ] (_create /foo/bar null) # => { bar = null; } _shortenTreeBase = targetBaseComponents: fileset: let recurse = index: # If we haven't reached the required depth yet if index < length fileset._internalBaseComponents then # Create an attribute set and recurse as the value, this can be lazily evaluated this way { ${elemAt fileset._internalBaseComponents index} = recurse (index + 1); } else # Otherwise we reached the appropriate depth, here's the original tree fileset._internalTree; in recurse (length targetBaseComponents); # Transforms the filesetTree of a file set to a longer base path, e.g. # _lengthenTreeBase [ "foo" "bar" ] (_create /foo { bar.baz = "regular"; }) # => { baz = "regular"; } _lengthenTreeBase = targetBaseComponents: fileset: let recurse = index: tree: # If the filesetTree is an attribute set and we haven't reached the required depth yet if isAttrs tree && index < length targetBaseComponents then # Recurse with the tree under the right component (which might not exist) recurse (index + 1) (tree.${elemAt targetBaseComponents index} or null) else # For all values here we can just return the tree itself: # tree == null -> the result is also null, everything is excluded # tree == "directory" -> the result is also "directory", # because the base path is always a directory and everything is included # isAttrs tree -> the result is `tree` # because we don't need to recurse any more since `index == length longestBaseComponents` tree; in recurse (length fileset._internalBaseComponents) fileset._internalTree; # Computes the union of a list of filesets. # The filesets must already be coerced and validated to be in the same filesystem root # Type: [ Fileset ] -> Fileset _unionMany = filesets: let # All filesets that have a base, aka not the ones that are the empty value without a base filesetsWithBase = filter (fileset: ! fileset._internalIsEmptyWithoutBase) filesets; # The first fileset that has a base. # This value is only accessed if there are at all. firstWithBase = head filesetsWithBase; # To be able to union filesetTree's together, they need to have the same base path. # Base paths can be unioned by taking their common prefix, # e.g. such that `union /foo/bar /foo/baz` has the base path `/foo` # A list of path components common to all base paths. # Note that commonPrefix can only be fully evaluated, # so this cannot cause a stack overflow due to a build-up of unevaluated thunks. commonBaseComponents = foldl' (components: el: commonPrefix components el._internalBaseComponents) firstWithBase._internalBaseComponents # We could also not do the `tail` here to avoid a list allocation, # but then we'd have to pay for a potentially expensive # but unnecessary `commonPrefix` call (tail filesetsWithBase); # The common base path assembled from a filesystem root and the common components commonBase = append firstWithBase._internalBaseRoot (join commonBaseComponents); # A list of filesetTree's that all have the same base path # This is achieved by nesting the trees into the components they have over the common base path # E.g. `union /foo/bar /foo/baz` has the base path /foo # So the tree under `/foo/bar` gets nested under `{ bar = ...; ... }`, # while the tree under `/foo/baz` gets nested under `{ baz = ...; ... }` # Therefore allowing combined operations over them. trees = map (_shortenTreeBase commonBaseComponents) filesetsWithBase; # Folds all trees together into a single one using _unionTree # We do not use a fold here because it would cause a thunk build-up # which could cause a stack overflow for a large number of trees resultTree = _unionTrees trees; in # If there's no values with a base, we have no files if filesetsWithBase == [ ] then _emptyWithoutBase else _create commonBase resultTree; # The union of multiple filesetTree's with the same base path. # Later elements are only evaluated if necessary. # Type: [ filesetTree ] -> filesetTree _unionTrees = trees: let stringIndex = findFirstIndex isString null trees; withoutNull = filter (tree: tree != null) trees; in if stringIndex != null then # If there's a string, it's always a fully included tree (dir or file), # no need to look at other elements elemAt trees stringIndex else if withoutNull == [ ] then # If all trees are null, then the resulting tree is also null null else # The non-null elements have to be attribute sets representing partial trees # We need to recurse into those zipAttrsWith (name: _unionTrees) withoutNull; # Computes the intersection of a list of filesets. # The filesets must already be coerced and validated to be in the same filesystem root # Type: Fileset -> Fileset -> Fileset _intersection = fileset1: fileset2: let # The common base components prefix, e.g. # (/foo/bar, /foo/bar/baz) -> /foo/bar # (/foo/bar, /foo/baz) -> /foo commonBaseComponentsLength = # TODO: Have a `lib.lists.commonPrefixLength` function such that we don't need the list allocation from commonPrefix here length ( commonPrefix fileset1._internalBaseComponents fileset2._internalBaseComponents ); # To be able to intersect filesetTree's together, they need to have the same base path. # Base paths can be intersected by taking the longest one (if any) # The fileset with the longest base, if any, e.g. # (/foo/bar, /foo/bar/baz) -> /foo/bar/baz # (/foo/bar, /foo/baz) -> null longestBaseFileset = if commonBaseComponentsLength == length fileset1._internalBaseComponents then # The common prefix is the same as the first path, so the second path is equal or longer fileset2 else if commonBaseComponentsLength == length fileset2._internalBaseComponents then # The common prefix is the same as the second path, so the first path is longer fileset1 else # The common prefix is neither the first nor the second path # This means there's no overlap between the two sets null; # Whether the result should be the empty value without a base resultIsEmptyWithoutBase = # If either fileset is the empty fileset without a base, the intersection is too fileset1._internalIsEmptyWithoutBase || fileset2._internalIsEmptyWithoutBase # If there is no overlap between the base paths || longestBaseFileset == null; # Lengthen each fileset's tree to the longest base prefix tree1 = _lengthenTreeBase longestBaseFileset._internalBaseComponents fileset1; tree2 = _lengthenTreeBase longestBaseFileset._internalBaseComponents fileset2; # With two filesetTree's with the same base, we can compute their intersection resultTree = _intersectTree tree1 tree2; in if resultIsEmptyWithoutBase then _emptyWithoutBase else _create longestBaseFileset._internalBase resultTree; # The intersection of two filesetTree's with the same base path # The second element is only evaluated as much as necessary. # Type: filesetTree -> filesetTree -> filesetTree _intersectTree = lhs: rhs: if isAttrs lhs && isAttrs rhs then # Both sides are attribute sets, we can recurse for the attributes existing on both sides mapAttrs (name: _intersectTree lhs.${name}) (builtins.intersectAttrs lhs rhs) else if lhs == null || isString rhs then # If the lhs is null, the result should also be null # And if the rhs is the identity element # (a string, aka it includes everything), then it's also the lhs lhs else # In all other cases it's the rhs rhs; }