Merge pull request #239622 from corngood/dotnet-misc

dotnet: misc fixes
This commit is contained in:
Sandro 2023-06-28 23:41:18 +02:00 committed by GitHub
commit 198bd0a4a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 119 additions and 72 deletions

View File

@ -123,7 +123,7 @@ To package Dotnet applications, you can use `buildDotnetModule`. This has simila
* `dotnetPackFlags` can be used to pass flags to `dotnet pack`. Used only if `packNupkg` is set to `true`.
* `dotnetFlags` can be used to pass flags to all of the above phases.
When packaging a new application, you need to fetch its dependencies. You can run `nix-build -A package.fetch-deps` to generate a script that will build a lockfile for you. After running the script you should have the location of the generated lockfile printed to the console, which can be copied to a stable directory. Then set `nugetDeps = ./deps.nix` and you're ready to build the derivation.
When packaging a new application, you need to fetch its dependencies. Create an empty `deps.nix`, set `nugetDeps = ./deps.nix`, then run `nix-build -A package.fetch-deps` to generate a script that will build the lockfile for you.
Here is an example `default.nix`, using some of the previously discussed arguments:
```nix

View File

@ -0,0 +1,72 @@
/*
To run:
nix-shell maintainers/scripts/update-dotnet-lockfiles.nix
This script finds all the derivations in nixpkgs that have a 'fetch-deps'
attribute, and runs all of them sequentially. This is useful to test changes
to 'fetch-deps', 'nuget-to-nix', or other changes to the dotnet build
infrastructure. Regular updates should be done through the individual packages
update scripts.
*/
let
pkgs = import ../.. {};
inherit (pkgs) lib;
packagesWith = cond: pkgs:
let
packagesWithInner = attrs:
lib.unique (
lib.concatLists (
lib.mapAttrsToList (name: elem:
let
result = builtins.tryEval elem;
in
if result.success then
let
value = result.value;
in
if lib.isDerivation value then
lib.optional (cond value) value
else
if lib.isAttrs value && (value.recurseForDerivations or false || value.recurseForRelease or false) then
packagesWithInner value
else []
else []) attrs));
in
packagesWithInner pkgs;
packages =
packagesWith (pkgs: pkgs ? fetch-deps) pkgs;
helpText = ''
Please run:
% nix-shell maintainers/scripts/update-dotnet-lockfiles.nix
'';
fetchScripts = map (p: p.fetch-deps) packages;
in pkgs.stdenv.mkDerivation {
name = "nixpkgs-update-dotnet-lockfiles";
buildCommand = ''
echo ""
echo "----------------------------------------------------------------"
echo ""
echo "Not possible to update packages using \`nix-build\`"
echo ""
echo "${helpText}"
echo "----------------------------------------------------------------"
exit 1
'';
shellHook = ''
unset shellHook # do not contaminate nested shells
set -e
for x in $fetchScripts; do
$x
done
exit
'';
inherit fetchScripts;
}

View File

@ -9,7 +9,6 @@ set -euo pipefail
trap 'echo "Error at ${BASH_SOURCE[0]}:$LINENO"' ERR
pkgName=$1
depsFile=$2
: ${getVersionFromTags:=}
: ${refetch:=}
@ -41,7 +40,7 @@ fi
if [[ $newVersion == $oldVersion && ! $refetch ]]; then
echo "nixpkgs already has the latest version $newVersion"
echo "Run this script with env var refetch=1 to re-verify the content hash via GPG"
echo "and to recreate $(basename "$depsFile"). This is useful for reviewing a version update."
echo "and to recreate deps.nix. This is useful for reviewing a version update."
exit 0
fi
@ -74,4 +73,4 @@ fi
echo
# Create deps file
$(nix-build "$nixpkgs" -A $pkgName.fetch-deps --no-out-link) "$depsFile"
$(nix-build "$nixpkgs" -A $pkgName.fetch-deps --no-out-link)

View File

@ -3,8 +3,6 @@
set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
DEPS_FILE="$(realpath "./deps.nix")"
# provide a github token so you don't get rate limited
# if you use gh cli you can use:
# `export GITHUB_TOKEN="$(cat ~/.config/gh/config.yml | yq '.hosts."github.com".oauth_token' -r)"`
@ -75,4 +73,4 @@ fi
echo "building Nuget lockfile"
$(nix-build -A ryujinx.fetch-deps --no-out-link) "$DEPS_FILE"
$(nix-build -A ryujinx.fetch-deps --no-out-link)

View File

@ -26,7 +26,6 @@ buildDotnetModule rec {
# How-to update deps:
# $ nix-build -A pinta.fetch-deps
# $ ./result
# $ cp /tmp/Pinta-deps.nix ./pkgs/applications/graphics/pinta/deps.nix
# TODO: create update script
nugetDeps = ./deps.nix;

View File

@ -3,8 +3,6 @@
set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
deps_file="$(realpath ./deps.nix)"
new_version="$(curl ${GITHUB_TOKEN:+" -u \":$GITHUB_TOKEN\""} -s "https://api.github.com/repos/JustArchiNET/ArchiSteamFarm/releases" | jq -r 'map(select(.prerelease == false)) | .[0].tag_name')"
old_version="$(sed -nE 's/\s*version = "(.*)".*/\1/p' ./default.nix)"
@ -22,7 +20,7 @@ if [[ "${1:-}" != "--deps-only" ]]; then
update-source-version ArchiSteamFarm "$new_version"
fi
$(nix-build -A ArchiSteamFarm.fetch-deps --no-out-link) "$deps_file"
$(nix-build -A ArchiSteamFarm.fetch-deps --no-out-link)
popd
"$asf_path/web-ui/update.sh"

View File

@ -20,5 +20,4 @@ hash="$(nix-prefetch ./.)"
sed -i -Ee "s/hash = \"sha256-[A-Za-z0-9=]{44}\"/hash = \"${hash}\"/" default.nix
nugetDeps="$(realpath ./deps.nix)"
$(nix-build ../../../.. -A git-credential-manager.fetch-deps --no-out-link) "$nugetDeps"
$(nix-build ../../../.. -A git-credential-manager.fetch-deps --no-out-link)

View File

@ -112,7 +112,11 @@ let
if (nugetDeps != null) then
if lib.isDerivation nugetDeps
then nugetDeps
else mkNugetDeps { inherit name; nugetDeps = import nugetDeps; }
else mkNugetDeps {
inherit name;
nugetDeps = import nugetDeps;
sourceFile = nugetDeps;
}
else throw "Defining the `nugetDeps` attribute is required, as to lock the NuGet dependencies. This file can be generated by running the `passthru.fetch-deps` script.";
# contains the actual package dependencies
@ -138,6 +142,8 @@ let
name = "${name}-nuget-source";
paths = [ dependenciesSource sdkSource ];
};
nugetDepsFile = _nugetDeps.sourceFile;
in
stdenvNoCC.mkDerivation (args // {
nativeBuildInputs = args.nativeBuildInputs or [ ] ++ [
@ -166,7 +172,7 @@ stdenvNoCC.mkDerivation (args // {
passthru = {
inherit nuget-source;
} // lib.optionalAttrs (nugetDepsFile != null) {
fetch-deps =
let
flags = dotnetFlags ++ dotnetRestoreFlags;
@ -180,8 +186,8 @@ stdenvNoCC.mkDerivation (args // {
# Note that toString is necessary here as it results in the path at
# eval time (i.e. to the file in your local Nixpkgs checkout) rather
# than the Nix store path of the path after it's been imported.
if lib.isPath nugetDeps && !lib.hasPrefix "${builtins.storeDir}/" (toString nugetDeps)
then toString nugetDeps
if lib.isPath nugetDepsFile && !lib.hasPrefix "${builtins.storeDir}/" (toString nugetDepsFile)
then toString nugetDepsFile
else ''$(mktemp -t "${pname}-deps-XXXXXX.nix")'';
in
writeShellScript "fetch-${pname}-deps" ''
@ -208,7 +214,7 @@ stdenvNoCC.mkDerivation (args // {
if [[ ''${TMPDIR:-} == /run/user/* ]]; then
# /run/user is usually a tmpfs in RAM, which may be too small
# to store all downloaded dotnet packages
TMPDIR=
unset TMPDIR
fi
export tmp=$(mktemp -td "deps-${pname}-XXXXXX")
@ -281,13 +287,15 @@ stdenvNoCC.mkDerivation (args // {
echo "Succesfully restored project"
echo "Writing lockfile..."
echo -e "# This file was automatically generated by passthru.fetch-deps.\n# Please dont edit it manually, your changes might get overwritten!\n" > "$depsFile"
excluded_sources="${lib.concatStringsSep " " sdkDeps}"
for excluded_source in ''${excluded_sources[@]}; do
ls "$excluded_source" >> "$tmp/excluded_list"
done
nuget-to-nix "$tmp/nuget_pkgs" "$tmp/excluded_list" >> "$depsFile"
tmpFile="$tmp"/deps.nix
echo -e "# This file was automatically generated by passthru.fetch-deps.\n# Please dont edit it manually, your changes might get overwritten!\n" > "$tmpFile"
nuget-to-nix "$tmp/nuget_pkgs" "$tmp/excluded_list" >> "$tmpFile"
mv "$tmpFile" "$depsFile"
echo "Succesfully wrote lockfile to $depsFile"
'';
} // args.passthru or { };

View File

@ -40,15 +40,15 @@ dotnetInstallHook() {
dotnetPack() {
local -r project="${1-}"
env dotnet pack ${project-} \
-p:ContinuousIntegrationBuild=true \
-p:Deterministic=true \
--output "$out/share" \
--configuration "@buildType@" \
--no-build \
env dotnet pack ${project-} \
-p:ContinuousIntegrationBuild=true \
-p:Deterministic=true \
--output "$out/share" \
--configuration "@buildType@" \
--no-build \
--runtime "@runtimeId@" \
${dotnetPackFlags[@]} \
${dotnetFlags[@]}
${dotnetPackFlags[@]} \
${dotnetFlags[@]}
}
if (( "${#projectFile[@]}" == 0 )); then

View File

@ -1,5 +1,5 @@
{ linkFarmFromDrvs, fetchurl }:
{ name, nugetDeps }:
{ name, nugetDeps, sourceFile ? null }:
linkFarmFromDrvs "${name}-nuget-deps" (nugetDeps {
fetchNuGet = { pname, version, sha256
, url ? "https://www.nuget.org/api/v2/package/${pname}/${version}" }:
@ -7,4 +7,6 @@ linkFarmFromDrvs "${name}-nuget-deps" (nugetDeps {
name = "${pname}.${version}.nupkg";
inherit url sha256;
};
})
}) // {
inherit sourceFile;
}

View File

@ -1,6 +1,7 @@
#!@runtimeShell@
set -euo pipefail
shopt -s nullglob
export PATH="@binPath@"
# used for glob ordering of package names

View File

@ -21,7 +21,7 @@ buildDotnetModule rec {
projectFile = [ "src/NuGet/Microsoft.Net.Compilers.Toolset/Microsoft.Net.Compilers.Toolset.Package.csproj" ];
nugetDeps = ./extended-deps.nix;
nugetDeps = ./deps.nix;
dontDotnetFixup = true;

View File

@ -19,6 +19,7 @@
(fetchNuGet { pname = "Microsoft.CodeAnalysis.PublicApiAnalyzers"; version = "3.3.4-beta1.22160.2"; sha256 = "0ih091ls51x5k9q998g14pfy4r3g1ygvzihj1gkrl79wydn7b0n3"; url = "https://pkgs.dev.azure.com/dnceng/9ee6d478-d288-47f7-aacc-f6e6d082ae6d/_packaging/49e5305d-d845-4a14-9d69-6f5dbfb9570c/nuget/v3/flat2/microsoft.codeanalysis.publicapianalyzers/3.3.4-beta1.22160.2/microsoft.codeanalysis.publicapianalyzers.3.3.4-beta1.22160.2.nupkg"; })
(fetchNuGet { pname = "Microsoft.CSharp"; version = "4.7.0"; sha256 = "0gd67zlw554j098kabg887b5a6pq9kzavpa3jjy5w53ccjzjfy8j"; url = "https://pkgs.dev.azure.com/dnceng/9ee6d478-d288-47f7-aacc-f6e6d082ae6d/_packaging/45bacae2-5efb-47c8-91e5-8ec20c22b4f8/nuget/v3/flat2/microsoft.csharp/4.7.0/microsoft.csharp.4.7.0.nupkg"; })
(fetchNuGet { pname = "Microsoft.DiaSymReader.Native"; version = "17.0.0-beta1.21524.1"; sha256 = "0gash3xgzvcb78w2xqv003l0cld199zpfilnjbagwbr5ikdh6f3s"; url = "https://pkgs.dev.azure.com/dnceng/9ee6d478-d288-47f7-aacc-f6e6d082ae6d/_packaging/45bacae2-5efb-47c8-91e5-8ec20c22b4f8/nuget/v3/flat2/microsoft.diasymreader.native/17.0.0-beta1.21524.1/microsoft.diasymreader.native.17.0.0-beta1.21524.1.nupkg"; })
(fetchNuGet { pname = "Microsoft.DotNet.Arcade.Sdk"; version = "7.0.0-beta.22171.2"; sha256 = "15y26skavivkwhnpfa984if3cnpnllbbwbdsjiyfdcalp32fhmjq"; url = "https://pkgs.dev.azure.com/dnceng/9ee6d478-d288-47f7-aacc-f6e6d082ae6d/_packaging/1a5f89f6-d8da-4080-b15f-242650c914a8/nuget/v3/flat2/microsoft.dotnet.arcade.sdk/7.0.0-beta.22171.2/microsoft.dotnet.arcade.sdk.7.0.0-beta.22171.2.nupkg"; })
(fetchNuGet { pname = "Microsoft.DotNet.XliffTasks"; version = "1.0.0-beta.22169.1"; sha256 = "12fcin3d4m0lawla9fflz9f2qispzgvzf1mwkpscmlk5lnvb0riw"; url = "https://pkgs.dev.azure.com/dnceng/9ee6d478-d288-47f7-aacc-f6e6d082ae6d/_packaging/1a5f89f6-d8da-4080-b15f-242650c914a8/nuget/v3/flat2/microsoft.dotnet.xlifftasks/1.0.0-beta.22169.1/microsoft.dotnet.xlifftasks.1.0.0-beta.22169.1.nupkg"; })
(fetchNuGet { pname = "Microsoft.Net.Compilers.Toolset"; version = "4.2.0-1.final"; sha256 = "02zas22hj29gv2w7h74q786i0cvxffgwqai21ri0zj41nb2hwhyq"; url = "https://pkgs.dev.azure.com/dnceng/9ee6d478-d288-47f7-aacc-f6e6d082ae6d/_packaging/45bacae2-5efb-47c8-91e5-8ec20c22b4f8/nuget/v3/flat2/microsoft.net.compilers.toolset/4.2.0-1.final/microsoft.net.compilers.toolset.4.2.0-1.final.nupkg"; })
(fetchNuGet { pname = "Microsoft.NETCore.App.Host.linux-arm64"; version = "3.1.32"; sha256 = "1zygp70xrk5zggs3q4a6yc6jfdwzcsjjsapqpwn6qyx35m69b72p"; url = "https://pkgs.dev.azure.com/dnceng/9ee6d478-d288-47f7-aacc-f6e6d082ae6d/_packaging/45bacae2-5efb-47c8-91e5-8ec20c22b4f8/nuget/v3/flat2/microsoft.netcore.app.host.linux-arm64/3.1.32/microsoft.netcore.app.host.linux-arm64.3.1.32.nupkg"; })

View File

@ -1,11 +0,0 @@
# Some required nuget packages are not picked up by the deps generation script,
# since they are referenced as a SDK reference, which unfortunately only gets
# downloaded during build time. So we include them manually.
{ fetchNuGet }: (import ./deps.nix { inherit fetchNuGet; }) ++ [
(fetchNuGet rec {
pname = "Microsoft.DotNet.Arcade.Sdk";
version = "7.0.0-beta.22171.2";
url = "https://pkgs.dev.azure.com/dnceng/9ee6d478-d288-47f7-aacc-f6e6d082ae6d/_packaging/1a5f89f6-d8da-4080-b15f-242650c914a8/nuget/v3/flat2/microsoft.dotnet.arcade.sdk/${version}/microsoft.dotnet.arcade.sdk.${version}.nupkg";
sha256 = "15y26skavivkwhnpfa984if3cnpnllbbwbdsjiyfdcalp32fhmjq";
})
]

View File

@ -15,5 +15,5 @@ if [[ "$latestVersion" == "$currentVersion" ]]; then
fi
update-source-version github-runner "$latestVersion"
$(nix-build -A github-runner.fetch-deps --no-out-link) "$(dirname "$BASH_SOURCE")/deps.nix"
$(nix-build -A github-runner.fetch-deps --no-out-link)

View File

@ -15,4 +15,4 @@ fi
update-source-version marksman "$version"
$(nix-build -A marksman.fetch-deps --no-out-link) "$(dirname -- "${BASH_SOURCE[0]}")/deps.nix"
$(nix-build -A marksman.fetch-deps --no-out-link)

View File

@ -6,8 +6,6 @@ set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
deps_file="$(realpath "./deps.nix")"
new_version="$(curl -s "https://api.github.com/repos/OmniSharp/omnisharp-roslyn/releases?per_page=1" | jq -r '.[0].name')"
old_version="$(sed -nE 's/\s*version = "(.*)".*/\1/p' ./default.nix)"
@ -19,4 +17,4 @@ fi
cd ../../../..
update-source-version omnisharp-roslyn "${new_version//v}"
$(nix-build -A omnisharp-roslyn.fetch-deps --no-out-link) "$deps_file"
$(nix-build -A omnisharp-roslyn.fetch-deps --no-out-link)

View File

@ -3,8 +3,6 @@
set -eo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
deps_file="$(realpath "./deps.nix")"
new_version="$(curl -s "https://api.github.com/repos/ppy/osu/releases?per_page=1" | jq -r '.[0].name')"
old_version="$(sed -nE 's/\s*version = "(.*)".*/\1/p' ./default.nix)"
if [[ "$new_version" == "$old_version" ]]; then
@ -18,4 +16,4 @@ if [[ "$1" != "--deps-only" ]]; then
update-source-version osu-lazer "$new_version"
fi
$(nix-build . -A osu-lazer.fetch-deps --no-out-link) "$deps_file"
$(nix-build . -A osu-lazer.fetch-deps --no-out-link)

View File

@ -3,8 +3,6 @@
set -eo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
deps_file="$(realpath "./deps.nix")"
new_version="$(curl -s "https://api.github.com/repos/space-wizards/SS14.Launcher/releases?per_page=1" | jq -r '.[0].tag_name' | sed 's/v//')"
old_version="$(sed -nE 's/\s*version = "(.*)".*/\1/p' ./space-station-14-launcher.nix)"
@ -15,4 +13,4 @@ fi
cd ../../..
update-source-version space-station-14-launcher.unwrapped "$new_version"
$(nix-build -A space-station-14-launcher.fetch-deps --no-out-link) "$deps_file"
$(nix-build -A space-station-14-launcher.fetch-deps --no-out-link)

View File

@ -3,8 +3,6 @@
set -eo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
deps_file="$(realpath "./deps.nix")"
new_version="$(curl -s "https://api.github.com/repos/jackett/jackett/releases?per_page=1" | jq -r '.[0].name')"
old_version="$(sed -nE 's/\s*version = "(.*)".*/\1/p' ./default.nix)"
@ -15,4 +13,4 @@ fi
cd ../../..
update-source-version jackett "${new_version//v}"
$(nix-build -A jackett.fetch-deps --no-out-link) "$deps_file"
$(nix-build -A jackett.fetch-deps --no-out-link)

View File

@ -11,10 +11,6 @@ if [[ "$currentVersion" == "$latestVersion" ]]; then
exit 0
fi
pushd "$(dirname "${BASH_SOURCE[0]}")"
nugetDepsFile=$(realpath ./nuget-deps.nix)
popd
update-source-version jellyfin "$latestVersion"
$(nix-build . -A jellyfin.fetch-deps --no-out-link) "$nugetDepsFile"
$(nix-build . -A jellyfin.fetch-deps --no-out-link)

View File

@ -6,8 +6,6 @@ set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
deps_file="$(realpath "./deps.nix")"
new_version="$(curl -s "https://api.github.com/repos/EventStore/EventStore/releases/latest" | jq -r '.name')"
new_version="${new_version#oss-v}"
old_version="$(sed -nE 's/\s*version = "(.*)".*/\1/p' ./default.nix)"
@ -20,4 +18,4 @@ fi
cd ../../../..
update-source-version eventstore "${new_version//v}"
$(nix-build -A eventstore.fetch-deps --no-out-link) "$deps_file"
$(nix-build -A eventstore.fetch-deps --no-out-link)

View File

@ -3,8 +3,6 @@
set -eo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
deps_file="$(realpath "./deps.nix")"
new_version="$(curl ${GITHUB_TOKEN:+" -u \":$GITHUB_TOKEN\""} -s "https://api.github.com/repos/OpenTabletDriver/OpenTabletDriver/releases" | jq -r 'map(select(.prerelease == false)) | .[0].tag_name' | cut -c2-)"
old_version="$(sed -nE 's/\s*version = "(.*)".*/\1/p' ./default.nix)"
@ -23,4 +21,4 @@ sed -i ./default.nix -re "s|\"$oldDebSha256\"|\"$newDebSha256\"|"
pushd ../../../..
update-source-version opentabletdriver "$new_version"
$(nix-build -A opentabletdriver.fetch-deps --no-out-link) "$deps_file"
$(nix-build -A opentabletdriver.fetch-deps --no-out-link)

View File

@ -3,8 +3,6 @@
set -eo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
deps_file="$(realpath "./deps.nix")"
new_version="$(curl -s "https://api.github.com/repos/tyrrrz/DiscordChatExporter/releases?per_page=1" | jq -r '.[0].name')"
old_version="$(sed -nE 's/\s*version = "(.*)".*/\1/p' ./default.nix)"
if [[ "$new_version" == "$old_version" ]]; then
@ -14,4 +12,4 @@ fi
cd ../../../..
update-source-version discordchatexporter-cli "$new_version"
$(nix-build -A discordchatexporter-cli.fetch-deps --no-out-link) "$deps_file"
$(nix-build -A discordchatexporter-cli.fetch-deps --no-out-link)

View File

@ -3,7 +3,6 @@
set -eou pipefail
depsFile="$(realpath "$(dirname "${BASH_SOURCE[0]}")/deps.nix")"
currentVersion="$(nix eval --raw -f . depotdownloader.version)"
latestVersion="$(curl -s ${GITHUB_TOKEN:+-u ":$GITHUB_TOKEN"} "https://api.github.com/repos/SteamRE/DepotDownloader/releases?per_page=1" \
| jq -r '.[].name' | cut -d' ' -f2)"
@ -14,4 +13,4 @@ if [[ "$currentVersion" = "$latestVersion" ]]; then
fi
update-source-version depotdownloader "$latestVersion"
$(nix-build -A depotdownloader.fetch-deps --no-out-link) "$depsFile"
$(nix-build -A depotdownloader.fetch-deps --no-out-link)