colin/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #99198 from timstott/terraform-013-compatible-providers

Browse Source 
terraform-providers: Terraform 0.13 compatibility
This commit is contained in:
zimbatm 2020-10-08 21:29:04 +00:00 committed by GitHub
commit be6e9f1e90
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 1185 additions and 1355 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1163
pkgs/applications/networking/cluster/terraform-providers/data.nix
View File

File diff suppressed because it is too large Load Diff

21
pkgs/applications/networking/cluster/terraform-providers/default.nix
View File

@ -2,11 +2,16 @@
, buildGoPackage
, fetchFromGitHub
, callPackage
, runtimeShell
}:
let
list = import ./data.nix;
list = lib.importJSON ./providers.json;
toDrv = data:
toDrv = name: data:
let
fallbackProviderSourceAddress = "nixpkgs/${data.owner}/${name}";
providerSourceAddress = data.provider-source-address or fallbackProviderSourceAddress;
in
buildGoPackage rec {
inherit (data) owner repo rev version sha256;
name = "${repo}-${version}";
@ -18,6 +23,9 @@ let
# Terraform allow checking the provider versions, but this breaks
# if the versions are not provided via file paths.
postBuild = "mv $NIX_BUILD_TOP/go/bin/${repo}{,_v${version}}";
passthru = {
inherit providerSourceAddress;
};
};
# Google is now using the vendored go modules, which works a bit differently
@ -48,7 +56,7 @@ let
});
# These providers are managed with the ./update-all script
automated-providers = lib.mapAttrs (_: toDrv) list;
automated-providers = lib.mapAttrs (toDrv) list;
# These are the providers that don't fall in line with the default model
special-providers = {
@ -57,6 +65,13 @@ let
google-beta = patchGoModVendor automated-providers.google-beta;
ibm = patchGoModVendor automated-providers.ibm;
acme = automated-providers.acme.overrideAttrs (attrs: {
prePatch = attrs.prePatch or "" + ''
substituteInPlace go.mod --replace terraform-providers/terraform-provider-acme getstackhead/terraform-provider-acme
substituteInPlace main.go --replace terraform-providers/terraform-provider-acme getstackhead/terraform-provider-acme
'';
});
# providers that were moved to the `hashicorp` organization,
# but haven't updated their references yet:

1037
pkgs/applications/networking/cluster/terraform-providers/providers.json Normal file
View File

File diff suppressed because it is too large Load Diff

177
pkgs/applications/networking/cluster/terraform-providers/update-all
View File

@ -1,177 +0,0 @@
#!/usr/bin/env nix-shell
#! nix-shell -i bash -p bash coreutils jq nix gitAndTools.hub
# vim: ft=sh sw=2 et
# shellcheck shell=bash
#
# This scripts scans the github terraform-providers repo for new releases,
# generates the corresponding nix code and finally generates an index of
# all the providers given in ./providers.txt.
set -euo pipefail
# the maximum number of attempts before giving up inside of GET and prefetch_github
readonly maxAttempts=30
get_tf_providers_org() {
# returns all terraform providers in a given organization, and their the
# latest tags, in the format
# $org/$repo $rev
local org=$1
hub api --paginate graphql -f query="
query(\$endCursor: String) {
repositoryOwner(login: \"${org}\") {
repositories(first: 100, after: \$endCursor) {
nodes {
nameWithOwner
name
refs(first: 1, refPrefix: \"refs/tags/\", orderBy: {field: TAG_COMMIT_DATE, direction: DESC}) {
nodes {
name
}
}
}
pageInfo {
hasNextPage
endCursor
}
}
}
}" | \
jq -r '.data.repositoryOwner.repositories.nodes[] | select(.name | startswith("terraform-provider-")) | select((.refs.nodes | length) > 0) | .nameWithOwner + " " + .refs.nodes[0].name'
# filter the result with jq:
# - repos need to start with `teraform-provider-`
# - they need to have at least one tag
# for each of the remaining repos, assemble a string $org/$repo $rev
}
get_latest_repo_tag() {
# of a given repo and owner, retrieve the latest tag
local owner=$1
local repo=$2
hub api --paginate "https://api.github.com/repos/$owner/$repo/git/refs/tags" | \
jq -r '.[].ref' | \
grep -v 'v\.' | \
cut -d '/' -f 3- | \
sort --version-sort | \
tail -1
}
prefetch_github() {
# of a given owner, repo and rev, fetch the tarball and return the output of
# `nix-prefetch-url`
local owner=$1
local repo=$2
local rev=$3
local retry=1
while ! nix-prefetch-url --unpack "https://github.com/$owner/$repo/archive/$rev.tar.gz"; do
echo "The nix-prefetch-url command has failed. Attempt $retry/${maxAttempts}" >&2
if [[ "${retry}" -eq "${maxAttempts}" ]]; then
exit 1
fi
retry=$(( retry + 1 ))
sleep 5
done
}
echo_entry() {
local owner=$1
local repo=$2
local rev=$3
local version=${rev#v}
local sha256=$4
cat <<EOF
{
owner = "$owner";
repo = "$repo";
rev = "$rev";
version = "$version";
sha256 = "$sha256";
};
EOF
}
indent() { sed 's/^/ /'; }
add_provider() {
org="${1}"
repo="${2}"
rev="${3}"
echo "*** $org/$repo $rev ***"
name=$(echo "$repo" | cut -d - -f 3-)
sha256=$(prefetch_github "$org" "$repo" "$rev")
{
echo " $name ="
echo_entry "$org" "$repo" "$rev" "$sha256" | indent
} >> data.nix
}
## Main ##
cd "$(dirname "$0")"
# individual repos to fetch
slugs=(
IBM-Cloud/terraform-provider-ibm
ajbosco/terraform-provider-segment
camptocamp/terraform-provider-pass
carlpett/terraform-provider-sops
poseidon/terraform-provider-matchbox
poseidon/terraform-provider-ct
tweag/terraform-provider-nixos
tweag/terraform-provider-secret
)
# a list of providers to ignore
blacklist=(
terraform-providers/terraform-provider-azure-classic
terraform-providers/terraform-provider-cidr
terraform-providers/terraform-provider-circonus
terraform-providers/terraform-provider-cloudinit
terraform-providers/terraform-provider-quorum
hashicorp/terraform-provider-time
terraform-providers/terraform-provider-vmc
)
cat <<HEADER > data.nix
# Generated with ./update-all
{
HEADER
# assemble list of terraform providers
providers=$(get_tf_providers_org "terraform-providers")
providers=$(echo "$providers";get_tf_providers_org "hashicorp")
# add terraform-providers from slugs
for slug in "${slugs[@]}"; do
# retrieve latest tag
org=${slug%/*}
repo=${slug#*/}
rev=$(get_latest_repo_tag "$org" "$repo")
# add to list
providers=$(echo "$providers";echo "$org/$repo $rev")
done
# filter out all providers on the blacklist
for repo in "${blacklist[@]}"; do
providers=$(echo "$providers" | grep -v "^${repo} ")
done
# sort results alphabetically by repo name
providers=$(echo "$providers" | sort -t "/" --key=2)
# render list
IFS=$'\n'
for provider in $providers; do
org=$(echo "$provider" | cut -d " " -f 1 | cut -d "/" -f1)
repo=$(echo "$provider" | cut -d " " -f 1 | cut -d "/" -f2)
rev=$(echo "$provider" | cut -d " " -f 2)
add_provider "${org}" "${repo}" "${rev}"
done
cat <<FOOTER >> data.nix
}
FOOTER
echo Done.

22
pkgs/applications/networking/cluster/terraform-providers/update-all-providers Executable file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env nix-shell
#! nix-shell -i bash -p jq
# shellcheck shell=bash
# Update all providers which have specified provider source address
set -euo pipefail
providers=$(
jq -r 'to_entries
| map_values(.value + { alias: .key })
| .[]
| select(."provider-source-address"?)
| .alias' providers.json
)
echo "Will update providers:"
echo "$providers"
for provider in $providers; do
echo "Updating $provider"
./update-provider "$provider"
done

77
pkgs/applications/networking/cluster/terraform-providers/update-provider Executable file
View File

@ -0,0 +1,77 @@
#!/usr/bin/env nix-shell
#! nix-shell -i bash -p coreutils curl jq
# shellcheck shell=bash
#
# Update a terraform provider to the latest version advertised at
# the provider source address.
set -euo pipefail
USAGE=$(cat<<DOC
Specify the terraform provider name to update.
Example:
To update nixpkgs.terraform-providers.aws run:
./update-provider aws
DOC
)
provider_name="${1:-}"
if [ -z "$provider_name" ]; then
echo "No providers specified!"
echo
echo "$USAGE"
exit 1
fi
provider_source_address="$(jq -r ".$provider_name.\"provider-source-address\"" providers.json)"
if [ "$provider_source_address" == "null" ]; then
echo "No provider source address specified with provider: $provider_name"
exit 1
fi
# The provider source address (used inside Terraform `required_providers` block) is
# used to compute the registry API endpoint
#
# registry.terraform.io/hashicorp/aws (provider source address)
# registry.terraform.io/providers/hashicorp/aws (provider URL for the website)
# registry.terraform.io/v1/providers/hashicorp/aws (provider URL for the JSON API)
registry_response=$(curl -s https://"${provider_source_address/\///v1/providers/}")
prefetch_github() {
# of a given owner, repo and rev, fetch the tarball and return the output of
# `nix-prefetch-url`
local owner=$1
local repo=$2
local rev=$3
nix-prefetch-url --unpack "https://github.com/$owner/$repo/archive/$rev.tar.gz"
}
provider_source_url="$(jq -r '.source' <<< "$registry_response")"
org="$(echo "$provider_source_url" | cut -d '/' -f 4)"
repo="$(echo "$provider_source_url" | cut -d '/' -f 5)"
rev="$(jq -r '.tag' <<< "$registry_response")"
sha256=$(prefetch_github "$org" "$repo" "$rev")
version="$(jq -r '.version' <<< "$registry_response")"
updated_provider="$(mktemp)"
cat <<EOF >> "$updated_provider"
{
"$provider_name": {
"owner": "$org",
"repo": "$repo",
"rev": "$rev",
"sha256": "$sha256",
"version": "$version",
"provider-source-address": "$provider_source_address"
}
}
EOF
original_provider_list="$(mktemp)"
cat providers.json > "$original_provider_list"
jq --sort-keys --slurp '.[0] * .[1]' "$original_provider_list" "$updated_provider" > providers.json

43
pkgs/applications/networking/cluster/terraform/default.nix
View File

@ -1,5 +1,5 @@
{ stdenv, lib, buildEnv, buildGoPackage, fetchFromGitHub, makeWrapper, coreutils
, runCommand, writeText, terraform-providers, fetchpatch }:
, runCommand, runtimeShell, writeText, terraform-providers, fetchpatch }:
let
goPackagePath = "github.com/hashicorp/terraform";
@ -43,12 +43,13 @@ let
homepage = "https://www.terraform.io/";
license = licenses.mpl20;
maintainers = with maintainers; [
zimbatm
peterhoeg
Chili-Man
babariviere
kalbasit
marsam
babariviere
Chili-Man
peterhoeg
timstott
zimbatm
];
};
} // attrs');
@ -59,6 +60,29 @@ let
let
actualPlugins = plugins terraform.plugins;
# Make providers available in Terraform 0.13 and 0.12 search paths.
pluginDir = lib.concatMapStrings (pl: let
inherit (pl) repo version GOOS GOARCH;
inherit (pl.passthru) providerSourceAddress;
shim = writeText "shim" ''
#!${runtimeShell}
exec ${pl}/bin/${repo}_v${version} \$@
'';
in ''
TF_0_13_PROVIDER_PATH=$out/plugins/${providerSourceAddress}/${version}/${GOOS}_${GOARCH}/${repo}_v${version}
mkdir -p "$(dirname $TF_0_13_PROVIDER_PATH)"
cp ${shim} "$TF_0_13_PROVIDER_PATH"
chmod +x "$TF_0_13_PROVIDER_PATH"
TF_0_12_PROVIDER_PATH=$out/plugins/${repo}_v${version}
cp ${shim} "$TF_0_12_PROVIDER_PATH"
chmod +x "$TF_0_12_PROVIDER_PATH"
''
) actualPlugins;
# Wrap PATH of plugins propagatedBuildInputs, plugins may have runtime dependencies on external binaries
wrapperInputs = lib.unique (lib.flatten
(lib.catAttrs "propagatedBuildInputs"
@ -87,15 +111,10 @@ let
inherit (terraform) name;
buildInputs = [ makeWrapper ];
buildCommand = ''
buildCommand = pluginDir + ''
mkdir -p $out/bin/
makeWrapper "${terraform}/bin/terraform" "$out/bin/terraform" \
--set NIX_TERRAFORM_PLUGIN_DIR "${
buildEnv {
name = "tf-plugin-env";
paths = actualPlugins;
}
}/bin" \
--set NIX_TERRAFORM_PLUGIN_DIR $out/plugins \
--prefix PATH : "${lib.makeBinPath wrapperInputs}"
'';