Merge pull request #40399 from rvl/datadog-agent

DataDog Agent v6

nixos/modules/module-list.nix

@@ -407,6 +407,7 @@
   ./services/monitoring/cadvisor.nix
   ./services/monitoring/collectd.nix
   ./services/monitoring/das_watchdog.nix
+  ./services/monitoring/datadog-agent.nix
   ./services/monitoring/dd-agent/dd-agent.nix
   ./services/monitoring/fusion-inventory.nix
   ./services/monitoring/grafana.nix

nixos/modules/services/monitoring/datadog-agent.nix

@@ -0,0 +1,236 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.datadog-agent;
+
+  ddConf = {
+    dd_url              = "https://app.datadoghq.com";
+    skip_ssl_validation = "no";
+    api_key             = "";
+    confd_path          = "/etc/datadog-agent/conf.d";
+    additional_checksd  = "/etc/datadog-agent/checks.d";
+    use_dogstatsd       = "yes";
+  }
+  // optionalAttrs (cfg.logLevel != null) { log_level = cfg.logLevel; }
+  // optionalAttrs (cfg.hostname != null) { inherit (cfg) hostname; }
+  // optionalAttrs (cfg.tags != null ) { tags = concatStringsSep ", " cfg.tags; }
+  // cfg.extraConfig;
+
+  # Generate Datadog configuration files for each configured checks.
+  # This works because check configurations have predictable paths,
+  # and because JSON is a valid subset of YAML.
+  makeCheckConfigs = entries: mapAttrsToList (name: conf: {
+    source = pkgs.writeText "${name}-check-conf.yaml" (builtins.toJSON conf);
+    target = "datadog-agent/conf.d/${name}.d/conf.yaml";
+  }) entries;
+
+  defaultChecks = {
+    disk = cfg.diskCheck;
+    network = cfg.networkCheck;
+  };
+
+  # Assemble all check configurations and the top-level agent
+  # configuration.
+  etcfiles = with pkgs; with builtins; [{
+    source = writeText "datadog.yaml" (toJSON ddConf);
+    target = "datadog-agent/datadog.yaml";
+  }] ++ makeCheckConfigs (cfg.checks // defaultChecks);
+
+  # Apply the configured extraIntegrations to the provided agent
+  # package. See the documentation of `dd-agent/integrations-core.nix`
+  # for detailed information on this.
+  datadogPkg = cfg.package.overrideAttrs(_: {
+    python = (pkgs.datadog-integrations-core cfg.extraIntegrations).python;
+  });
+in {
+  options.services.datadog-agent = {
+    enable = mkOption {
+      description = ''
+        Whether to enable the datadog-agent v6 monitoring service
+      '';
+      default = false;
+      type = types.bool;
+    };
+
+    package = mkOption {
+      default = pkgs.datadog-agent;
+      defaultText = "pkgs.datadog-agent";
+      description = ''
+        Which DataDog v6 agent package to use. Note that the provided
+        package is expected to have an overridable `python`-attribute
+        which configures the Python environment with the Datadog
+        checks.
+      '';
+      type = types.package;
+    };
+
+    apiKeyFile = mkOption {
+      description = ''
+        Path to a file containing the Datadog API key to associate the
+        agent with your account.
+      '';
+      example = "/run/keys/datadog_api_key";
+      type = types.path;
+    };
+
+    tags = mkOption {
+      description = "The tags to mark this Datadog agent";
+      example = [ "test" "service" ];
+      default = null;
+      type = types.nullOr (types.listOf types.str);
+    };
+
+    hostname = mkOption {
+      description = "The hostname to show in the Datadog dashboard (optional)";
+      default = null;
+      example = "mymachine.mydomain";
+      type = types.uniq (types.nullOr types.string);
+    };
+
+    logLevel = mkOption {
+      description = "Logging verbosity.";
+      default = null;
+      type = types.nullOr (types.enum ["DEBUG" "INFO" "WARN" "ERROR"]);
+    };
+
+    extraIntegrations = mkOption {
+      default = {};
+      type    = types.attrs;
+
+      description = ''
+        Extra integrations from the Datadog core-integrations
+        repository that should be built and included.
+
+        By default the included integrations are disk, mongo, network,
+        nginx and postgres.
+
+        To include additional integrations the name of the derivation
+        and a function to filter its dependencies from the Python
+        package set must be provided.
+      '';
+
+      example = {
+        ntp = (pythonPackages: [ pythonPackages.ntplib ]);
+      };
+    };
+
+    extraConfig = mkOption {
+      default = {};
+      type = types.attrs;
+      description = ''
+        Extra configuration options that will be merged into the
+        main config file <filename>datadog.yaml</filename>.
+      '';
+     };
+
+    checks = mkOption {
+      description = ''
+        Configuration for all Datadog checks. Keys of this attribute
+        set will be used as the name of the check to create the
+        appropriate configuration in `conf.d/$check.d/conf.yaml`.
+
+        The configuration is converted into JSON from the plain Nix
+        language configuration, meaning that you should write
+        configuration adhering to Datadog's documentation - but in Nix
+        language.
+
+        Refer to the implementation of this module (specifically the
+        definition of `defaultChecks`) for an example.
+
+        Note: The 'disk' and 'network' check are configured in
+        separate options because they exist by default. Attempting to
+        override their configuration here will have no effect.
+      '';
+
+      example = {
+        http_check = {
+          init_config = null; # sic!
+          instances = [
+            {
+              name = "some-service";
+              url = "http://localhost:1337/healthz";
+              tags = [ "some-service" ];
+            }
+          ];
+        };
+      };
+
+      default = {};
+
+      # sic! The structure of the values is up to the check, so we can
+      # not usefully constrain the type further.
+      type = with types; attrsOf attrs;
+    };
+
+    diskCheck = mkOption {
+      description = "Disk check config";
+      type = types.attrs;
+      default = {
+        init_config = {};
+        instances = [ { use-mount = "no"; } ];
+      };
+    };
+
+    networkCheck = mkOption {
+      description = "Network check config";
+      type = types.attrs;
+      default = {
+        init_config = {};
+        # Network check only supports one configured instance
+        instances = [ { collect_connection_state = false;
+          excluded_interfaces = [ "lo" "lo0" ]; } ];
+      };
+    };
+  };
+  config = mkIf cfg.enable {
+    environment.systemPackages = [ datadogPkg pkgs.sysstat pkgs.procps ];
+
+    users.extraUsers.datadog = {
+      description = "Datadog Agent User";
+      uid = config.ids.uids.datadog;
+      group = "datadog";
+      home = "/var/log/datadog/";
+      createHome = true;
+    };
+
+    users.extraGroups.datadog.gid = config.ids.gids.datadog;
+
+    systemd.services = let
+      makeService = attrs: recursiveUpdate {
+        path = [ datadogPkg pkgs.python pkgs.sysstat pkgs.procps ];
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig = {
+          User = "datadog";
+          Group = "datadog";
+          Restart = "always";
+          RestartSec = 2;
+          PrivateTmp = true;
+        };
+        restartTriggers = [ datadogPkg ] ++ map (etc: etc.source) etcfiles;
+      } attrs;
+    in {
+      datadog-agent = makeService {
+        description = "Datadog agent monitor";
+        preStart = ''
+          chown -R datadog: /etc/datadog-agent
+          rm -f /etc/datadog-agent/auth_token
+        '';
+        script = ''
+          export DD_API_KEY=$(head -n 1 ${cfg.apiKeyFile})
+          exec ${datadogPkg}/bin/agent start -c /etc/datadog-agent/datadog.yaml
+        '';
+        serviceConfig.PermissionsStartOnly = true;
+      };
+
+      dd-jmxfetch = lib.mkIf (lib.hasAttr "jmx" cfg.checks) (makeService {
+        description = "Datadog JMX Fetcher";
+        path = [ datadogPkg pkgs.python pkgs.sysstat pkgs.procps pkgs.jdk ];
+        serviceConfig.ExecStart = "${datadogPkg}/bin/dd-jmxfetch";
+      });
+    };
+
+    environment.etc = etcfiles;
+  };
+}

nixos/modules/services/monitoring/dd-agent/dd-agent.nix

@@ -114,13 +114,22 @@ let
 in {
   options.services.dd-agent = {
     enable = mkOption {
-      description = "Whether to enable the dd-agent montioring service";
+      description = ''
+        Whether to enable the dd-agent v5 monitoring service.
+        For datadog-agent v6, see <option>services.datadog-agent.enable</option>.
+      '';
       default = false;
       type = types.bool;
     };
 
     api_key = mkOption {
-      description = "The Datadog API key to associate the agent with your account";
+      description = ''
+        The Datadog API key to associate the agent with your account.
+
+        Warning: this key is stored in cleartext within the world-readable
+        Nix store! Consider using the new v6
+        <option>services.datadog-agent</option> module instead.
+      '';
       example = "ae0aa6a8f08efa988ba0a17578f009ab";
       type = types.str;
     };
@@ -188,48 +197,41 @@ in {
 
     users.groups.datadog.gid = config.ids.gids.datadog;
 
-    systemd.services.dd-agent = {
-      description = "Datadog agent monitor";
-      path = [ pkgs."dd-agent" pkgs.python pkgs.sysstat pkgs.procps pkgs.gohai ];
-      wantedBy = [ "multi-user.target" ];
-      serviceConfig = {
-        ExecStart = "${pkgs.dd-agent}/bin/dd-agent foreground";
-        User = "datadog";
-        Group = "datadog";
-        Restart = "always";
-        RestartSec = 2;
+    systemd.services = let
+      makeService = attrs: recursiveUpdate {
+        path = [ pkgs.dd-agent pkgs.python pkgs.sysstat pkgs.procps pkgs.gohai ];
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig = {
+          User = "datadog";
+          Group = "datadog";
+          Restart = "always";
+          RestartSec = 2;
+          PrivateTmp = true;
+        };
+        restartTriggers = [ pkgs.dd-agent ddConf diskConfig networkConfig postgresqlConfig nginxConfig mongoConfig jmxConfig processConfig ];
+      } attrs;
+    in {
+      dd-agent = makeService {
+        description = "Datadog agent monitor";
+        serviceConfig.ExecStart = "${pkgs.dd-agent}/bin/dd-agent foreground";
       };
-      restartTriggers = [ pkgs.dd-agent ddConf diskConfig networkConfig postgresqlConfig nginxConfig mongoConfig jmxConfig processConfig ];
-    };
 
-    systemd.services.dogstatsd = {
-      description = "Datadog statsd";
-      path = [ pkgs."dd-agent" pkgs.python pkgs.procps ];
-      wantedBy = [ "multi-user.target" ];
-      serviceConfig = {
-        ExecStart = "${pkgs.dd-agent}/bin/dogstatsd start";
-        User = "datadog";
-        Group = "datadog";
-        Type = "forking";
-        PIDFile = "/tmp/dogstatsd.pid";
-        Restart = "always";
-        RestartSec = 2;
+      dogstatsd = makeService {
+        description = "Datadog statsd";
+        environment.TMPDIR = "/run/dogstatsd";
+        serviceConfig = {
+          ExecStart = "${pkgs.dd-agent}/bin/dogstatsd start";
+          Type = "forking";
+          PIDFile = "/run/dogstatsd/dogstatsd.pid";
+          RuntimeDirectory = "dogstatsd";
+        };
       };
-      restartTriggers = [ pkgs.dd-agent ddConf diskConfig networkConfig postgresqlConfig nginxConfig mongoConfig jmxConfig processConfig ];
-    };
 
-    systemd.services.dd-jmxfetch = lib.mkIf (cfg.jmxConfig != null) {
-      description = "Datadog JMX Fetcher";
-      path = [ pkgs."dd-agent" pkgs.python pkgs.sysstat pkgs.procps pkgs.jdk ];
-      wantedBy = [ "multi-user.target" ];
-      serviceConfig = {
-        ExecStart = "${pkgs.dd-agent}/bin/dd-jmxfetch";
-        User = "datadog";
-        Group = "datadog";
-        Restart = "always";
-        RestartSec = 2;
+      dd-jmxfetch = lib.mkIf (cfg.jmxConfig != null) {
+        description = "Datadog JMX Fetcher";
+        path = [ pkgs.dd-agent pkgs.python pkgs.sysstat pkgs.procps pkgs.jdk ];
+        serviceConfig.ExecStart = "${pkgs.dd-agent}/bin/dd-jmxfetch";
       };
-      restartTriggers = [ pkgs.dd-agent ddConf diskConfig networkConfig postgresqlConfig nginxConfig mongoConfig jmxConfig ];
     };
 
     environment.etc = etcfiles;

pkgs/development/python-modules/uuid/default.nix

@@ -0,0 +1,16 @@
+{ lib, stdenv, buildPythonPackage, fetchPypi }:
+
+buildPythonPackage rec {
+  pname = "uuid";
+  version = "1.30";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "0gqrjsm85nnkxkmd1vk8350wqj2cigjflnvcydk084n5980cr1qz";
+  };
+
+  meta = with lib; {
+    description = "UUID object and generation functions (Python 2.3 or higher)";
+    homepage = http://zesty.ca/python/;
+  };
+}

pkgs/tools/networking/dd-agent/5.nix

@@ -89,7 +89,10 @@ in stdenv.mkDerivation rec {
   '';
 
   meta = {
-    description = "Event collector for the DataDog analysis service";
+    description = ''
+      Event collector for the DataDog analysis service
+      -- v5 Python implementation
+    '';
     homepage    = https://www.datadoghq.com;
     license     = stdenv.lib.licenses.bsd3;
     platforms   = stdenv.lib.platforms.all;

pkgs/tools/networking/dd-agent/6.nix

@@ -0,0 +1,77 @@
+{ stdenv, fetchFromGitHub, buildGoPackage, makeWrapper, pythonPackages, pkgconfig }:
+
+let
+  # keep this in sync with github.com/DataDog/agent-payload dependency
+  payloadVersion = "4.7";
+
+in buildGoPackage rec {
+  name = "datadog-agent-${version}";
+  version = "6.1.4";
+  owner   = "DataDog";
+  repo    = "datadog-agent";
+
+  src = fetchFromGitHub {
+    inherit owner repo;
+    rev    = "a8ee76deb11fa334470d9b8f2356214999980894";
+    sha256 = "06grcwwbfvcw1k1d4nqrasrf76qkpik1gsw60zwafllfd9ffhl1v";
+  };
+
+  subPackages = [
+    "cmd/agent"
+    "cmd/dogstatsd"
+    "cmd/py-launcher"
+    "cmd/cluster-agent"
+  ];
+  goDeps = ./deps.nix;
+  goPackagePath = "github.com/${owner}/${repo}";
+
+  # Explicitly set this here to allow it to be overridden.
+  python = pythonPackages.python;
+
+  nativeBuildInputs = [ pkgconfig makeWrapper ];
+  PKG_CONFIG_PATH = "${python}/lib/pkgconfig";
+
+  buildFlagsArray = let
+    ldFlags = stdenv.lib.concatStringsSep " " [
+      "-X ${goPackagePath}/pkg/version.Commit=${src.rev}"
+      "-X ${goPackagePath}/pkg/version.AgentVersion=${version}"
+      "-X ${goPackagePath}/pkg/serializer.AgentPayloadVersion=${payloadVersion}"
+      "-X ${goPackagePath}/pkg/collector/py.pythonHome=${python}"
+      "-r ${python}/lib"
+    ];
+  in [
+    "-ldflags=${ldFlags}"
+  ];
+  buildFlags = "-tags cpython";
+
+  # DataDog use paths relative to the agent binary, so fix these.
+  postPatch = ''
+    sed -e "s|PyChecksPath =.*|PyChecksPath = \"$bin/${python.sitePackages}\"|" \
+        -e "s|distPath =.*|distPath = \"$bin/share/datadog-agent\"|" \
+        -i cmd/agent/common/common_nix.go
+  '';
+
+  # Install the config files and python modules from the "dist" dir
+  # into standard paths.
+  postInstall = ''
+    mkdir -p $bin/${python.sitePackages} $bin/share/datadog-agent
+    cp -R $src/cmd/agent/dist/{conf.d,trace-agent.conf} $bin/share/datadog-agent
+    cp -R $src/cmd/agent/dist/{checks,utils,config.py} $bin/${python.sitePackages}
+
+    cp -R $src/pkg/status/dist/templates $bin/share/datadog-agent
+
+    wrapProgram "$bin/bin/agent" \
+        --set PYTHONPATH "$bin/${python.sitePackages}"
+  '';
+
+  meta = with stdenv.lib; {
+    description = ''
+      Event collector for the DataDog analysis service
+      -- v6 new golang implementation.
+    '';
+    homepage    = https://www.datadoghq.com;
+    license     = licenses.bsd3;
+    platforms   = platforms.all;
+    maintainers = with maintainers; [ thoughtpolice domenkozar rvl ];
+  };
+}

pkgs/tools/networking/dd-agent/deps.nix

@@ -0,0 +1,353 @@
+[
+  {
+    goPackagePath = "github.com/DataDog/agent-payload";
+    fetch = {
+      type = "git";
+      url = "https://github.com/DataDog/agent-payload";
+      rev = "3b793015ecfa5b829e8a466bd7cce836891502cc";
+      sha256 = "0lg7c1whmvk4a13mrivdjfzfxqan07kvs2calgylncy7yf4szdp6";
+    };
+  }
+  {
+    goPackagePath = "github.com/DataDog/gohai";
+    fetch = {
+      type = "git";
+      url = "https://github.com/DataDog/gohai";
+      rev = "d80d0f562a71fa2380fbeccc93ba5a2e325606e4";
+      sha256 = "1frslms7f5i8dc8n0v9pb64mf4zdj3q2c005qxajl8j8i9nhj7yb";
+    };
+  }
+  {
+    goPackagePath = "github.com/DataDog/mmh3";
+    fetch = {
+      type = "git";
+      url = "https://github.com/DataDog/mmh3";
+      rev = "2cfb68475274527a10701355c739f31dd404718c";
+      sha256 = "09jgzxi08pkxllxk3f5qwipz96jxrw5v035fj2bkid1d4akn8y0b";
+    };
+  }
+  {
+    goPackagePath = "github.com/beevik/ntp";
+    fetch = {
+      type = "git";
+      url = "https://github.com/beevik/ntp";
+      rev = "cb3dae3a7588ae35829eb5724df611cd75152fba";
+      sha256 = "0nc6f7d0xw23y18z9qxrmm8kvnywihassyk706mn9v4makmhalnz";
+    };
+  }
+  {
+    goPackagePath = "github.com/cihub/seelog";
+    fetch = {
+      type = "git";
+      url = "https://github.com/cihub/seelog";
+      rev = "f561c5e57575bb1e0a2167028b7339b3a8d16fb4";
+      sha256 = "0r3228hvgljgpaggj6b9mvxfsizfw25q2c1761wsvcif8gz49cvl";
+    };
+  }
+  {
+    goPackagePath = "github.com/docker/docker";
+    fetch = {
+      type = "git";
+      url = "https://github.com/docker/docker";
+      rev = "092cba3727bb9b4a2f0e922cd6c0f93ea270e363";
+      sha256 = "0l9kjibnpwcgk844sibxk9ppyqniw9r0np1mzp95f8f461jb0iar";
+    };
+  }
+  {
+    goPackagePath = "github.com/dsnet/compress";
+    fetch = {
+      type = "git";
+      url = "https://github.com/dsnet/compress";
+      rev = "cc9eb1d7ad760af14e8f918698f745e80377af4f";
+      sha256 = "159liclywmyb6zx88ga5gn42hfl4cpk1660zss87fkx31hdq9fgx";
+    };
+  }
+  {
+    goPackagePath = "github.com/fatih/color";
+    fetch = {
+      type = "git";
+      url = "https://github.com/fatih/color";
+      rev = "507f6050b8568533fb3f5504de8e5205fa62a114";
+      sha256 = "0k1v9dkhrxiqhg48yqkwzpd7x40xx38gv2pgknswbsy4r8w644i7";
+    };
+  }
+  {
+    goPackagePath = "github.com/fsnotify/fsnotify";
+    fetch = {
+      type = "git";
+      url = "https://github.com/fsnotify/fsnotify";
+      rev = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9";
+      sha256 = "07va9crci0ijlivbb7q57d2rz9h27zgn2fsm60spjsqpdbvyrx4g";
+    };
+  }
+  {
+    goPackagePath = "github.com/go-ini/ini";
+    fetch = {
+      type = "git";
+      url = "https://github.com/go-ini/ini";
+      rev = "bda519ae5f4cbc60d391ff8610711627a08b86ae";
+      sha256 = "05vcc3ssxyrk8g3sr4gs888vllgjqfq11na63qz2pvaiy7m0rqrs";
+    };
+  }
+  {
+    goPackagePath = "github.com/gogo/protobuf";
+    fetch = {
+      type = "git";
+      url = "https://github.com/gogo/protobuf";
+      rev = "1ef32a8b9fc3f8ec940126907cedb5998f6318e4";
+      sha256 = "0zk2n0n35ksskr5cd83k5k8wg21ckrcggjy88bym2s21ngj5w4fh";
+    };
+  }
+  {
+    goPackagePath = "github.com/golang/snappy";
+    fetch = {
+      type = "git";
+      url = "https://github.com/golang/snappy";
+      rev = "553a641470496b2327abcac10b36396bd98e45c9";
+      sha256 = "0kssxnih1l722hx9219c7javganjqkqhvl3i0hp0hif6xm6chvqk";
+    };
+  }
+  {
+    goPackagePath = "github.com/gorilla/mux";
+    fetch = {
+      type = "git";
+      url = "https://github.com/gorilla/mux";
+      rev = "ded0c29b24f96f46cf349e6701b099db601cf8ec";
+      sha256 = "125dxfxs7his95fd2r28bn1rpb78pldfgm3lmw84ha1c0v5gfh33";
+    };
+  }
+  {
+    goPackagePath = "github.com/hashicorp/hcl";
+    fetch = {
+      type = "git";
+      url = "https://github.com/hashicorp/hcl";
+      rev = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168";
+      sha256 = "1qalfsc31fra7hcw2lc3s20aj7al62fq3j5fn5kga3mg99b82nyr";
+    };
+  }
+  {
+    goPackagePath = "github.com/kardianos/osext";
+    fetch = {
+      type = "git";
+      url = "https://github.com/kardianos/osext";
+      rev = "ae77be60afb1dcacde03767a8c37337fad28ac14";
+      sha256 = "056dkgxrqjj5r18bnc3knlpgdz5p3yvp12y4y978hnsfhwaqvbjz";
+    };
+  }
+  {
+    goPackagePath = "github.com/magiconair/properties";
+    fetch = {
+      type = "git";
+      url = "https://github.com/magiconair/properties";
+      rev = "2c9e9502788518c97fe44e8955cd069417ee89df";
+      sha256 = "1w0rl9rla61m0qbha75jg48yiq1vs91sfy96rgqa6nags9v9b1rl";
+    };
+  }
+  {
+    goPackagePath = "github.com/mholt/archiver";
+    fetch = {
+      type = "git";
+      url = "https://github.com/mholt/archiver";
+      rev = "e4ef56d48eb029648b0e895bb0b6a393ef0829c3";
+      sha256 = "1krxyh6iq0s0rwhz7gg6dn795j9qq64rsgq9nivav7fhrqpgr6hb";
+    };
+  }
+  {
+    goPackagePath = "github.com/mitchellh/mapstructure";
+    fetch = {
+      type = "git";
+      url = "https://github.com/mitchellh/mapstructure";
+      rev = "00c29f56e2386353d58c599509e8dc3801b0d716";
+      sha256 = "1vw8fvhax0d567amgvxr7glcl12lvzg2sbzs007q5k5bbwn1szyb";
+    };
+  }
+  {
+    goPackagePath = "github.com/nwaples/rardecode";
+    fetch = {
+      type = "git";
+      url = "https://github.com/nwaples/rardecode";
+      rev = "e06696f847aeda6f39a8f0b7cdff193b7690aef6";
+      sha256 = "1aj7l8ii7hxnn3q4wzxlx3f92b1aspck6ncyqgb4h2g228phcibw";
+    };
+  }
+  {
+    goPackagePath = "github.com/patrickmn/go-cache";
+    fetch = {
+      type = "git";
+      url = "https://github.com/patrickmn/go-cache";
+      rev = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0";
+      sha256 = "10020inkzrm931r4bixf8wqr9n39wcrb78vfyxmbvjavvw4zybgs";
+    };
+  }
+  {
+    goPackagePath = "github.com/pelletier/go-toml";
+    fetch = {
+      type = "git";
+      url = "https://github.com/pelletier/go-toml";
+      rev = "66540cf1fcd2c3aee6f6787dfa32a6ae9a870f12";
+      sha256 = "1n8na0yg90gm0rpifmzrby5r385vvd62cdam3ls7ssy02bjvfw15";
+    };
+  }
+  {
+    goPackagePath = "github.com/pierrec/lz4";
+    fetch = {
+      type = "git";
+      url = "https://github.com/pierrec/lz4";
+      rev = "ed8d4cc3b461464e69798080a0092bd028910298";
+      sha256 = "0flsn2ka108lb63gkxfzl90bkhndh1znnscv4v1k6j5h2s3zksls";
+    };
+  }
+  {
+    goPackagePath = "github.com/pierrec/xxHash";
+    fetch = {
+      type = "git";
+      url = "https://github.com/pierrec/xxHash";
+      rev = "a0006b13c722f7f12368c00a3d3c2ae8a999a0c6";
+      sha256 = "1hf7hqrqx0cbgx0alfwnqs0mrxg9rnwsijn5d0lv06w6vzqbvnzj";
+    };
+  }
+  {
+    goPackagePath = "github.com/shirou/gopsutil";
+    fetch = {
+      type = "git";
+      url = "https://github.com/shirou/gopsutil";
+      rev = "57f370e13068146efe1cb7129f79e5d51da8a242";
+      sha256 = "1ij0bbnfjj65afin8vhccr3cxvg6r0awmvcvb2ilza5wbbsslggb";
+    };
+  }
+  {
+    goPackagePath = "github.com/spf13/afero";
+    fetch = {
+      type = "git";
+      url = "https://github.com/spf13/afero";
+      rev = "63644898a8da0bc22138abf860edaf5277b6102e";
+      sha256 = "13piahaq4vw1y1sklq5scrsflqx0a8hzmdqfz1fy4871kf2gl8qw";
+    };
+  }
+  {
+    goPackagePath = "github.com/spf13/cast";
+    fetch = {
+      type = "git";
+      url = "https://github.com/spf13/cast";
+      rev = "8965335b8c7107321228e3e3702cab9832751bac";
+      sha256 = "177bk7lq40jbgv9p9r80aydpaccfk8ja3a7jjhfwiwk9r1pa4rr2";
+    };
+  }
+  {
+    goPackagePath = "github.com/spf13/cobra";
+    fetch = {
+      type = "git";
+      url = "https://github.com/spf13/cobra";
+      rev = "ef82de70bb3f60c65fb8eebacbb2d122ef517385";
+      sha256 = "1q1nsx05svyv9fv3fy6xv6gs9ffimkyzsfm49flvl3wnvf1ncrkd";
+    };
+  }
+  {
+    goPackagePath = "github.com/spf13/jwalterweatherman";
+    fetch = {
+      type = "git";
+      url = "https://github.com/spf13/jwalterweatherman";
+      rev = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394";
+      sha256 = "132p84i20b9s5r6fs597lsa6648vd415ch7c0d018vm8smzqpd0h";
+    };
+  }
+  {
+    goPackagePath = "github.com/spf13/pflag";
+    fetch = {
+      type = "git";
+      url = "https://github.com/spf13/pflag";
+      rev = "583c0c0531f06d5278b7d917446061adc344b5cd";
+      sha256 = "0nr4mdpfhhk94hq4ymn5b2sxc47b29p1akxd8b0hx4dvdybmipb5";
+    };
+  }
+  {
+    goPackagePath = "github.com/spf13/viper";
+    fetch = {
+      type = "git";
+      url = "https://github.com/spf13/viper";
+      rev = "8dc2790b029dc41e2b8ff772c63c26adbb1db70d";
+      sha256 = "147zq6v34pgb79r4m0m2mwm8739jxwawxs8mpqvvhq7gxwvhng40";
+    };
+  }
+  {
+    goPackagePath = "github.com/stretchr/testify";
+    fetch = {
+      type = "git";
+      url = "https://github.com/stretchr/testify";
+      rev = "c679ae2cc0cb27ec3293fea7e254e47386f05d69";
+      sha256 = "1rrdn7k83j492rzhqwkh6956sj8m2nbk44d7r1xa9nsn3hfwj691";
+    };
+  }
+  {
+    goPackagePath = "github.com/ulikunitz/xz";
+    fetch = {
+      type = "git";
+      url = "https://github.com/ulikunitz/xz";
+      rev = "0c6b41e72360850ca4f98dc341fd999726ea007f";
+      sha256 = "0a6l7sp67ipxim093qh6fvw8knbxj24l7bj5lykcddi5gwfi78n3";
+    };
+  }
+  {
+    goPackagePath = "github.com/urfave/negroni";
+    fetch = {
+      type = "git";
+      url = "https://github.com/urfave/negroni";
+      rev = "22c5532ea862c34fdad414e90f8cc00b4f6f4cab";
+      sha256 = "0jxd10cr3imm96xa01mdgyad4nq6mc7yr49z830fv3vywfr7bac8";
+    };
+  }
+  {
+    goPackagePath = "golang.org/x/net";
+    fetch = {
+      type = "git";
+      url = "https://go.googlesource.com/net";
+      rev = "640f4622ab692b87c2f3a94265e6f579fe38263d";
+      sha256 = "097m4qhcljhp180171j5fjhq4740iirfkkajfd7yrxqhp4s9hljx";
+    };
+  }
+  {
+    goPackagePath = "golang.org/x/sys";
+    fetch = {
+      type = "git";
+      url = "https://go.googlesource.com/sys";
+      rev = "6f686a352de66814cdd080d970febae7767857a3";
+      sha256 = "1z3pwvxlzq8kghjdsd9xmf184iiz13h8h66ipp626k4rc7kydr03";
+    };
+  }
+  {
+    goPackagePath = "golang.org/x/text";
+    fetch = {
+      type = "git";
+      url = "https://go.googlesource.com/text";
+      rev = "7922cc490dd5a7dbaa7fd5d6196b49db59ac042f";
+      sha256 = "06sicjc24hv7v9p1l6psaq87w4lycx3mjixd6gsd1wnd4jhqvlnr";
+    };
+  }
+  {
+    goPackagePath = "gopkg.in/yaml.v2";
+    fetch = {
+      type = "git";
+      url = "https://gopkg.in/yaml.v2";
+      rev = "5420a8b6744d3b0345ab293f6fcba19c978f1183";
+      sha256 = "0dwjrs2lp2gdlscs7bsrmyc5yf6mm4fvgw71bzr9mv2qrd2q73s1";
+    };
+  }
+  {
+    goPackagePath = "github.com/sbinet/go-python";
+    fetch = {
+      type = "git";
+      url = "https://github.com/sbinet/go-python";
+      rev = "6d13f941744b9332d6ed00dc2cd2722acd79a47e";
+      sha256 = "0x5q4nyv6gck9q33g54gy2ajmyjksxjmzh0jfqqn97jpgf4qfaym";
+    };
+  }
+  {
+    goPackagePath = "github.com/mitchellh/reflectwalk";
+    fetch = {
+      type = "git";
+      url = "https://github.com/mitchellh/reflectwalk";
+      rev = "63d60e9d0dbc60cf9164e6510889b0db6683d98c";
+      sha256 = "1hpq6sjr6l1h25x68mz13q7sd52dv1mjfxbl5p7m3j7cv85khnvc";
+    };
+  }
+]

pkgs/tools/networking/dd-agent/integrations-core.nix

@@ -0,0 +1,91 @@
+# The declarations in this file build the Datadog agent's core
+# integrations. These integrations are tracked in a separate
+# repository[1] outside of the agent's primary repository and provide
+# checks for various kinds of services.
+#
+# Not all services are relevant for all users, however. As some of
+# them depend on various tools and Python packages it is nonsensical
+# to build *all* integrations by default.
+#
+# A set of default integrations is defined and built either way.
+# Additional integrations can be specified by overriding
+# `extraIntegrations` in datadog-integrations-core.
+#
+# In practice the syntax for using this with additional integrations
+# is not the most beautiful, but it works. For example to use
+# datadog-agent from the top-level with the `ntp`-integration
+# included, one could say:
+#
+# let
+#   integrationsWithNtp = datadog-integrations-core {
+#     # Extra integrations map from the integration name (as in the
+#     # integrations-core repository) to a function that receives the
+#     # Python package set and returns the required dependencies.g
+#     ntp = (ps: [ ps.ntplib ]);
+#   };
+#
+# in ddAgentWithNtp = datadog-agent.overrideAttrs(_ : {
+#   python = integrationsWithNtp.python;
+# });
+#
+# The NixOS module 'datadog-agent' provides a simplified interface to
+# this. Please see the module itself for more information.
+#
+# [1]: https://github.com/DataDog/integrations-core
+
+{ pkgs, python, extraIntegrations ? {} }:
+
+with pkgs.lib;
+
+let
+  src = pkgs.fetchFromGitHub {
+    owner = "DataDog";
+    repo = "integrations-core";
+    rev = "7be76e73969a8b9c993903681b300e1dd32f4b4d";
+    sha256 = "1qsqzm5iswgv9jrflh5mvbz9a7js7jf42cb28lzdzsp45iwfs2aa";
+  };
+  version = "git-2018-05-27";
+
+  # Build helper to build a single datadog integration package.
+  buildIntegration = { pname, ... }@args: python.pkgs.buildPythonPackage (args // {
+    inherit src version;
+    name = "datadog-integration-${pname}-${version}";
+
+    postPatch = ''
+      # jailbreak install_requires
+      sed -i 's/==.*//' requirements.in
+      cp requirements.in requirements.txt
+    '';
+    sourceRoot = "source/${args.sourceRoot or pname}";
+    doCheck = false;
+  });
+
+  # Base package depended on by all other integrations.
+  datadog_checks_base = buildIntegration {
+    pname = "checks-base";
+    sourceRoot = "datadog_checks_base";
+    propagatedBuildInputs = with python.pkgs; [
+      requests protobuf prometheus_client uuid simplejson uptime
+    ];
+  };
+
+  # Default integrations that should be built:
+  defaultIntegrations = {
+    disk     = (ps: [ ps.psutil ]);
+    mongo    = (ps: [ ps.pymongo ]);
+    network  = (ps: [ ps.psutil ]);
+    nginx    = (ps: []);
+    postgres = (ps: with ps; [ pg8000 psycopg2 ]);
+  };
+
+  # All integrations (default + extra):
+  integrations = defaultIntegrations // extraIntegrations;
+  builtIntegrations = mapAttrs (pname: fdeps: buildIntegration {
+    inherit pname;
+    propagatedBuildInputs = (fdeps python.pkgs) ++ [ datadog_checks_base ];
+  }) integrations;
+
+in builtIntegrations // {
+  inherit datadog_checks_base;
+  python = python.withPackages (_: (attrValues builtIntegrations));
+}

pkgs/top-level/all-packages.nix

@@ -15652,7 +15652,14 @@ with pkgs;
 
   dbvisualizer = callPackage ../applications/misc/dbvisualizer {};
 
-  dd-agent = callPackage ../tools/networking/dd-agent { };
+  dd-agent = callPackage ../tools/networking/dd-agent/5.nix { };
+  datadog-agent = callPackage ../tools/networking/dd-agent/6.nix {
+    pythonPackages = datadog-integrations-core {};
+  };
+  datadog-integrations-core = extras: callPackage ../tools/networking/dd-agent/integrations-core.nix {
+    python = python27;
+    extraIntegrations = extras;
+  };
 
   ddgr = callPackage ../applications/misc/ddgr { };
 

pkgs/top-level/python-packages.nix

@@ -17282,6 +17282,8 @@ EOF
 
   uranium = callPackage ../development/python-modules/uranium { };
 
+  uuid = callPackage ../development/python-modules/uuid { };
+
   versioneer = callPackage ../development/python-modules/versioneer { };
 
   vine = callPackage ../development/python-modules/vine { };