beegfs: init at 6.17

package, kernel module, nixos module, and nixos test
This commit is contained in:
Markus Kowalewski 2017-12-28 17:51:42 -08:00 committed by Orivej Desh
parent e0a4d1b158
commit b7fdefc8a4
7 changed files with 629 additions and 0 deletions

View File

@ -404,6 +404,7 @@
./services/monitoring/vnstat.nix
./services/monitoring/zabbix-agent.nix
./services/monitoring/zabbix-server.nix
./services/network-filesystems/beegfs.nix
./services/network-filesystems/cachefilesd.nix
./services/network-filesystems/davfs2.nix
./services/network-filesystems/drbd.nix

View File

@ -0,0 +1,343 @@
{ config, lib, pkgs, ...} :
with lib;
let
cfg = config.services.beegfs;
# functions for the generations of config files
configMgmtd = name: cfg: pkgs.writeText "mgmt-${name}.conf" ''
storeMgmtdDirectory = ${cfg.mgmtd.storeDir}
storeAllowFirstRunInit = false
connAuthFile = ${cfg.connAuthFile}
connPortShift = ${toString cfg.connPortShift}
${cfg.mgmtd.extraConfig}
'';
configAdmon = name: cfg: pkgs.writeText "admon-${name}.conf" ''
sysMgmtdHost = ${cfg.mgmtdHost}
connAuthFile = ${cfg.connAuthFile}
connPortShift = ${toString cfg.connPortShift}
${cfg.admon.extraConfig}
'';
configMeta = name: cfg: pkgs.writeText "meta-${name}.conf" ''
storeMetaDirectory = ${cfg.meta.storeDir}
sysMgmtdHost = ${cfg.mgmtdHost}
connAuthFile = ${cfg.connAuthFile}
connPortShift = ${toString cfg.connPortShift}
storeAllowFirstRunInit = false
${cfg.mgmtd.extraConfig}
'';
configStorage = name: cfg: pkgs.writeText "storage-${name}.conf" ''
storeStorageDirectory = ${cfg.storage.storeDir}
sysMgmtdHost = ${cfg.mgmtdHost}
connAuthFile = ${cfg.connAuthFile}
connPortShift = ${toString cfg.connPortShift}
storeAllowFirstRunInit = false
${cfg.storage.extraConfig}
'';
configHelperd = name: cfg: pkgs.writeText "helperd-${name}.conf" ''
connAuthFile = ${cfg.connAuthFile}
${cfg.helperd.extraConfig}
'';
configClientFilename = name : "/etc/beegfs/client-${name}.conf";
configClient = name: cfg: ''
sysMgmtdHost = ${cfg.mgmtdHost}
connAuthFile = ${cfg.connAuthFile}
connPortShift = ${toString cfg.connPortShift}
${cfg.client.extraConfig}
'';
serviceList = [
{ service = "admon"; cfgFile = configAdmon; }
{ service = "meta"; cfgFile = configMeta; }
{ service = "mgmtd"; cfgFile = configMgmtd; }
{ service = "storage"; cfgFile = configStorage; }
];
# functions to generate systemd.service entries
systemdEntry = service: cfgFile: (mapAttrs' ( name: cfg:
(nameValuePair "beegfs-${service}-${name}" (mkIf cfg."${service}".enable {
wantedBy = [ "multi-user.target" ];
requires = [ "network-online.target" ];
after = [ "network-online.target" ];
serviceConfig = rec {
ExecStart = ''
${pkgs.beegfs}/bin/beegfs-${service} \
cfgFile=${cfgFile name cfg} \
pidFile=${PIDFile}
'';
PIDFile = "/run/beegfs-${service}-${name}.pid";
TimeoutStopSec = "300";
};
}))) cfg);
systemdHelperd = mapAttrs' ( name: cfg:
(nameValuePair "beegfs-helperd-${name}" (mkIf cfg.client.enable {
wantedBy = [ "multi-user.target" ];
requires = [ "network-online.target" ];
after = [ "network-online.target" ];
serviceConfig = rec {
ExecStart = ''
${pkgs.beegfs}/bin/beegfs-helperd \
cfgFile=${configHelperd name cfg} \
pidFile=${PIDFile}
'';
PIDFile = "/run/beegfs-helperd-${name}.pid";
TimeoutStopSec = "300";
};
}))) cfg;
# wrappers to beegfs tools. Avoid typing path of config files
utilWrappers = mapAttrsToList ( name: cfg:
( pkgs.runCommand "beegfs-utils-${name}" { nativeBuildInputs = [ pkgs.makeWrapper ]; } ''
mkdir -p $out/bin
makeWrapper ${pkgs.beegfs}/bin/beegfs-check-servers \
$out/bin/beegfs-check-servers-${name} \
--add-flags "-c ${configClientFilename name}" \
--prefix PATH : ${lib.makeBinPath [ pkgs.beegfs ]}
makeWrapper ${pkgs.beegfs}/bin/beegfs-ctl \
$out/bin/beegfs-ctl-${name} \
--add-flags "--cfgFile=${configClientFilename name}"
makeWrapper ${pkgs.beegfs}/bin/beegfs-ctl \
$out/bin/beegfs-df-${name} \
--add-flags "--cfgFile=${configClientFilename name}" \
--add-flags --listtargets \
--add-flags --hidenodeid \
--add-flags --pools \
--add-flags --spaceinfo
makeWrapper ${pkgs.beegfs}/bin/beegfs-fsck \
$out/bin/beegfs-fsck-${name} \
--add-flags "--cfgFile=${configClientFilename name}"
''
)) cfg;
in
{
###### interface
options = {
services.beegfsEnable = mkEnableOption "BeeGFS";
services.beegfs = mkOption {
default = {};
description = ''
BeeGFS configurations. Every mount point requires a separate configuration.
'';
type = with types; attrsOf (submodule ({ config, ... } : {
options = {
mgmtdHost = mkOption {
type = types.str;
default = null;
example = "master";
description = ''Hostname of managament host.'';
};
connAuthFile = mkOption {
type = types.str;
default = "";
example = "/etc/my.key";
description = "File containing shared secret authentication.";
};
connPortShift = mkOption {
type = types.int;
default = 0;
example = 5;
description = ''
For each additional beegfs configuration shift all
service TCP/UDP ports by at least 5.
'';
};
client = {
enable = mkEnableOption "BeeGFS client";
mount = mkOption {
type = types.bool;
default = true;
description = "Create fstab entry automatically";
};
mountPoint = mkOption {
type = types.str;
default = "/run/beegfs";
description = ''
Mount point under which the beegfs filesytem should be mounted.
If mounted manually the mount option specifing the config file is needed:
cfgFile=/etc/beegfs/beegfs-client-<name>.conf
'';
};
extraConfig = mkOption {
type = types.lines;
default = "";
description = ''
Additional lines for beegfs-client.conf.
See documentation for further details.
'';
};
};
helperd = {
extraConfig = mkOption {
type = types.lines;
default = "";
description = ''
Additional lines for beegfs-helperd.conf. See documentation
for further details.
'';
};
};
mgmtd = {
enable = mkEnableOption "BeeGFS mgmtd daemon";
storeDir = mkOption {
type = types.path;
default = null;
example = "/data/beegfs-mgmtd";
description = ''
Data directory for mgmtd.
Must not be shared with other beegfs daemons.
This directory must exist and it must be initialized
with beegfs-setup-mgmtd, e.g. "beegfs-setup-mgmtd -C -p <storeDir>"
'';
};
extraConfig = mkOption {
type = types.lines;
default = "";
description = ''
Additional lines for beegfs-mgmtd.conf. See documentation
for further details.
'';
};
};
admon = {
enable = mkEnableOption "BeeGFS admon daemon";
extraConfig = mkOption {
type = types.lines;
default = "";
description = ''
Additional lines for beegfs-admon.conf. See documentation
for further details.
'';
};
};
meta = {
enable = mkEnableOption "BeeGFS meta data daemon";
storeDir = mkOption {
type = types.path;
default = null;
example = "/data/beegfs-meta";
description = ''
Data directory for meta data service.
Must not be shared with other beegfs daemons.
The underlying filesystem must be mounted with xattr turned on.
This directory must exist and it must be initialized
with beegfs-setup-meta, e.g.
"beegfs-setup-meta -C -s <serviceID> -p <storeDir>"
'';
};
extraConfig = mkOption {
type = types.str;
default = "";
description = ''
Additional lines for beegfs-meta.conf. See documentation
for further details.
'';
};
};
storage = {
enable = mkEnableOption "BeeGFS storage daemon";
storeDir = mkOption {
type = types.path;
default = null;
example = "/data/beegfs-storage";
description = ''
Data directories for storage service.
Must not be shared with other beegfs daemons.
The underlying filesystem must be mounted with xattr turned on.
This directory must exist and it must be initialized
with beegfs-setup-storage, e.g.
"beegfs-setup-storage -C -s <serviceID> -i <storageTargetID> -p <storeDir>"
'';
};
extraConfig = mkOption {
type = types.str;
default = "";
description = ''
Addional lines for beegfs-storage.conf. See documentation
for further details.
'';
};
};
};
}));
};
};
###### implementation
config =
mkIf config.services.beegfsEnable {
environment.systemPackages = utilWrappers;
# Put the client.conf files in /etc since they are needed
# by the commandline tools
environment.etc = mapAttrs' ( name: cfg:
(nameValuePair "beegfs/client-${name}.conf" (mkIf (cfg.client.enable)
{
enable = true;
text = configClient name cfg;
}))) cfg;
# Kernel module, we need it only once per host.
boot = mkIf (
foldr (a: b: a || b) false
(map (x: x.client.enable) (collect (x: x ? client) cfg)))
{
kernelModules = [ "beegfs" ];
extraModulePackages = [ pkgs.linuxPackages.beegfs-module ];
};
# generate fstab entries
fileSystems = mapAttrs' (name: cfg:
(nameValuePair cfg.client.mountPoint (optionalAttrs cfg.client.mount (mkIf cfg.client.enable {
device = "beegfs_nodev";
fsType = "beegfs";
mountPoint = cfg.client.mountPoint;
options = [ "cfgFile=${configClientFilename name}" "_netdev" ];
})))) cfg;
# generate systemd services
systemd.services = systemdHelperd //
foldr (a: b: a // b) {}
(map (x: systemdEntry x.service x.cfgFile) serviceList);
};
}

View File

@ -232,6 +232,7 @@ in rec {
tests.atd = callTest tests/atd.nix {};
tests.acme = callTest tests/acme.nix {};
tests.avahi = callTest tests/avahi.nix {};
tests.beegfs = callTest tests/beegfs.nix {};
tests.bittorrent = callTest tests/bittorrent.nix {};
tests.blivet = callTest tests/blivet.nix {};
tests.boot = callSubTests tests/boot.nix {};

115
nixos/tests/beegfs.nix Normal file
View File

@ -0,0 +1,115 @@
import ./make-test.nix ({ pkgs, ... } :
let
connAuthFile="beegfs/auth-def.key";
client = { config, pkgs, lib, ... } : {
networking.firewall.enable = false;
services.beegfsEnable = true;
services.beegfs.default = {
mgmtdHost = "mgmt";
connAuthFile = "/etc/${connAuthFile}";
client = {
mount = false;
enable = true;
};
};
fileSystems = pkgs.lib.mkVMOverride # FIXME: this should be creatd by the module
[ { mountPoint = "/beegfs";
device = "default";
fsType = "beegfs";
options = [ "cfgFile=/etc/beegfs/client-default.conf" "_netdev" ];
}
];
environment.etc."${connAuthFile}" = {
enable = true;
text = "ThisIsALousySecret";
mode = "0600";
};
};
server = service : { config, pkgs, lib, ... } : {
networking.firewall.enable = false;
boot.initrd.postDeviceCommands = ''
${pkgs.e2fsprogs}/bin/mkfs.ext4 -L data /dev/vdb
'';
virtualisation.emptyDiskImages = [ 4096 ];
fileSystems = pkgs.lib.mkVMOverride
[ { mountPoint = "/data";
device = "/dev/disk/by-label/data";
fsType = "ext4";
}
];
environment.systemPackages = with pkgs; [ beegfs ];
environment.etc."${connAuthFile}" = {
enable = true;
text = "ThisIsALousySecret";
mode = "0600";
};
services.beegfsEnable = true;
services.beegfs.default = {
mgmtdHost = "mgmt";
connAuthFile = "/etc/${connAuthFile}";
"${service}" = {
enable = true;
storeDir = "/data";
};
};
};
in
{
name = "beegfs";
nodes = {
meta = server "meta";
mgmt = server "mgmtd";
storage1 = server "storage";
storage2 = server "storage";
client1 = client;
client2 = client;
};
testScript = ''
# Initalize the data directories
$mgmt->waitForUnit("default.target");
$mgmt->succeed("beegfs-setup-mgmtd -C -f -p /data");
$mgmt->succeed("systemctl start beegfs-mgmtd-default");
$meta->waitForUnit("default.target");
$meta->succeed("beegfs-setup-meta -C -f -s 1 -p /data");
$meta->succeed("systemctl start beegfs-meta-default");
$storage1->waitForUnit("default.target");
$storage1->succeed("beegfs-setup-storage -C -f -s 1 -i 1 -p /data");
$storage1->succeed("systemctl start beegfs-storage-default");
$storage2->waitForUnit("default.target");
$storage2->succeed("beegfs-setup-storage -C -f -s 2 -i 2 -p /data");
$storage2->succeed("systemctl start beegfs-storage-default");
#
# Basic test
$client1->waitForUnit("beegfs.mount");
$client1->succeed("beegfs-check-servers-default");
$client1->succeed("echo test > /beegfs/test");
$client2->waitForUnit("beegfs.mount");
$client2->succeed("test -e /beegfs/test");
$client2->succeed("cat /beegfs/test | grep test");
# test raid0/stripping
$client1->succeed("dd if=/dev/urandom bs=1M count=10 of=/beegfs/striped");
$client2->succeed("cat /beegfs/striped > /dev/null");
# check if fs is still healthy
$client1->succeed("beegfs-fsck-default --checkfs");
'';
})

View File

@ -0,0 +1,122 @@
{ stdenv, fetchurl, pkgconfig, unzip, which
, libuuid, attr, xfsprogs, cppunit
, zlib, openssl, sqlite, jre, openjdk, ant
} :
let
version = "6.17";
subdirs = [
"beegfs_thirdparty/build"
"beegfs_opentk_lib/build"
"beegfs_common/build"
"beegfs_admon/build"
"beegfs_java_lib/build"
"beegfs_ctl/build"
"beegfs_fsck/build"
"beegfs_helperd/build"
"beegfs_meta/build"
"beegfs_mgmtd/build"
"beegfs_online_cfg/build"
"beegfs_storage/build"
"beegfs_utils/build"
];
in stdenv.mkDerivation rec {
name = "beegfs-${version}";
src = fetchurl {
url = "https://git.beegfs.com/pub/v6/repository/archive.tar.bz2?ref=${version}";
sha256 = "10xs7gzdmlg23k6zn1b7jij3lljn7rr1j6h476hq4lbg981qk3n3";
};
nativeBuildInputs = [ which unzip pkgconfig cppunit openjdk ant];
buildInputs = [ libuuid attr xfsprogs zlib openssl sqlite jre ];
postPatch = ''
patchShebangs ./
find -type f -name Makefile -exec sed -i "s:/bin/bash:${stdenv.shell}:" \{} \;
find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \;
find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \;
'';
buildPhase = ''
for i in ${toString subdirs}; do
make -C $i
done
make -C beegfs_admon/build admon_gui
'';
installPhase = ''
binDir=$out/bin
docDir=$out/share/doc/beegfs
includeDir=$out/include/beegfs
libDir=$out/lib
libDirPkg=$out/lib/beegfs
mkdir -p $binDir $libDir $libDirPkg $docDir $includeDir
cp beegfs_admon/build/beegfs-admon $binDir
cp beegfs_admon/build/dist/usr/bin/beegfs-admon-gui $binDir
cp beegfs_admon_gui/dist/beegfs-admon-gui.jar $libDirPkg
cp beegfs_admon/build/dist/etc/beegfs-admon.conf $docDir
cp beegfs_java_lib/build/jbeegfs.jar $libDirPkg
cp beegfs_java_lib/build/libjbeegfs.so $libDir
cp beegfs_ctl/build/beegfs-ctl $binDir
cp beegfs_fsck/build/beegfs-fsck $binDir
cp beegfs_utils/scripts/beegfs-check-servers $binDir
cp beegfs_utils/scripts/beegfs-df $binDir
cp beegfs_utils/scripts/beegfs-net $binDir
cp beegfs_helperd/build/beegfs-helperd $binDir
cp beegfs_helperd/build/dist/etc/beegfs-helperd.conf $docDir
cp beegfs_client_module/build/dist/sbin/beegfs-setup-client $binDir
cp beegfs_client_module/build/dist/etc/beegfs-client.conf $docDir
cp beegfs_meta/build/beegfs-meta $binDir
cp beegfs_meta/build/dist/sbin/beegfs-setup-meta $binDir
cp beegfs_meta/build/dist/etc/beegfs-meta.conf $docDir
cp beegfs_mgmtd/build/beegfs-mgmtd $binDir
cp beegfs_mgmtd/build/dist/sbin/beegfs-setup-mgmtd $binDir
cp beegfs_mgmtd/build/dist/etc/beegfs-mgmtd.conf $docDir
cp beegfs_storage/build/beegfs-storage $binDir
cp beegfs_storage/build/dist/sbin/beegfs-setup-storage $binDir
cp beegfs_storage/build/dist/etc/beegfs-storage.conf $docDir
cp beegfs_opentk_lib/build/libbeegfs-opentk.so $libDir
cp beegfs_client_devel/build/dist/usr/share/doc/beegfs-client-devel/examples/* $docDir
cp -r beegfs_client_devel/include/* $includeDir
'';
postFixup = ''
substituteInPlace $out/bin/beegfs-admon-gui \
--replace " java " " ${jre}/bin/java " \
--replace "/opt/beegfs/beegfs-admon-gui/beegfs-admon-gui.jar" \
"$libDirPkg/beegfs-admon-gui.jar"
'';
doCheck = true;
checkPhase = ''
beegfs_common/build/test-runner --text
'';
meta = with stdenv.lib; {
description = "High performance distributed filesystem with RDMA support";
homepage = "https://www.beegfs.io";
platforms = [ "i686-linux" "x86_64-linux" ];
license = {
fullName = "BeeGFS_EULA";
url = "https://www.beegfs.io/docs/BeeGFS_EULA.txt";
free = false;
};
maintainers = with maintainers; [ markuskowa ];
};
}

View File

@ -0,0 +1,43 @@
{ stdenv, fetchurl, which
, kmod, kernel
} :
let
version = "6.17";
in stdenv.mkDerivation {
name = "beegfs-module-${version}-${kernel.version}";
src = fetchurl {
url = "https://git.beegfs.com/pub/v6/repository/archive.tar.bz2?ref=${version}";
sha256 = "10xs7gzdmlg23k6zn1b7jij3lljn7rr1j6h476hq4lbg981qk3n3";
};
hardeningDisable = [ "fortify" "pic" "stackprotector" ];
nativeBuildInputs = [ which kmod ];
makeFlags = [ "KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build/" ];
postPatch = ''
patchShebangs ./
find -type f -name Makefile -exec sed -i "s:/bin/bash:${stdenv.shell}:" \{} \;
find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \;
find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \;
'';
preBuild = "cd beegfs_client_module/build";
installPhase = ''
instdir=$out/lib/modules/${kernel.modDirVersion}/extras/fs/beegfs
mkdir -p $instdir
cp beegfs.ko $instdir
'';
meta = with stdenv.lib; {
description = "High performance distributed filesystem with RDMA support";
homepage = "https://www.beegfs.io";
platforms = [ "i686-linux" "x86_64-linux" ];
license = licenses.gpl2;
maintainers = with maintainers; [ markuskowa ];
};
}

View File

@ -1338,6 +1338,8 @@ with pkgs;
beanstalkd = callPackage ../servers/beanstalkd { };
beegfs = callPackage ../os-specific/linux/beegfs { };
beets = callPackage ../tools/audio/beets {
pythonPackages = python2Packages;
};
@ -12821,6 +12823,8 @@ with pkgs;
bbswitch = callPackage ../os-specific/linux/bbswitch {};
beegfs-module = callPackage ../os-specific/linux/beegfs/kernel-module.nix { };
ati_drivers_x11 = callPackage ../os-specific/linux/ati-drivers { };
blcr = callPackage ../os-specific/linux/blcr { };