Merge pull request #263048 from Madouura/pr/triton
Make openai-triton/pytorch free again (and various fixes)
This commit is contained in:
Martin Weinelt
GitHub
commit
6cd5fb6f8b
@ -22,7 +22,7 @@
|
||||
}:
|
||||
|
||||
stdenv.mkDerivation (finalAttrs: {
|
||||
pname = "triton-llvm";
|
||||
pname = "openai-triton-llvm";
|
||||
version = "14.0.6-f28c006a5895";
|
||||
|
||||
outputs = [
|
||||
@ -0,0 +1,35 @@
|
||||
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
|
||||
index 11c1b04c..762eaa75 100644
|
||||
--- a/test/CMakeLists.txt
|
||||
+++ b/test/CMakeLists.txt
|
||||
@@ -1,20 +1,14 @@
|
||||
-find_package(GTestSrc)
|
||||
+enable_testing()
|
||||
|
||||
-IF(GTESTSRC_FOUND)
|
||||
- enable_testing()
|
||||
+if (WIN32 AND (NOT CYGWIN) AND (NOT MINGW))
|
||||
+ set(gtest_disable_pthreads ON)
|
||||
+ set(gtest_force_shared_crt ON)
|
||||
+endif()
|
||||
|
||||
- if (WIN32 AND (NOT CYGWIN) AND (NOT MINGW))
|
||||
- set(gtest_disable_pthreads ON)
|
||||
- set(gtest_force_shared_crt ON)
|
||||
- endif()
|
||||
+include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
|
||||
|
||||
- add_subdirectory(${GTEST_SOURCE_DIR} ${CMAKE_BINARY_DIR}/googletest)
|
||||
- include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
|
||||
+set(TEST_LIBRARIES gtest gtest_main)
|
||||
|
||||
- set(TEST_LIBRARIES gtest gtest_main)
|
||||
-
|
||||
- add_custom_target(tests ALL)
|
||||
- add_subdirectory(perftest)
|
||||
- add_subdirectory(unittest)
|
||||
-
|
||||
-ENDIF(GTESTSRC_FOUND)
|
||||
+add_custom_target(tests ALL)
|
||||
+add_subdirectory(perftest)
|
||||
+add_subdirectory(unittest)
|
||||
@ -0,0 +1,36 @@
|
||||
diff --git a/test/valgrind.supp b/test/valgrind.supp
|
||||
index c9d3d226..63af7bf9 100644
|
||||
--- a/test/valgrind.supp
|
||||
+++ b/test/valgrind.supp
|
||||
@@ -24,3 +24,31 @@
|
||||
fun:*Uri*Parse_UTF16_Std*
|
||||
}
|
||||
|
||||
+{
|
||||
+ Suppress memcpy_chk valgrind report 5
|
||||
+ Memcheck:Overlap
|
||||
+ fun:__memcpy_chk
|
||||
+ fun:*dtoa_normal*
|
||||
+}
|
||||
+
|
||||
+{
|
||||
+ Suppress memcpy_chk valgrind report 6
|
||||
+ Memcheck:Overlap
|
||||
+ fun:__memcpy_chk
|
||||
+ fun:*dtoa_maxDecimalPlaces*
|
||||
+}
|
||||
+
|
||||
+{
|
||||
+ Suppress memcpy_chk valgrind report 7
|
||||
+ Memcheck:Overlap
|
||||
+ fun:__memcpy_chk
|
||||
+ ...
|
||||
+ fun:*Reader*ParseDoubleHandler*
|
||||
+}
|
||||
+
|
||||
+{
|
||||
+ Suppress memcpy_chk valgrind report 8
|
||||
+ Memcheck:Overlap
|
||||
+ fun:__memcpy_chk
|
||||
+ fun:*Reader*ParseNumber_NormalPrecisionError*
|
||||
+}
|
||||
@ -1,17 +1,30 @@
|
||||
{ lib
|
||||
, stdenv
|
||||
, fetchFromGitHub
|
||||
, fetchpatch
|
||||
, pkg-config
|
||||
, cmake
|
||||
, doxygen
|
||||
, graphviz
|
||||
, gtest
|
||||
, valgrind
|
||||
# One of "11" or "17"; default in source is CXX 11
|
||||
, cxxStandard ? "11"
|
||||
, buildDocs ? true
|
||||
, buildTests ? !stdenv.hostPlatform.isStatic && !stdenv.isDarwin
|
||||
, buildExamples ? true
|
||||
}:
|
||||
|
||||
stdenv.mkDerivation rec {
|
||||
stdenv.mkDerivation (finalAttrs: {
|
||||
pname = "rapidjson";
|
||||
version = "unstable-2023-09-28";
|
||||
|
||||
outputs = [
|
||||
"out"
|
||||
] ++ lib.optionals buildDocs [
|
||||
"doc"
|
||||
] ++ lib.optionals buildExamples [
|
||||
"example"
|
||||
];
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "Tencent";
|
||||
repo = "rapidjson";
|
||||
@ -19,26 +32,44 @@ stdenv.mkDerivation rec {
|
||||
hash = "sha256-rl7iy14jn1K2I5U2DrcZnoTQVEGEDKlxmdaOCF/3hfY=";
|
||||
};
|
||||
|
||||
patches = lib.optionals buildTests [
|
||||
./0000-unstable-use-nixpkgs-gtest.patch
|
||||
# https://github.com/Tencent/rapidjson/issues/2214
|
||||
./0001-unstable-valgrind-suppress-failures.patch
|
||||
];
|
||||
|
||||
nativeBuildInputs = [
|
||||
pkg-config
|
||||
cmake
|
||||
] ++ lib.optionals buildDocs [
|
||||
doxygen
|
||||
graphviz
|
||||
];
|
||||
|
||||
patches = [
|
||||
(fetchpatch {
|
||||
name = "do-not-include-gtest-src-dir.patch";
|
||||
url = "https://git.alpinelinux.org/aports/plain/community/rapidjson/do-not-include-gtest-src-dir.patch?id=9e5eefc7a5fcf5938a8dc8a3be8c75e9e6809909";
|
||||
hash = "sha256-BjSZEwfCXA/9V+kxQ/2JPWbc26jQn35CfN8+8NW24s4=";
|
||||
})
|
||||
cmakeFlags = [
|
||||
(lib.cmakeBool "RAPIDJSON_BUILD_DOC" buildDocs)
|
||||
(lib.cmakeBool "RAPIDJSON_BUILD_TESTS" buildTests)
|
||||
(lib.cmakeBool "RAPIDJSON_BUILD_EXAMPLES" buildExamples)
|
||||
(lib.cmakeBool "RAPIDJSON_BUILD_CXX11" (cxxStandard == "11"))
|
||||
(lib.cmakeBool "RAPIDJSON_BUILD_CXX17" (cxxStandard == "17"))
|
||||
] ++ lib.optionals buildTests [
|
||||
(lib.cmakeFeature "GTEST_INCLUDE_DIR" "${lib.getDev gtest}")
|
||||
];
|
||||
|
||||
# for tests, adding gtest to checkInputs does not work
|
||||
# https://github.com/NixOS/nixpkgs/pull/212200
|
||||
buildInputs = [ gtest ];
|
||||
cmakeFlags = [ "-DGTEST_SOURCE_DIR=${gtest.dev}/include" ];
|
||||
doCheck = buildTests;
|
||||
|
||||
nativeCheckInputs = [ valgrind ];
|
||||
doCheck = !stdenv.hostPlatform.isStatic && !stdenv.isDarwin;
|
||||
nativeCheckInputs = [
|
||||
gtest
|
||||
valgrind
|
||||
];
|
||||
|
||||
postInstall = lib.optionalString buildExamples ''
|
||||
mkdir -p $example/bin
|
||||
|
||||
find bin -type f -executable \
|
||||
-not -name "perftest" \
|
||||
-not -name "unittest" \
|
||||
-exec cp -a {} $example/bin \;
|
||||
'';
|
||||
|
||||
meta = with lib; {
|
||||
description = "Fast JSON parser/generator for C++ with both SAX/DOM style API";
|
||||
@ -46,5 +77,6 @@ stdenv.mkDerivation rec {
|
||||
license = licenses.mit;
|
||||
platforms = platforms.unix;
|
||||
maintainers = with maintainers; [ Madouura ];
|
||||
broken = (cxxStandard != "11" && cxxStandard != "17");
|
||||
};
|
||||
}
|
||||
})
|
||||
|
||||
@ -28,6 +28,7 @@
|
||||
, pyro-ppl
|
||||
#, pystan (not packaged)
|
||||
, numpyro
|
||||
, bokeh
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
@ -72,6 +73,7 @@ buildPythonPackage rec {
|
||||
pytestCheckHook
|
||||
torchvision
|
||||
zarr
|
||||
bokeh
|
||||
];
|
||||
|
||||
preCheck = ''
|
||||
@ -97,6 +99,8 @@ buildPythonPackage rec {
|
||||
# An issue has been opened upstream: https://github.com/arviz-devs/arviz/issues/2282
|
||||
"test_plot_ppc_discrete"
|
||||
"test_plot_ppc_discrete_save_animation"
|
||||
# Assertion error
|
||||
"test_data_zarr"
|
||||
];
|
||||
|
||||
pythonImportsCheck = [
|
||||
|
||||
@ -69,6 +69,10 @@ buildPythonPackage rec {
|
||||
disabledTests = [
|
||||
# exact comparison of floating points
|
||||
"test_optimize_l0_homotopy"
|
||||
# AssertionError: 5 != 2
|
||||
"test_get_standard_plots_moo"
|
||||
# AssertionError: Expected 'warning' to be called once. Called 3 times
|
||||
"test_validate_kwarg_typing"
|
||||
];
|
||||
pythonImportsCheck = [ "ax" ];
|
||||
|
||||
|
||||
@ -6,23 +6,27 @@
|
||||
, termcolor
|
||||
, tqdm
|
||||
, pandas
|
||||
, setuptools
|
||||
# test dependencies
|
||||
, pytestCheckHook
|
||||
, pytest-lazy-fixture
|
||||
, tensorflow
|
||||
, torch
|
||||
, datasets
|
||||
, torchvision
|
||||
, keras
|
||||
, fasttext
|
||||
, hypothesis
|
||||
, wget
|
||||
, matplotlib
|
||||
, skorch
|
||||
}:
|
||||
let
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "cleanlab";
|
||||
version = "2.5.0";
|
||||
in
|
||||
buildPythonPackage {
|
||||
inherit pname version;
|
||||
format = "setuptools";
|
||||
|
||||
disabled = pythonOlder "3.8";
|
||||
pyproject = true;
|
||||
disabled = pythonOlder "3.7";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "cleanlab";
|
||||
@ -31,11 +35,7 @@ buildPythonPackage {
|
||||
hash = "sha256-5XQQVrhjpvjwtFM79DqttObmw/GQLkMQVXb5jhiC8e0=";
|
||||
};
|
||||
|
||||
# postPatch = ''
|
||||
# substituteInPlace pyproject.toml \
|
||||
# --replace '"rich <= 13.0.1"' '"rich"' \
|
||||
# --replace '"numpy < 1.24.0"' '"numpy"'
|
||||
# '';
|
||||
nativeBuildInputs = [ setuptools ];
|
||||
|
||||
propagatedBuildInputs = [
|
||||
scikit-learn
|
||||
@ -44,13 +44,38 @@ buildPythonPackage {
|
||||
pandas
|
||||
];
|
||||
|
||||
# This is ONLY turned off when we have testing enabled.
|
||||
# The reason we do this is because of duplicate packages in the enclosure
|
||||
# when using the packages in nativeCheckInputs.
|
||||
# Affected packages: grpcio protobuf tensorboard tensorboard-plugin-profile
|
||||
catchConflicts = (!doCheck);
|
||||
doCheck = true;
|
||||
|
||||
nativeCheckInputs = [
|
||||
pytestCheckHook
|
||||
pytest-lazy-fixture
|
||||
tensorflow
|
||||
torch
|
||||
datasets
|
||||
torchvision
|
||||
keras
|
||||
fasttext
|
||||
hypothesis
|
||||
wget
|
||||
matplotlib
|
||||
skorch
|
||||
];
|
||||
|
||||
disabledTests = [
|
||||
# Requires the datasets we prevent from downloading
|
||||
"test_create_imagelab"
|
||||
];
|
||||
|
||||
disabledTestPaths = [
|
||||
# Requires internet
|
||||
"tests/test_dataset.py"
|
||||
# Requires the datasets we just prevented from downloading
|
||||
"tests/datalab/test_cleanvision_integration.py"
|
||||
];
|
||||
|
||||
meta = with lib; {
|
||||
|
||||
@ -12,19 +12,20 @@
|
||||
, scs
|
||||
, setuptools
|
||||
, wheel
|
||||
, pybind11
|
||||
, useOpenmp ? (!stdenv.isDarwin)
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "cvxpy";
|
||||
version = "1.3.2";
|
||||
version = "1.4.1";
|
||||
format = "pyproject";
|
||||
|
||||
disabled = pythonOlder "3.7";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
hash = "sha256-C2heUEDxmfPXA/MPXSLR+GVZdiNFUVPR3ddwJFrvCXU=";
|
||||
hash = "sha256-ep7zTjxX/4yETYbwo4NPtVda8ZIzlHY53guld8YSLj4=";
|
||||
};
|
||||
|
||||
# we need to patch out numpy version caps from upstream
|
||||
@ -35,6 +36,7 @@ buildPythonPackage rec {
|
||||
nativeBuildInputs = [
|
||||
setuptools
|
||||
wheel
|
||||
pybind11
|
||||
];
|
||||
|
||||
propagatedBuildInputs = [
|
||||
@ -44,7 +46,6 @@ buildPythonPackage rec {
|
||||
osqp
|
||||
scipy
|
||||
scs
|
||||
setuptools
|
||||
];
|
||||
|
||||
nativeCheckInputs = [
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
, buildPythonPackage
|
||||
, pythonOlder
|
||||
, fetchFromGitHub
|
||||
, fetchpatch
|
||||
|
||||
# Native build inputs
|
||||
, cython
|
||||
@ -27,12 +28,12 @@
|
||||
, hypothesis
|
||||
, pytestCheckHook
|
||||
}:
|
||||
let
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "fairseq";
|
||||
version = "0.12.3";
|
||||
in
|
||||
buildPythonPackage rec {
|
||||
inherit version pname;
|
||||
pyproject = true;
|
||||
disabled = pythonOlder "3.7";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "pytorch";
|
||||
@ -41,17 +42,25 @@ buildPythonPackage rec {
|
||||
hash = "sha256-XX/grU5ljQCwx33miGoFc/7Uj9fZDtmhm4Fz7L4U+Bc=";
|
||||
};
|
||||
|
||||
disabled = pythonOlder "3.7";
|
||||
patches = [
|
||||
# https://github.com/facebookresearch/fairseq/pull/5359
|
||||
(fetchpatch {
|
||||
url = "https://github.com/facebookresearch/fairseq/commit/2fa0768c2115b0a4c207cfa3e1b3e4ff3ad9a00c.patch";
|
||||
hash = "sha256-aYYP/knQX6q6vhyA6q9uOOYfRhDAuJCo9QJWfFEDuuA=";
|
||||
})
|
||||
];
|
||||
|
||||
nativeBuildInputs = [
|
||||
cython
|
||||
pythonRelaxDepsHook
|
||||
which
|
||||
];
|
||||
|
||||
pythonRelaxDeps = [
|
||||
"hydra-core"
|
||||
"omegaconf"
|
||||
];
|
||||
|
||||
propagatedBuildInputs = [
|
||||
cffi
|
||||
hydra-core
|
||||
@ -74,6 +83,7 @@ buildPythonPackage rec {
|
||||
];
|
||||
|
||||
pythonImportsCheck = [ "fairseq" ];
|
||||
|
||||
preCheck = ''
|
||||
export HOME=$TMPDIR
|
||||
cd tests
|
||||
@ -82,6 +92,7 @@ buildPythonPackage rec {
|
||||
pytestFlagsArray = [
|
||||
"--import-mode append"
|
||||
];
|
||||
|
||||
disabledTests = [
|
||||
# this test requires xformers
|
||||
"test_xformers_single_forward_parity"
|
||||
@ -96,6 +107,11 @@ buildPythonPackage rec {
|
||||
"test_librispeech_s2t_conformer_s_checkpoint"
|
||||
];
|
||||
|
||||
disabledTestPaths = [
|
||||
# ValueError: mutable default ... for field bar is not allowed: use default_factory
|
||||
"test_dataclass_utils.py"
|
||||
];
|
||||
|
||||
meta = with lib; {
|
||||
description = "Facebook AI Research Sequence-to-Sequence Toolkit";
|
||||
homepage = "https://github.com/pytorch/fairseq";
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
, pygments
|
||||
, pyqt5
|
||||
, pythonOlder
|
||||
, pythonAtLeast
|
||||
, traitsui
|
||||
, vtk
|
||||
, wrapQtAppsHook
|
||||
@ -18,8 +19,7 @@ buildPythonPackage rec {
|
||||
pname = "mayavi";
|
||||
version = "4.8.1";
|
||||
format = "setuptools";
|
||||
|
||||
disabled = pythonOlder "3.8";
|
||||
disabled = pythonOlder "3.8" || pythonAtLeast "3.11";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
diff --git a/omegaconf/omegaconf.py b/omegaconf/omegaconf.py
|
||||
index efde14a..a2a050e 100644
|
||||
--- a/omegaconf/omegaconf.py
|
||||
+++ b/omegaconf/omegaconf.py
|
||||
@@ -7,6 +7,7 @@ import pathlib
|
||||
import sys
|
||||
import warnings
|
||||
from collections import defaultdict
|
||||
+from dataclasses import _MISSING_TYPE
|
||||
from contextlib import contextmanager
|
||||
from enum import Enum
|
||||
from textwrap import dedent
|
||||
@@ -828,6 +829,8 @@ class OmegaConf:
|
||||
|
||||
if obj is _DEFAULT_MARKER_:
|
||||
obj = {}
|
||||
+ if isinstance(obj, _MISSING_TYPE):
|
||||
+ return OmegaConf.create({}, parent=parent, flags=flags)
|
||||
if isinstance(obj, str):
|
||||
obj = yaml.load(obj, Loader=get_yaml_loader())
|
||||
if obj is None:
|
||||
@ -3,6 +3,7 @@
|
||||
, antlr4-python3-runtime
|
||||
, buildPythonPackage
|
||||
, fetchFromGitHub
|
||||
, setuptools
|
||||
, jre_minimal
|
||||
, pydevd
|
||||
, pytest-mock
|
||||
@ -15,8 +16,7 @@
|
||||
buildPythonPackage rec {
|
||||
pname = "omegaconf";
|
||||
version = "2.3.0";
|
||||
format = "setuptools";
|
||||
|
||||
pyproject = true;
|
||||
disabled = pythonOlder "3.6";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
@ -31,6 +31,9 @@ buildPythonPackage rec {
|
||||
src = ./antlr4.patch;
|
||||
antlr_jar = "${antlr4.out}/share/java/antlr-${antlr4.version}-complete.jar";
|
||||
})
|
||||
|
||||
# https://github.com/omry/omegaconf/pull/1137
|
||||
./0000-add-support-for-dataclasses_missing_type.patch
|
||||
];
|
||||
|
||||
postPatch = ''
|
||||
@ -42,6 +45,7 @@ buildPythonPackage rec {
|
||||
'';
|
||||
|
||||
nativeBuildInputs = [
|
||||
setuptools
|
||||
jre_minimal
|
||||
];
|
||||
|
||||
|
||||
@ -0,0 +1,39 @@
|
||||
diff --git a/python/setup.py b/python/setup.py
|
||||
index 2ac3accd2..f26161c72 100644
|
||||
--- a/python/setup.py
|
||||
+++ b/python/setup.py
|
||||
@@ -101,25 +101,6 @@ def get_thirdparty_packages(triton_cache_path):
|
||||
# ---- package data ---
|
||||
|
||||
|
||||
-def download_and_copy_ptxas():
|
||||
- base_dir = os.path.dirname(__file__)
|
||||
- src_path = "bin/ptxas"
|
||||
- url = "https://conda.anaconda.org/nvidia/label/cuda-12.0.0/linux-64/cuda-nvcc-12.0.76-0.tar.bz2"
|
||||
- dst_prefix = os.path.join(base_dir, "triton")
|
||||
- dst_suffix = os.path.join("third_party", "cuda", src_path)
|
||||
- dst_path = os.path.join(dst_prefix, dst_suffix)
|
||||
- if not os.path.exists(dst_path):
|
||||
- print(f'downloading and extracting {url} ...')
|
||||
- ftpstream = urllib.request.urlopen(url)
|
||||
- file = tarfile.open(fileobj=ftpstream, mode="r|*")
|
||||
- with tempfile.TemporaryDirectory() as temp_dir:
|
||||
- file.extractall(path=temp_dir)
|
||||
- src_path = os.path.join(temp_dir, src_path)
|
||||
- os.makedirs(os.path.split(dst_path)[0], exist_ok=True)
|
||||
- shutil.copy(src_path, dst_path)
|
||||
- return dst_suffix
|
||||
-
|
||||
-
|
||||
# ---- cmake extension ----
|
||||
|
||||
|
||||
@@ -200,8 +181,6 @@ class CMakeBuild(build_ext):
|
||||
subprocess.check_call(["cmake", "--build", "."] + build_args, cwd=self.build_temp)
|
||||
|
||||
|
||||
-download_and_copy_ptxas()
|
||||
-
|
||||
setup(
|
||||
name="triton",
|
||||
version="2.0.0",
|
||||
@ -1,5 +1,5 @@
|
||||
{ lib
|
||||
, callPackage
|
||||
, config
|
||||
, buildPythonPackage
|
||||
, fetchFromGitHub
|
||||
, addOpenGLRunpath
|
||||
@ -14,10 +14,12 @@
|
||||
, ncurses
|
||||
, libxml2
|
||||
, lit
|
||||
, llvm
|
||||
, filelock
|
||||
, torchWithRocm
|
||||
, python
|
||||
, cudaPackages
|
||||
, cudaSupport ? config.cudaSupport
|
||||
}:
|
||||
|
||||
let
|
||||
@ -33,7 +35,6 @@ let
|
||||
# be executed on the GPU.
|
||||
# Cf. https://nixos.org/manual/nixpkgs/unstable/#sec-cross-infra
|
||||
ptxas = "${pkgsTargetTarget.cudaPackages.cuda_nvcc}/bin/ptxas"; # Make sure cudaPackages is the right version each update (See python/setup.py)
|
||||
llvm = callPackage ./llvm.nix { }; # Use a custom llvm, see llvm.nix for details
|
||||
in
|
||||
buildPythonPackage rec {
|
||||
pname = "triton";
|
||||
@ -57,6 +58,8 @@ buildPythonPackage rec {
|
||||
# url = "https://github.com/openai/triton/commit/fc7c0b0e437a191e421faa61494b2ff4870850f1.patch";
|
||||
# hash = "sha256-f0shIqHJkVvuil2Yku7vuqWFn7VCRKFSFjYRlwx25ig=";
|
||||
# })
|
||||
] ++ lib.optionals (!cudaSupport) [
|
||||
./0000-dont-download-ptxas.patch
|
||||
];
|
||||
|
||||
nativeBuildInputs = [
|
||||
@ -102,21 +105,18 @@ buildPythonPackage rec {
|
||||
substituteInPlace bin/CMakeLists.txt \
|
||||
--replace "add_subdirectory(FileCheck)" ""
|
||||
|
||||
# Use our linker flags
|
||||
substituteInPlace python/triton/compiler.py \
|
||||
--replace '${oldStr}' '${newStr}'
|
||||
|
||||
# Don't fetch googletest
|
||||
substituteInPlace unittest/CMakeLists.txt \
|
||||
--replace "include (\''${CMAKE_CURRENT_SOURCE_DIR}/googletest.cmake)" ""\
|
||||
--replace "include(GoogleTest)" "find_package(GTest REQUIRED)"
|
||||
'' + lib.optionalString cudaSupport ''
|
||||
# Use our linker flags
|
||||
substituteInPlace python/triton/compiler.py \
|
||||
--replace '${oldStr}' '${newStr}'
|
||||
'';
|
||||
|
||||
# Avoid GLIBCXX mismatch with other cuda-enabled python packages
|
||||
preConfigure = ''
|
||||
export CC=${cudaPackages.backendStdenv.cc}/bin/cc;
|
||||
export CXX=${cudaPackages.backendStdenv.cc}/bin/c++;
|
||||
|
||||
# Upstream's setup.py tries to write cache somewhere in ~/
|
||||
export HOME=$(mktemp -d)
|
||||
|
||||
@ -127,6 +127,9 @@ buildPythonPackage rec {
|
||||
|
||||
# The rest (including buildPhase) is relative to ./python/
|
||||
cd python
|
||||
'' + lib.optionalString cudaSupport ''
|
||||
export CC=${cudaPackages.backendStdenv.cc}/bin/cc;
|
||||
export CXX=${cudaPackages.backendStdenv.cc}/bin/c++;
|
||||
|
||||
# Work around download_and_copy_ptxas()
|
||||
mkdir -p $PWD/triton/third_party/cuda/bin
|
||||
@ -137,7 +140,7 @@ buildPythonPackage rec {
|
||||
dontUseCmakeConfigure = true;
|
||||
|
||||
# Setuptools (?) strips runpath and +x flags. Let's just restore the symlink
|
||||
postFixup = ''
|
||||
postFixup = lib.optionalString cudaSupport ''
|
||||
rm -f $out/${python.sitePackages}/triton/third_party/cuda/bin/ptxas
|
||||
ln -s ${ptxas} $out/${python.sitePackages}/triton/third_party/cuda/bin/ptxas
|
||||
'';
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
, torch
|
||||
, torchvision
|
||||
, typing-extensions
|
||||
, pythonAtLeast
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
@ -41,6 +42,7 @@ buildPythonPackage rec {
|
||||
"tests/pytorch_pfn_extras_tests/onnx_tests/test_export.py"
|
||||
"tests/pytorch_pfn_extras_tests/onnx_tests/test_torchvision.py"
|
||||
"tests/pytorch_pfn_extras_tests/onnx_tests/utils.py"
|
||||
"tests/pytorch_pfn_extras_tests/onnx_tests/test_lax.py"
|
||||
|
||||
# RuntimeError: No Op registered for Gradient with domain_version of 9
|
||||
"tests/pytorch_pfn_extras_tests/onnx_tests/test_grad.py"
|
||||
@ -53,13 +55,15 @@ buildPythonPackage rec {
|
||||
"tests/pytorch_pfn_extras_tests/nn_tests/modules_tests/test_lazy.py"
|
||||
"tests/pytorch_pfn_extras_tests/profiler_tests/test_record.py"
|
||||
"tests/pytorch_pfn_extras_tests/runtime_tests/test_to.py"
|
||||
"tests/pytorch_pfn_extras_tests/test_handler.py"
|
||||
"tests/pytorch_pfn_extras_tests/test_logic.py"
|
||||
"tests/pytorch_pfn_extras_tests/handler_tests/test_handler.py"
|
||||
"tests/pytorch_pfn_extras_tests/test_reporter.py"
|
||||
"tests/pytorch_pfn_extras_tests/training_tests/test_trainer.py"
|
||||
"tests/pytorch_pfn_extras_tests/utils_tests/test_checkpoint.py"
|
||||
"tests/pytorch_pfn_extras_tests/utils_tests/test_comparer.py"
|
||||
"tests/pytorch_pfn_extras_tests/utils_tests/test_new_comparer.py"
|
||||
] ++ lib.optionals (pythonAtLeast "3.11") [
|
||||
# Remove this when https://github.com/NixOS/nixpkgs/pull/259068 is merged
|
||||
"tests/pytorch_pfn_extras_tests/dynamo_tests/test_compile.py"
|
||||
];
|
||||
|
||||
meta = with lib; {
|
||||
|
||||
@ -1,9 +1,8 @@
|
||||
{ lib
|
||||
, buildPythonPackage
|
||||
, fetchFromGitHub
|
||||
, fetchpatch
|
||||
, pythonOlder
|
||||
, pytestCheckHook
|
||||
, setuptools
|
||||
, gym
|
||||
, gymnasium
|
||||
, torch
|
||||
@ -11,13 +10,13 @@
|
||||
, tqdm
|
||||
, wandb
|
||||
, packaging
|
||||
, pytestCheckHook
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "skrl";
|
||||
version = "1.0.0";
|
||||
format = "setuptools";
|
||||
|
||||
pyproject = true;
|
||||
disabled = pythonOlder "3.6";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
@ -27,14 +26,7 @@ buildPythonPackage rec {
|
||||
hash = "sha256-89OoJanmaB74SLF1qMI8WFBdN1czS7Yr7BmojaRdo4M=";
|
||||
};
|
||||
|
||||
patches = [
|
||||
# remove after next release:
|
||||
(fetchpatch {
|
||||
name = "fix-python_requires-specification";
|
||||
url = "https://github.com/Toni-SM/skrl/pull/62/commits/9b554adfe2da6cd97cccbbcd418a349cc8f1de80.patch";
|
||||
hash = "sha256-GeASMU1Pgy8U1zaIAVroBDjYaY+n93XP5uFyP4U9lok=";
|
||||
})
|
||||
];
|
||||
nativeBuildInputs = [ setuptools ];
|
||||
|
||||
propagatedBuildInputs = [
|
||||
gym
|
||||
|
||||
@ -197,6 +197,30 @@
|
||||
"sha256": "0dmhv1fa46hi78jgv562v4x3mfl7svchs6kiz35s63ph9ik5r6f2",
|
||||
"license": "cc-by-sa-40"
|
||||
},
|
||||
{
|
||||
"pname": "ja_core_news_lg",
|
||||
"version": "3.7.0",
|
||||
"sha256": "1nb77kivzy0wixsw8ijmw78fffkpqa63kykqph04jzmh75ra4wvg",
|
||||
"license": "cc-by-sa-40"
|
||||
},
|
||||
{
|
||||
"pname": "ja_core_news_md",
|
||||
"version": "3.7.0",
|
||||
"sha256": "0p22bwc24q76cl7ndszvhqgllvq3ws3i3vbjsp5xvhxxls6p49x9",
|
||||
"license": "cc-by-sa-40"
|
||||
},
|
||||
{
|
||||
"pname": "ja_core_news_sm",
|
||||
"version": "3.7.0",
|
||||
"sha256": "0bfvkds4dqynjshk2lxfya9yfcnbvwjfhc6n7yh0852ms1ycicaw",
|
||||
"license": "cc-by-sa-40"
|
||||
},
|
||||
{
|
||||
"pname": "ja_core_news_trf",
|
||||
"version": "3.7.0",
|
||||
"sha256": "0n2lqql4flnilgf671n5mcdp8vi5pdjfz3vymxsapc1gyp29jykk",
|
||||
"license": "cc-by-sa-30"
|
||||
},
|
||||
{
|
||||
"pname": "it_core_news_lg",
|
||||
"version": "3.7.0",
|
||||
@ -377,6 +401,30 @@
|
||||
"sha256": "11mh1rd0q024xfagdqkly1n4nndksrlq650n51jl1x1pmzlsdgzl",
|
||||
"license": "mit"
|
||||
},
|
||||
{
|
||||
"pname": "sl_core_news_lg",
|
||||
"version": "3.7.0",
|
||||
"sha256": "128ayhp21szc31ckiq3i8vib42i9xnz4lpi1709gjdc38cpmpnlp",
|
||||
"license": "cc-by-sa-40"
|
||||
},
|
||||
{
|
||||
"pname": "sl_core_news_md",
|
||||
"version": "3.7.0",
|
||||
"sha256": "07gx174gw5q1zgyyg1xhfplihhnr311f9562ri5pdd2hgjyz58yb",
|
||||
"license": "cc-by-sa-40"
|
||||
},
|
||||
{
|
||||
"pname": "sl_core_news_sm",
|
||||
"version": "3.7.0",
|
||||
"sha256": "005xwsnh5y3pf8y64lhvrzcbh8y34yr3in204as6hv7krsfg8bxa",
|
||||
"license": "cc-by-sa-40"
|
||||
},
|
||||
{
|
||||
"pname": "sl_core_news_trf",
|
||||
"version": "3.7.0",
|
||||
"sha256": "0x97lwm1i2dq4kdg6rvarh9mnlcx45cnwq80qpjwv3b7zmviyq8c",
|
||||
"license": "cc-by-sa-40"
|
||||
},
|
||||
{
|
||||
"pname": "sv_core_news_lg",
|
||||
"version": "3.7.0",
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
diff --git a/stytra/gui/fishplots.py b/stytra/gui/fishplots.py
|
||||
index 49ef1fe..fd1cc50 100644
|
||||
--- a/stytra/gui/fishplots.py
|
||||
+++ b/stytra/gui/fishplots.py
|
||||
@@ -13,7 +13,7 @@ from lightparam.gui import ParameterGui
|
||||
from scipy.ndimage.filters import gaussian_filter1d
|
||||
|
||||
|
||||
-class StreamingPositionPlot(pg.GraphicsWindow):
|
||||
+class StreamingPosition(pg.GraphicsView):
|
||||
"""Plot that displays the virtual position of the fish"""
|
||||
|
||||
def __init__(self, *args, data_accumulator, n_points=500, **kwargs):
|
||||
diff --git a/stytra/utilities.py b/stytra/utilities.py
|
||||
index f79c4db..feaa7ef 100644
|
||||
--- a/stytra/utilities.py
|
||||
+++ b/stytra/utilities.py
|
||||
@@ -239,7 +239,7 @@ def recursive_update(d, u):
|
||||
:return:
|
||||
"""
|
||||
for k, v in u.items():
|
||||
- if isinstance(v, collections.Mapping):
|
||||
+ if isinstance(v, collections.ChainMap):
|
||||
d[k] = recursive_update(d.get(k, {}), v)
|
||||
else:
|
||||
d[k] = v
|
||||
@ -33,8 +33,7 @@
|
||||
buildPythonPackage rec {
|
||||
pname = "stytra";
|
||||
version = "0.8.34";
|
||||
format = "setuptools";
|
||||
|
||||
pyproject = true;
|
||||
disabled = pythonOlder "3.7";
|
||||
|
||||
src = fetchPypi {
|
||||
@ -42,10 +41,10 @@ buildPythonPackage rec {
|
||||
sha256 = "aab9d07575ef599a9c0ae505656e3c03ec753462df3c15742f1f768f2b578f0a";
|
||||
};
|
||||
|
||||
# crashes python
|
||||
preCheck = ''
|
||||
rm stytra/tests/test_z_experiments.py
|
||||
'';
|
||||
patches = [
|
||||
# https://github.com/portugueslab/stytra/issues/87
|
||||
./0000-workaround-pyqtgraph.patch
|
||||
];
|
||||
|
||||
propagatedBuildInputs = [
|
||||
opencv4
|
||||
@ -79,6 +78,11 @@ buildPythonPackage rec {
|
||||
pyserial
|
||||
];
|
||||
|
||||
disabledTestPaths = [
|
||||
# Crashes python
|
||||
"stytra/tests/test_z_experiments.py"
|
||||
];
|
||||
|
||||
meta = with lib; {
|
||||
description = "A modular package to control stimulation and track behaviour";
|
||||
homepage = "https://github.com/portugueslab/stytra";
|
||||
|
||||
@ -302,7 +302,7 @@ in buildPythonPackage rec {
|
||||
])
|
||||
++ lib.optionals rocmSupport [ rocmtoolkit_joined ];
|
||||
|
||||
buildInputs = [ blas blas.provider pybind11 ]
|
||||
buildInputs = [ blas blas.provider ]
|
||||
++ lib.optionals stdenv.isLinux [ linuxHeaders_5_19 ] # TMP: avoid "flexible array member" errors for now
|
||||
++ lib.optionals cudaSupport (with cudaPackages; [
|
||||
cuda_cccl.dev # <thrust/*>
|
||||
@ -355,17 +355,15 @@ in buildPythonPackage rec {
|
||||
|
||||
# the following are required for tensorboard support
|
||||
pillow six future tensorboard protobuf
|
||||
|
||||
# ROCm build and `torch.compile` requires openai-triton
|
||||
openai-triton
|
||||
|
||||
# torch/csrc requires `pybind11` at runtime
|
||||
pybind11
|
||||
]
|
||||
++ lib.optionals MPISupport [ mpi ]
|
||||
++ lib.optionals rocmSupport [ rocmtoolkit_joined ]
|
||||
# rocm build requires openai-triton;
|
||||
# openai-triton currently requires cuda_nvcc,
|
||||
# so not including it in the cpu-only build;
|
||||
# torch.compile relies on openai-triton,
|
||||
# so we include it for the cuda build as well
|
||||
++ lib.optionals (rocmSupport || cudaSupport) [
|
||||
openai-triton
|
||||
];
|
||||
++ lib.optionals rocmSupport [ rocmtoolkit_joined ];
|
||||
|
||||
# Tests take a long time and may be flaky, so just sanity-check imports
|
||||
doCheck = false;
|
||||
|
||||
@ -53,6 +53,8 @@ buildPythonPackage rec {
|
||||
"test_flan_t5_small"
|
||||
# AssertionError in output
|
||||
"test_google"
|
||||
# "addmm_impl_cpu_" not implemented for 'Half'
|
||||
"test_input_size_half_precision"
|
||||
];
|
||||
|
||||
disabledTestPaths = [
|
||||
|
||||
@ -14,6 +14,8 @@
|
||||
, pytestCheckHook
|
||||
, torchmetrics
|
||||
, pytorch-lightning
|
||||
, pytest-doctestplus
|
||||
, pytest-xdist
|
||||
}:
|
||||
|
||||
let
|
||||
@ -22,10 +24,11 @@ let
|
||||
in
|
||||
buildPythonPackage {
|
||||
inherit pname version;
|
||||
pyproject = true;
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "PyTorchLightning";
|
||||
repo = "metrics";
|
||||
owner = "Lightning-AI";
|
||||
repo = "torchmetrics";
|
||||
rev = "refs/tags/v${version}";
|
||||
hash = "sha256-g5JuTbiRd8yWx2nM3UE8ejOhuZ0XpAQdS5AC9AlrSFY=";
|
||||
};
|
||||
@ -51,22 +54,33 @@ buildPythonPackage {
|
||||
cloudpickle
|
||||
psutil
|
||||
pytestCheckHook
|
||||
pytest-doctestplus
|
||||
pytest-xdist
|
||||
];
|
||||
|
||||
# A cyclic dependency in: integrations/test_lightning.py
|
||||
doCheck = false;
|
||||
passthru.tests.check = torchmetrics.overridePythonAttrs (_: {
|
||||
pname = "${pname}-check";
|
||||
doCheck = true;
|
||||
# We don't have to install because the only purpose
|
||||
# of this passthru test is to, well, test.
|
||||
# This fixes having to set `catchConflicts` to false.
|
||||
dontInstall = true;
|
||||
});
|
||||
|
||||
disabledTests = [
|
||||
# `IndexError: list index out of range`
|
||||
"test_metric_lightning_log"
|
||||
];
|
||||
|
||||
disabledTestPaths = [
|
||||
# These require too many "leftpad-level" dependencies
|
||||
"tests/text"
|
||||
"tests/audio"
|
||||
"tests/image"
|
||||
# Also too cross-dependent
|
||||
"tests/unittests"
|
||||
|
||||
# A few non-deterministic things like test_check_compute_groups_is_faster
|
||||
"tests/bases/test_collections.py"
|
||||
# A trillion import path mismatch errors
|
||||
"src/torchmetrics"
|
||||
];
|
||||
|
||||
pythonImportsCheck = [
|
||||
@ -75,11 +89,10 @@ buildPythonPackage {
|
||||
|
||||
meta = with lib; {
|
||||
description = "Machine learning metrics for distributed, scalable PyTorch applications (used in pytorch-lightning)";
|
||||
homepage = "https://torchmetrics.readthedocs.io";
|
||||
homepage = "https://lightning.ai/docs/torchmetrics/";
|
||||
license = licenses.asl20;
|
||||
maintainers = with maintainers; [
|
||||
SomeoneSerge
|
||||
];
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -4160,7 +4160,9 @@ self: super: with self; {
|
||||
|
||||
oelint-parser = callPackage ../development/python-modules/oelint-parser { };
|
||||
|
||||
openllm = callPackage ../development/python-modules/openllm { };
|
||||
openllm = callPackage ../development/python-modules/openllm {
|
||||
openai-triton = self.openai-triton-cuda;
|
||||
};
|
||||
|
||||
openllm-client = callPackage ../development/python-modules/openllm-client { };
|
||||
|
||||
@ -6662,8 +6664,7 @@ self: super: with self; {
|
||||
maya = callPackage ../development/python-modules/maya { };
|
||||
|
||||
mayavi = pkgs.libsForQt5.callPackage ../development/python-modules/mayavi {
|
||||
inherit buildPythonPackage pythonOlder;
|
||||
inherit (self) pyface pygments numpy packaging vtk traitsui envisage apptools pyqt5;
|
||||
inherit (self) buildPythonPackage pythonOlder pythonAtLeast pyface pygments numpy packaging vtk traitsui envisage apptools pyqt5;
|
||||
};
|
||||
|
||||
mbddns = callPackage ../development/python-modules/mbddns { };
|
||||
@ -8429,12 +8430,24 @@ self: super: with self; {
|
||||
|
||||
open-meteo = callPackage ../development/python-modules/open-meteo { };
|
||||
|
||||
openai-triton = callPackage ../development/python-modules/openai-triton { cudaPackages = pkgs.cudaPackages_12_0; };
|
||||
openai-triton = callPackage ../development/python-modules/openai-triton {
|
||||
llvm = pkgs.openai-triton-llvm;
|
||||
cudaPackages = pkgs.cudaPackages_12_0;
|
||||
};
|
||||
|
||||
openai-triton-cuda = self.openai-triton.override {
|
||||
cudaSupport = true;
|
||||
};
|
||||
|
||||
openai-triton-no-cuda = self.openai-triton.override {
|
||||
cudaSupport = false;
|
||||
};
|
||||
|
||||
openai-triton-bin = callPackage ../development/python-modules/openai-triton/bin.nix { };
|
||||
|
||||
openai-whisper = callPackage ../development/python-modules/openai-whisper {
|
||||
inherit (pkgs.config) cudaSupport;
|
||||
openai-triton = self.openai-triton-cuda;
|
||||
};
|
||||
|
||||
openant = callPackage ../development/python-modules/openant { };
|
||||
@ -14062,6 +14075,7 @@ self: super: with self; {
|
||||
|
||||
torchWithCuda = self.torch.override {
|
||||
magma = pkgs.magma-cuda-static;
|
||||
openai-triton = self.openai-triton-cuda;
|
||||
cudaSupport = true;
|
||||
rocmSupport = false;
|
||||
};
|
||||
@ -14072,6 +14086,7 @@ self: super: with self; {
|
||||
|
||||
torchWithRocm = self.torch.override {
|
||||
magma = pkgs.magma-hip;
|
||||
openai-triton = self.openai-triton-no-cuda;
|
||||
rocmSupport = true;
|
||||
cudaSupport = false;
|
||||
};
|
||||
@ -15770,7 +15785,9 @@ self: super: with self; {
|
||||
inherit (pkgs) graphviz;
|
||||
};
|
||||
|
||||
xformers = callPackage ../development/python-modules/xformers { };
|
||||
xformers = callPackage ../development/python-modules/xformers {
|
||||
openai-triton = self.openai-triton-cuda;
|
||||
};
|
||||
|
||||
xgboost = callPackage ../development/python-modules/xgboost {
|
||||
inherit (pkgs) xgboost;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user