Merge pull request #54400 from veprbl/pr/arrow_0.12.0

arrow-cpp, pythonPackages.pyarrow: 0.11.0 -> 0.12.0
This commit is contained in:
Silvan Mosberger 2019-01-24 00:42:02 +01:00 committed by GitHub
commit b6da1ca26c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 13 additions and 73 deletions

View File

@ -1,4 +1,4 @@
{ stdenv, symlinkJoin, fetchurl, fetchFromGitHub, boost, brotli, cmake, double-conversion, flatbuffers, gflags, glog, gtest, lz4, perl, python, rapidjson, snappy, thrift, which, zlib, zstd }:
{ stdenv, symlinkJoin, fetchurl, fetchFromGitHub, autoconf, boost, brotli, cmake, double-conversion, flatbuffers, gflags, glog, gtest, lz4, perl, python, rapidjson, snappy, thrift, which, zlib, zstd }:
let
parquet-testing = fetchFromGitHub {
@ -11,27 +11,21 @@ in
stdenv.mkDerivation rec {
name = "arrow-cpp-${version}";
version = "0.11.0";
version = "0.12.0";
src = fetchurl {
url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz";
sha256 = "0pc5pqr0dbnx8s1ji102dhw9bbrsq3ml4ac3mmi2022yfyizlf0q";
sha256 = "163s4i2cywq95jgrxbaq48qwmww0ibkq61k1aad4w9z9vpjfgnil";
};
sourceRoot = "apache-arrow-${version}/cpp";
patches = [
# fix ARROW-3467
./double-conversion_cmake.patch
# patch to fix python-test
./darwin.patch
# facebook/zstd#1385
./zstd136.patch
];
nativeBuildInputs = [ cmake ];
nativeBuildInputs = [ cmake autoconf /* for vendored jemalloc */ ];
buildInputs = [ boost double-conversion glog python.pkgs.python python.pkgs.numpy ];
preConfigure = ''
@ -58,6 +52,7 @@ stdenv.mkDerivation rec {
ZSTD_HOME = zstd;
cmakeFlags = [
"-DARROW_BUILD_TESTS=ON"
"-DARROW_PYTHON=ON"
"-DARROW_PARQUET=ON"
];

View File

@ -1,43 +0,0 @@
diff --git a/CMakeLists.txt b/cpp/CMakeLists.txt
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -615,7 +615,7 @@ endif(UNIX)
set(ARROW_LINK_LIBS)
# Libraries to link statically with libarrow.so
-set(ARROW_STATIC_LINK_LIBS double-conversion)
+set(ARROW_STATIC_LINK_LIBS ${DOUBLE_CONVERSION_TARGET})
if (ARROW_WITH_BROTLI)
SET(ARROW_STATIC_LINK_LIBS
@@ -694,7 +694,7 @@ else ()
set(ARROW_MIN_TEST_LIBS
arrow_shared
${ARROW_LINK_LIBS}
- double-conversion
+ ${DOUBLE_CONVERSION_TARGET}
${BOOST_SYSTEM_LIBRARY}
${BOOST_FILESYSTEM_LIBRARY}
${BOOST_REGEX_LIBRARY}
diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
@@ -469,14 +469,16 @@ if("${DOUBLE_CONVERSION_HOME}" STREQUAL "")
set(DOUBLE_CONVERSION_VENDORED 1)
else()
find_package(double-conversion REQUIRED)
+ set(DOUBLE_CONVERSION_TARGET double-conversion::double-conversion)
set(DOUBLE_CONVERSION_VENDORED 0)
endif()
include_directories(SYSTEM ${DOUBLE_CONVERSION_INCLUDE_DIR})
-ADD_THIRDPARTY_LIB(double-conversion
- STATIC_LIB ${DOUBLE_CONVERSION_STATIC_LIB})
if (DOUBLE_CONVERSION_VENDORED)
+ ADD_THIRDPARTY_LIB(double-conversion
+ STATIC_LIB ${DOUBLE_CONVERSION_STATIC_LIB})
+ set(DOUBLE_CONVERSION_TARGET double-conversion)
add_dependencies(arrow_dependencies double-conversion_ep)
endif()

View File

@ -1,17 +0,0 @@
--- a/src/arrow/util/compression_zstd.cc
+++ b/src/arrow/util/compression_zstd.cc
@@ -35,8 +35,13 @@ namespace util {
Status ZSTDCodec::Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
uint8_t* output_buffer) {
+ void *safe_output_buffer = static_cast<void*>(output_buffer);
+ int dummy {};
+ if ((output_len == 0) && (output_buffer == NULL)) {
+ safe_output_buffer = static_cast<void*>(&dummy);
+ }
int64_t decompressed_size =
- ZSTD_decompress(output_buffer, static_cast<size_t>(output_len), input,
+ ZSTD_decompress(safe_output_buffer, static_cast<size_t>(output_len), input,
static_cast<size_t>(input_len));
if (decompressed_size != output_len) {
return Status::IOError("Corrupt ZSTD compressed data.");

View File

@ -1,4 +1,4 @@
{ lib, buildPythonPackage, python, isPy3k, fetchurl, arrow-cpp, cmake, cython, futures, numpy, pandas, pytest, pkgconfig, setuptools_scm, six }:
{ lib, buildPythonPackage, python, isPy3k, fetchurl, arrow-cpp, cmake, cython, futures, hypothesis, numpy, pandas, pytest, pkgconfig, setuptools_scm, six }:
let
_arrow-cpp = arrow-cpp.override { inherit python; };
@ -13,10 +13,15 @@ buildPythonPackage rec {
nativeBuildInputs = [ cmake cython pkgconfig setuptools_scm ];
propagatedBuildInputs = [ numpy six ] ++ lib.optionals (!isPy3k) [ futures ];
checkInputs = [ pandas pytest ];
checkInputs = [ hypothesis pandas pytest ];
PYARROW_BUILD_TYPE = "release";
PYARROW_CMAKE_OPTIONS = "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib";
PYARROW_CMAKE_OPTIONS = [
"-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib"
# for some reason cmake won't set -std=c++11 for clang
"-DPYARROW_CXXFLAGS=-std=c++11"
];
preCheck = ''
rm pyarrow/tests/test_jvm.py