Merge pull request #128632 from ivan-tkatchev/airflow

2021-07-09 14:36:13 +02:00 · 2021-07-09 14:36:13 +02:00 · 35092090bf
commit 35092090bf
parent 3191d4f0af 2fee354ae4
11 changed files with 16595 additions and 101 deletions
--- a/pkgs/development/python-modules/apache-airflow/default.nix
+++ b/pkgs/development/python-modules/apache-airflow/default.nix
@ -1,165 +1,202 @@
 { lib
 , stdenv
+, python
 , buildPythonPackage
 , fetchFromGitHub
-, fetchpatch
+, writeText
 , alembic
+, argcomplete
+, attrs
+, blinker
 , cached-property
-, configparser
+, cattrs
+, clickclick
 , colorlog
 , croniter
+, cryptography
 , dill
 , flask
 , flask-appbuilder
-, flask-admin
 , flask-caching
 , flask_login
-, flask-swagger
 , flask_wtf
-, flask-bcrypt
-, funcsigs
-, future
 , GitPython
+, graphviz
 , gunicorn
+, httpx
 , iso8601
-, json-merge-patch
+, importlib-resources
+, importlib-metadata
+, inflection
+, itsdangerous
 , jinja2
-, ldap3
-, lxml
+, jsonschema
 , lazy-object-proxy
+, lockfile
 , markdown
+, markupsafe
+, marshmallow-oneofschema
+, numpy
+, openapi-spec-validator
 , pandas
 , pendulum
 , psutil
 , pygments
+, pyjwt
 , python-daemon
 , python-dateutil
-, requests
+, python-nvd3
+, python-slugify
+, python3-openid
+, pyyaml
+, rich
 , setproctitle
-, snakebite
 , sqlalchemy
+, sqlalchemy-jsonfield
+, swagger-ui-bundle
 , tabulate
 , tenacity
 , termcolor
-, text-unidecode
-, thrift
-, tzlocal
 , unicodecsv
-, zope_deprecation
-, nose
-, pythonOlder
-, pythonAtLeast
+, werkzeug
+, pytest
+, freezegun
+, mkYarnPackage
 }:
+let

-buildPythonPackage rec {
-  pname = "apache-airflow";
-  version = "1.10.5";
-  # Upstream does not yet support python 3.8
-  # https://github.com/apache/airflow/issues/8674
-  disabled = pythonOlder "3.5" || pythonAtLeast "3.8";
+  version = "2.1.1rc1";

-  src = fetchFromGitHub rec {
+  airflow-src = fetchFromGitHub rec {
    owner = "apache";
    repo = "airflow";
    rev = version;
-    sha256 = "14fmhfwx977c9jdb2kgm93i6acx43l45ggj30rb37r68pzpb6l6h";
+    sha256 = "1vzzmcfgqni9rkf7ggh8mswnm3ffwaishcz1ysrwx0a96ilhm9q2";
  };

-  patches = [
-       # Not yet accepted: https://github.com/apache/airflow/pull/6562
-     (fetchpatch {
-       name = "avoid-warning-from-abc.collections";
-       url = "https://patch-diff.githubusercontent.com/raw/apache/airflow/pull/6562.patch";
-       sha256 = "0swpay1qlb7f9kgc56631s1qd9k82w4nw2ggvkm7jvxwf056k61z";
-     })
-       # Not yet accepted: https://github.com/apache/airflow/pull/6561
-     (fetchpatch {
-       name = "pendulum2-compatibility";
-       url = "https://patch-diff.githubusercontent.com/raw/apache/airflow/pull/6561.patch";
-       sha256 = "17hw8qyd4zxvib9zwpbn32p99vmrdz294r31gnsbkkcl2y6h9knk";
-     })
-  ];
+  # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
+  # This replicates this shell script, fixing bugs in yarn.lock and package.json
+
+  airflow-frontend = mkYarnPackage {
+    name = "airflow-frontend";
+
+    src = "${airflow-src}/airflow/www";
+    packageJSON = ./package.json;
+    yarnLock = ./yarn.lock;
+    yarnNix = ./yarn.nix;
+
+    distPhase = "true";
+
+    configurePhase = ''
+      cp -r $node_modules node_modules
+    '';
+
+    buildPhase = ''
+      yarn --offline build
+      find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > static/dist/sum.md5
+    '';
+
+    installPhase = ''
+      mkdir -p $out/static/
+      cp -r static/dist $out/static
+    '';
+  };
+
+in
+buildPythonPackage rec {
+  pname = "apache-airflow";
+  inherit version;
+  src = airflow-src;

  propagatedBuildInputs = [
    alembic
+    argcomplete
+    attrs
+    blinker
    cached-property
+    cattrs
+    clickclick
    colorlog
-    configparser
    croniter
+    cryptography
    dill
    flask
-    flask-admin
    flask-appbuilder
-    flask-bcrypt
    flask-caching
    flask_login
-    flask-swagger
    flask_wtf
-    funcsigs
-    future
    GitPython
+    graphviz
    gunicorn
+    httpx
    iso8601
-    json-merge-patch
+    importlib-resources
+    importlib-metadata
+    inflection
+    itsdangerous
    jinja2
-    ldap3
-    lxml
+    jsonschema
    lazy-object-proxy
+    lockfile
    markdown
+    markupsafe
+    marshmallow-oneofschema
+    numpy
+    openapi-spec-validator
    pandas
    pendulum
    psutil
    pygments
+    pyjwt
    python-daemon
    python-dateutil
-    requests
+    python-nvd3
+    python-slugify
+    python3-openid
+    pyyaml
+    rich
    setproctitle
    sqlalchemy
+    sqlalchemy-jsonfield
+    swagger-ui-bundle
    tabulate
    tenacity
    termcolor
-    text-unidecode
-    thrift
-    tzlocal
    unicodecsv
-    zope_deprecation
+    werkzeug
+  ];
+
+  buildInputs = [
+    airflow-frontend
  ];

  checkInputs = [
-    snakebite
-    nose
+    freezegun
+    pytest
  ];

+  INSTALL_PROVIDERS_FROM_SOURCES = "true";
+
  postPatch = ''
-    substituteInPlace setup.py \
+    substituteInPlace setup.cfg \
+      --replace "importlib_resources~=1.4" "importlib_resources" \
+      --replace "importlib_metadata~=1.7" "importlib_metadata" \
+      --replace "tenacity~=6.2.0" "tenacity" \
+      --replace "pyjwt<2" "pyjwt" \
      --replace "flask>=1.1.0, <2.0" "flask" \
-      --replace "jinja2>=2.10.1, <2.11.0" "jinja2" \
-      --replace "pandas>=0.17.1, <1.0.0" "pandas" \
-      --replace "flask-caching>=1.3.3, <1.4.0" "flask-caching" \
-      --replace "flask-appbuilder>=1.12.5, <2.0.0" "flask-appbuilder" \
-      --replace "flask-admin==1.5.3" "flask-admin" \
      --replace "flask-login>=0.3, <0.5" "flask-login" \
-      --replace "cached_property~=1.5" "cached_property" \
-      --replace "dill>=0.2.2, <0.3" "dill" \
-      --replace "configparser>=3.5.0, <3.6.0" "configparser" \
-      --replace "colorlog==4.0.2" "colorlog" \
-      --replace "funcsigs==1.0.0" "funcsigs" \
-      --replace "flask-swagger==0.2.13" "flask-swagger" \
-      --replace "python-daemon>=2.1.1, <2.2" "python-daemon" \
-      --replace "alembic>=1.0, <2.0" "alembic" \
-      --replace "markdown>=2.5.2, <3.0" "markdown" \
-      --replace "future>=0.16.0, <0.17" "future" \
-      --replace "tenacity==4.12.0" "tenacity" \
-      --replace "text-unidecode==1.2" "text-unidecode" \
-      --replace "tzlocal>=1.4,<2.0.0" "tzlocal" \
-      --replace "sqlalchemy~=1.3" "sqlalchemy" \
-      --replace "gunicorn>=19.5.0, <20.0" "gunicorn"
+      --replace "flask-wtf>=0.14.3, <0.15" "flask-wtf" \
+      --replace "jinja2>=2.10.1, <2.12.0" "jinja2" \
+      --replace "attrs>=20.0, <21.0" "attrs" \
+      --replace "cattrs~=1.1, <1.7.0" "cattrs" \
+      --replace "markupsafe>=1.1.1, <2.0" "markupsafe" \
+      --replace "docutils<0.17" "docutils" \
+      --replace "sqlalchemy>=1.3.18, <1.4" "sqlalchemy" \
+      --replace "sqlalchemy_jsonfield~=1.0" "sqlalchemy-jsonfield" \
+      --replace "werkzeug~=1.0, >=1.0.1" "werkzeug" \
+      --replace "itsdangerous>=1.1.0, <2.0" "itsdangerous"

-    # dumb-init is only needed for CI and Docker, not relevant for NixOS.
-    substituteInPlace setup.py \
-      --replace "'dumb-init>=1.2.2'," ""
-
-    substituteInPlace tests/core.py \
+    substituteInPlace tests/core/test_core.py \
      --replace "/bin/bash" "${stdenv.shell}"
  '';

@ -174,11 +211,14 @@ buildPythonPackage rec {
   export PATH=$PATH:$out/bin

   airflow version
-   airflow initdb
-   airflow resetdb -y
-   nosetests tests.core.CoreTest
-   ## all tests
-   # nosetests --cover-package=airflow
+   airflow db init
+   airflow db reset -y
+
+   pytest tests/core/test_core.py
+  '';
+
+  postInstall = ''
+    cp -rv ${airflow-frontend}/static/dist $out/lib/${python.libPrefix}/site-packages/airflow/www/static
  '';

  meta = with lib; {
--- a/pkgs/development/python-modules/apache-airflow/package.json
+++ b/pkgs/development/python-modules/apache-airflow/package.json
@ -0,0 +1,80 @@
+{
+  "name": "airflow-frontend",
+  "version": "2.1.1rc1",
+  "description": "Apache Airflow is a platform to programmatically author, schedule and monitor workflows.",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "dev": "NODE_ENV=dev webpack --watch --colors --progress --debug --output-pathinfo --devtool eval-cheap-source-map --mode development",
+    "prod": "NODE_ENV=production node --max_old_space_size=4096 ./node_modules/webpack/bin/webpack.js -p --colors --progress",
+    "build": "NODE_ENV=production webpack --colors --progress",
+    "lint": "eslint --ignore-path=.eslintignore --ext .js,.html .",
+    "lint:fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.html ."
+  },
+  "author": "Apache",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/apache/airflow.git"
+  },
+  "homepage": "https://airflow.apache.org/",
+  "keywords": [
+    "big",
+    "data",
+    "workflow",
+    "airflow",
+    "d3",
+    "nerds",
+    "database",
+    "flask"
+  ],
+  "devDependencies": {
+    "babel": "^6.23.0",
+    "babel-core": "^6.26.3",
+    "babel-eslint": "^10.1.0",
+    "babel-loader": "^8.1.0",
+    "babel-plugin-css-modules-transform": "^1.6.1",
+    "babel-polyfill": "^6.26.0",
+    "clean-webpack-plugin": "^3.0.0",
+    "copy-webpack-plugin": "^6.0.3",
+    "css-loader": "^3.4.2",
+    "eslint": "^7.5.0",
+    "eslint-config-airbnb-base": "^14.2.0",
+    "eslint-plugin-html": "^6.0.2",
+    "eslint-plugin-import": "^2.22.0",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-promise": "^4.2.1",
+    "eslint-plugin-standard": "^4.0.1",
+    "file-loader": "^6.0.0",
+    "imports-loader": "^1.1.0",
+    "mini-css-extract-plugin": "1.6.0",
+    "moment-locales-webpack-plugin": "^1.2.0",
+    "optimize-css-assets-webpack-plugin": "6.0.0",
+    "style-loader": "^1.2.1",
+    "stylelint": "^13.6.1",
+    "stylelint-config-standard": "^20.0.0",
+    "url-loader": "4.1.0",
+    "webpack": "^4.16.3",
+    "webpack-cli": "^3.1.0",
+    "webpack-manifest-plugin": "^2.2.0"
+  },
+  "dependencies": {
+    "bootstrap-3-typeahead": "^4.0.2",
+    "codemirror": "^5.59.1",
+    "d3": "^3.4.4",
+    "d3-shape": "^2.1.0",
+    "d3-tip": "^0.9.1",
+    "dagre-d3": "^0.6.4",
+    "datatables.net": "^1.10.23",
+    "datatables.net-bs": "^1.10.23",
+    "eonasdan-bootstrap-datetimepicker": "^4.17.47",
+    "jquery": ">=3.4.0",
+    "jshint": "^2.12.0",
+    "moment-timezone": "^0.5.28",
+    "nvd3": "^1.8.6",
+    "redoc": "^2.0.0-rc.48",
+    "url-search-params-polyfill": "^8.1.0"
+  },
+  "resolutions": {
+    "lodash": "^4.17.21"
+  }
+}
--- a/pkgs/development/python-modules/apache-airflow/yarn.lock
+++ b/pkgs/development/python-modules/apache-airflow/yarn.lock
--- a/pkgs/development/python-modules/apache-airflow/yarn.nix
+++ b/pkgs/development/python-modules/apache-airflow/yarn.nix
--- a/pkgs/development/python-modules/cattrs/default.nix
+++ b/pkgs/development/python-modules/cattrs/default.nix
@ -1,6 +1,7 @@
 { lib
 , attrs
 , bson
+, pythonOlder
 , buildPythonPackage
 , fetchFromGitHub
 , hypothesis
@ -18,6 +19,10 @@ buildPythonPackage rec {
  version = "1.7.0";
  format = "pyproject";

+  # https://cattrs.readthedocs.io/en/latest/history.html#id33:
+  # "Python 2, 3.5 and 3.6 support removal. If you need it, use a version below 1.1.0."
+  disabled = pythonOlder "3.7";
+
  src = fetchFromGitHub {
    owner = "Tinche";
    repo = pname;
--- a/pkgs/development/python-modules/flask-appbuilder/default.nix
+++ b/pkgs/development/python-modules/flask-appbuilder/default.nix
@ -5,7 +5,6 @@
 , colorama
 , click
 , email_validator
-, fetchpatch
 , flask
 , flask-babel
 , flask_login
@ -26,23 +25,18 @@

 buildPythonPackage rec {
  pname = "flask-appbuilder";
-  version = "3.3.0";
+  version = "3.3.1";

  src = fetchPypi {
    pname = "Flask-AppBuilder";
    inherit version;
-    sha256 = "00dsfv1apl6483wy20aj91f9h5ak2casbx5vcajv2nd3i7c7v8gx";
+    sha256 = "13rlpdf3ipm39zpc62sywn8qjn6gwfbgr43x7lqpxr28br2jcg3j";
  };

-  patches = [
-    # https://github.com/dpgaspar/Flask-AppBuilder/pull/1610
-    (fetchpatch {
-      name = "flask_jwt_extended-and-pyjwt-patch";
-      url = "https://github.com/dpgaspar/Flask-AppBuilder/commit/7097a7b133f27c78d2b54d2a46e4a4c24478a066.patch";
-      sha256 = "sha256-ZpY8+2Hoz3z01GVtw2OIbQcsmAwa7iwilFWzgcGhY1w=";
-      includes = [ "flask_appbuilder/security/manager.py" "setup.py" ];
-    })
-  ];
+  # See here: https://github.com/dpgaspar/Flask-AppBuilder/commit/7097a7b133f27c78d2b54d2a46e4a4c24478a066.patch
+  #           https://github.com/dpgaspar/Flask-AppBuilder/pull/1610
+  # The patch from the PR doesn't apply cleanly so I edited it manually.
+  patches = [ ./upgrade-to-flask_jwt_extended-4.patch ];

  propagatedBuildInputs = [
    apispec
@ -70,9 +64,15 @@ buildPythonPackage rec {
  postPatch = ''
    substituteInPlace setup.py \
      --replace "apispec[yaml]>=3.3, <4" "apispec[yaml] >=3.3, <5" \
+      --replace "click>=6.7, <8" "click" \
+      --replace "Flask>=0.12, <2" "Flask" \
      --replace "Flask-Login>=0.3, <0.5" "Flask-Login >=0.3, <0.6" \
      --replace "Flask-Babel>=1, <2" "Flask-Babel >=1, <3" \
-      --replace "marshmallow-sqlalchemy>=0.22.0, <0.24.0" "marshmallow-sqlalchemy >=0.22.0, <0.25.0"
+      --replace "Flask-WTF>=0.14.2, <0.15.0" "Flask-WTF" \
+      --replace "marshmallow-sqlalchemy>=0.22.0, <0.24.0" "marshmallow-sqlalchemy" \
+      --replace "Flask-JWT-Extended>=3.18, <4" "Flask-JWT-Extended>=4.1.0" \
+      --replace "PyJWT>=1.7.1, <2.0.0" "PyJWT>=2.0.1" \
+      --replace "SQLAlchemy<1.4.0" "SQLAlchemy"
  '';

  # Majority of tests require network access or mongo
--- a/pkgs/development/python-modules/flask-appbuilder/upgrade-to-flask_jwt_extended-4.patch
+++ b/pkgs/development/python-modules/flask-appbuilder/upgrade-to-flask_jwt_extended-4.patch
@ -0,0 +1,45 @@
+diff --git a/flask_appbuilder/security/api.py b/flask_appbuilder/security/api.py
+index 2e2dfd612..df1bd5a25 100644
+--- a/flask_appbuilder/security/api.py
+++ b/flask_appbuilder/security/api.py
+@@ -3,7 +3,7 @@
+     create_access_token,
+     create_refresh_token,
+     get_jwt_identity,
+-    jwt_refresh_token_required,
+    jwt_required,
+ )
+ 
+ from ..api import BaseApi, safe
+@@ -118,7 +118,7 @@ def login(self):
+         return self.response(200, **resp)
+ 
+     @expose("/refresh", methods=["POST"])
+-    @jwt_refresh_token_required
+    @jwt_required(refresh=True)
+     @safe
+     def refresh(self):
+         """
+diff --git a/flask_appbuilder/security/manager.py b/flask_appbuilder/security/manager.py
+index fe7697007..3b22ab255 100644
+--- a/flask_appbuilder/security/manager.py
+++ b/flask_appbuilder/security/manager.py
+@@ -297,7 +297,7 @@ def create_jwt_manager(self, app) -> JWTManager:
+         """
+         jwt_manager = JWTManager()
+         jwt_manager.init_app(app)
+-        jwt_manager.user_loader_callback_loader(self.load_user_jwt)
+        jwt_manager.user_lookup_loader(self.load_user_jwt)
+         return jwt_manager
+ 
+     def create_builtin_roles(self):
+@@ -1944,7 +1944,8 @@ def del_permission_role(self, role, perm_view):
+     def load_user(self, pk):
+         return self.get_user_by_id(int(pk))
+ 
+-    def load_user_jwt(self, pk):
+    def load_user_jwt(self, _jwt_header, jwt_data):
+        pk = jwt_data["sub"]
+         user = self.load_user(pk)
+         # Set flask g.user to JWT user, we can't do it on before request
+         g.user = user
--- a/pkgs/development/python-modules/marshmallow-oneofschema/default.nix
+++ b/pkgs/development/python-modules/marshmallow-oneofschema/default.nix
@ -0,0 +1,22 @@
+{ lib, buildPythonPackage, fetchPypi, marshmallow, setuptools }:
+
+buildPythonPackage rec {
+  pname = "marshmallow-oneofschema";
+  version = "2.1.0";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "0s0yr7nv06sfgxglghl2vq74g3m49j60k1hi2qzfsv4bj8hvs35k";
+  };
+
+  propagatedBuildInputs = [ marshmallow setuptools ];
+
+  pythonImportsCheck = [ "marshmallow_oneofschema" ];
+
+  meta = with lib; {
+    homepage = "https://github.com/marshmallow-code/marshmallow-oneofschema";
+    description = "Marshmallow library extension that allows schema (de)multiplexing";
+    license = licenses.mit;
+    maintainers = [ maintainers.ivan-tkatchev ];
+  };
+}
--- a/pkgs/development/python-modules/python-nvd3/default.nix
+++ b/pkgs/development/python-modules/python-nvd3/default.nix
@ -0,0 +1,27 @@
+{ lib, buildPythonPackage, fetchFromGitHub, python-slugify, jinja2, setuptools, coverage }:
+
+buildPythonPackage rec {
+  pname = "python-nvd3";
+  version = "0.15.0";
+
+  src = fetchFromGitHub {
+    owner = "areski";
+    repo = "python-nvd3";
+    rev = "dc8e772597ed72f413b229856fc9a3318e57fcfc";
+    sha256 = "1vjnicszcc9j0rgb58104fk9sry5xad1xli64jana9bkx42c6x1v";
+  };
+
+  propagatedBuildInputs = [ python-slugify jinja2 setuptools ];
+  checkInputs = [ coverage ];
+
+  checkPhase = ''
+    coverage run --source=nvd3 setup.py test
+  '';
+
+  meta = with lib; {
+    homepage = "https://github.com/areski/python-nvd3";
+    description = "Python Wrapper for NVD3 - It's time for beautiful charts";
+    license = licenses.mit;
+    maintainers = [ maintainers.ivan-tkatchev ];
+  };
+}
--- a/pkgs/development/python-modules/sqlalchemy-jsonfield/default.nix
+++ b/pkgs/development/python-modules/sqlalchemy-jsonfield/default.nix
@ -0,0 +1,44 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, sqlalchemy
+, setuptools-scm
+, setuptools
+, tox
+, sphinx
+, pytest
+, pytest-cov
+, pytest-html
+, pytest-sugar
+, coverage
+, pymysql
+, psycopg2 }:
+
+buildPythonPackage rec {
+  pname = "sqlalchemy-jsonfield";
+  version = "1.0.0";
+
+  src = fetchFromGitHub {
+    owner = "penguinolog";
+    repo = "sqlalchemy_jsonfield";
+    rev = version;
+    sha256 = "015pl4z84spfw8389hk1szlm37jgw2basvbmzmkacdqi0685zx24";
+  };
+
+  SETUPTOOLS_SCM_PRETEND_VERSION = "v${version}";
+
+  nativeBuildInputs = [ setuptools-scm ];
+  propagatedBuildInputs = [ sqlalchemy setuptools ];
+  checkInputs = [ tox sphinx pytest pytest-cov pytest-html pytest-sugar coverage pymysql psycopg2 ];
+
+  checkPhase = ''
+    TOX_TESTENV_PASSENV="PYTHONPATH SETUPTOOLS_SCM_PRETEND_VERSION" tox -e functional
+  '';
+
+  meta = with lib; {
+    homepage = "https://github.com/penguinolog/sqlalchemy_jsonfield";
+    description = "SQLALchemy JSONField implementation for storing dicts at SQL independently from JSON type support";
+    license = licenses.asl20;
+    maintainers = [ maintainers.ivan-tkatchev ];
+  };
+}
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@ -4321,6 +4321,8 @@ in {

  marshmallow-enum = callPackage ../development/python-modules/marshmallow-enum { };

+  marshmallow-oneofschema = callPackage ../development/python-modules/marshmallow-oneofschema { };
+
  marshmallow-polyfield = callPackage ../development/python-modules/marshmallow-polyfield { };

  marshmallow-sqlalchemy = callPackage ../development/python-modules/marshmallow-sqlalchemy { };
@ -4888,6 +4890,8 @@ in {

  nvchecker = callPackage ../development/python-modules/nvchecker { };

+  python-nvd3 = callPackage ../development/python-modules/python-nvd3 { };
+
  nwdiag = callPackage ../development/python-modules/nwdiag { };

  oath = callPackage ../development/python-modules/oath { };
@ -8238,6 +8242,8 @@ in {

  SQLAlchemy-ImageAttach = callPackage ../development/python-modules/sqlalchemy-imageattach { };

+  sqlalchemy-jsonfield = callPackage ../development/python-modules/sqlalchemy-jsonfield { };
+
  sqlalchemy_migrate = callPackage ../development/python-modules/sqlalchemy-migrate { };

  sqlalchemy-utils = callPackage ../development/python-modules/sqlalchemy-utils { };