Merge pull request #198606 from figsoda/nvim-treesitter

vimPlugins.nvim-treesitter: add tree sitter grammars

doc/languages-frameworks/vim.section.md

@@ -170,8 +170,8 @@ of precompiled grammars, you can use `nvim-treesitter.withPlugins` function:
       start = [
         (nvim-treesitter.withPlugins (
           plugins: with plugins; [
-            tree-sitter-nix
-            tree-sitter-python
+            nix
+            python
           ]
         ))
       ];
@@ -180,7 +180,7 @@ of precompiled grammars, you can use `nvim-treesitter.withPlugins` function:
 })
 ```
 
-To enable all grammars packaged in nixpkgs, use `(pkgs.vimPlugins.nvim-treesitter.withPlugins (plugins: pkgs.tree-sitter.allGrammars))`.
+To enable all grammars packaged in nixpkgs, use `pkgs.vimPlugins.nvim-treesitter.withAllGrammars`.
 
 ## Managing plugins with vim-plug {#managing-plugins-with-vim-plug}
 
@@ -203,6 +203,8 @@ Note: this is not possible anymore for Neovim.
 
 Nix expressions for Vim plugins are stored in [pkgs/applications/editors/vim/plugins](https://github.com/NixOS/nixpkgs/tree/master/pkgs/applications/editors/vim/plugins). For the vast majority of plugins, Nix expressions are automatically generated by running [`./update.py`](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/editors/vim/plugins/update.py). This creates a [generated.nix](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/editors/vim/plugins/generated.nix) file based on the plugins listed in [vim-plugin-names](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/editors/vim/plugins/vim-plugin-names). Plugins are listed in alphabetical order in `vim-plugin-names` using the format `[github username]/[repository]@[gitref]`. For example https://github.com/scrooloose/nerdtree becomes `scrooloose/nerdtree`.
 
+After running `./update.py`, if nvim-treesitter received an update, also run [`nvim-treesitter/update.py`](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/editors/vim/plugins/update.py) to update the tree sitter grammars for `nvim-treesitter`.
+
 Some plugins require overrides in order to function properly. Overrides are placed in [overrides.nix](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/editors/vim/plugins/overrides.nix). Overrides are most often required when a plugin requires some dependencies, or extra steps are required during the build process. For example `deoplete-fish` requires both `deoplete-nvim` and `vim-fish`, and so the following override was added:
 
 ```nix

pkgs/applications/editors/vim/plugins/nvim-treesitter/overrides.nix

@@ -0,0 +1,38 @@
+{ lib, callPackage, tree-sitter, nodejs }:
+
+self: super:
+
+let
+  builtGrammars = callPackage ./generated.nix {
+    buildGrammar = callPackage ../../../../../development/tools/parsing/tree-sitter/grammar.nix { };
+  };
+
+  allGrammars = lib.filter lib.isDerivation (lib.attrValues builtGrammars);
+
+  # Usage:
+  # pkgs.vimPlugins.nvim-treesitter.withPlugins (p: [ p.c p.java ... ])
+  # or for all grammars:
+  # pkgs.vimPlugins.nvim-treesitter.withAllGrammars
+  withPlugins =
+    grammarFn: self.nvim-treesitter.overrideAttrs (_: {
+      postPatch =
+        let
+          grammars = tree-sitter.withPlugins (ps: grammarFn (ps // builtGrammars));
+        in
+        ''
+          rm -r parser
+          ln -s ${grammars} parser
+        '';
+    });
+in
+
+{
+  passthru = {
+    inherit builtGrammars allGrammars withPlugins;
+
+    tests.builtGrammars = lib.recurseIntoAttrs builtGrammars;
+
+    withAllGrammars = withPlugins (_: allGrammars);
+  };
+}
+

pkgs/applications/editors/vim/plugins/nvim-treesitter/update-shell.nix

@@ -0,0 +1,17 @@
+{ pkgs ? import ../../../../../.. { } }:
+
+with pkgs;
+
+let
+  inherit (vimPlugins) nvim-treesitter;
+
+  neovim = pkgs.neovim.override {
+    configure.packages.all.start = [ nvim-treesitter ];
+  };
+in
+
+mkShell {
+  packages = [ neovim nix-prefetch python3 ];
+
+  NVIM_TREESITTER = nvim-treesitter;
+}

pkgs/applications/editors/vim/plugins/nvim-treesitter/update.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env nix-shell
+#!nix-shell update-shell.nix -i python
+
+import json
+import re
+import subprocess
+from os import getenv
+from os.path import dirname, join
+
+lockfile = json.load(open(join(getenv("NVIM_TREESITTER"), "lockfile.json")))
+
+configs = json.loads(
+    subprocess.check_output(
+        [
+            "nvim",
+            "--headless",
+            "-u",
+            "NONE",
+            "+lua io.write(vim.json.encode(require('nvim-treesitter.parsers').get_parser_configs()))",
+            "+quit!",
+        ]
+    )
+)
+
+regex = re.compile("^https?://(github.com|gitlab.com)/(.+?)/(.+?)(.git)?$")
+
+
+def generate_grammar(item):
+    lang, lock = item
+    cfg = configs.get(lang)
+    if not cfg:
+        return ""
+
+    info = cfg["install_info"]
+    url = info["url"]
+    rev = lock["revision"]
+
+    generated = f"""  {lang} = buildGrammar {{
+    language = "{lang}";
+    version = "{rev[:7]}";
+    source = """
+
+    m = regex.fullmatch(url)
+    cmd = ["nix-prefetch", "--rev", rev]
+
+    match m and m.group(1, 2, 3):
+        case "github.com", owner, repo:
+            cmd += [
+                "fetchFromGitHub",
+                "--owner",
+                owner,
+                "--repo",
+                repo,
+            ]
+
+            generated += f"""fetchFromGitHub {{
+      owner = "{owner}";
+      repo = "{repo}";"""
+
+        case "gitlab.com", owner, repo:
+            cmd += [
+                "fetchFromGitLab",
+                "--owner",
+                owner,
+                "--repo",
+                repo,
+            ]
+
+            generated += f"""fetchFromGitLab {{
+      owner = "{owner}";
+      repo = "{repo}";"""
+
+        case _:
+            cmd += ["fetchgit", "url"]
+            generated += f"""fetchgit {{
+      url = "{url}";"""
+
+    if info.get("requires_generate_from_grammar"):
+        cmd += [
+            "--arg",
+            "nativeBuildInputs",
+            "[ nodejs tree-sitter ]",
+            "--postFetch",
+            "pushd $out && tree-sitter generate && popd",
+        ]
+
+        generated += """
+      nativeBuildInputs = [ nodejs tree-sitter ];
+      postFetch = "pushd $out && tree-sitter generate && popd";"""
+
+    hash = subprocess.check_output(cmd, text=True).strip()
+
+    generated += f"""
+      rev = "{rev}";
+      hash = "{hash}";
+    }};"""
+
+    location = info.get("location")
+    if location:
+        generated += f"""
+    location = "{location}";
+"""
+
+    generated += """
+  };
+"""
+
+    return generated
+
+
+generated_file = """# generated by pkgs/applications/editors/vim/plugins/nvim-treesitter/update.py
+
+{ buildGrammar, fetchFromGitHub, fetchFromGitLab, fetchgit, nodejs, tree-sitter }:
+
+{
+"""
+
+for generated in map(generate_grammar, lockfile.items()):
+    generated_file += generated
+
+generated_file += "}\n"
+
+open(join(dirname(__file__), "generated.nix"), "w").write(generated_file)

pkgs/applications/editors/vim/plugins/overrides.nix

@@ -67,7 +67,7 @@
 , CoreServices
 
   # nvim-treesitter dependencies
-, tree-sitter
+, callPackage
 
   # sved dependencies
 , glib
@@ -652,23 +652,9 @@ self: super: {
     dependencies = with self; [ plenary-nvim ];
   });
 
-  # Usage:
-  # pkgs.vimPlugins.nvim-treesitter.withPlugins (p: [ p.tree-sitter-c p.tree-sitter-java ... ])
-  # or for all grammars:
-  # pkgs.vimPlugins.nvim-treesitter.withPlugins (_: tree-sitter.allGrammars)
-  nvim-treesitter = super.nvim-treesitter.overrideAttrs (old: {
-    passthru.withPlugins =
-      grammarFn: self.nvim-treesitter.overrideAttrs (_: {
-        postPatch =
-          let
-            grammars = tree-sitter.withPlugins grammarFn;
-          in
-          ''
-            rm -r parser
-            ln -s ${grammars} parser
-          '';
-      });
-  });
+  nvim-treesitter = super.nvim-treesitter.overrideAttrs (old:
+    callPackage ./nvim-treesitter/overrides.nix { } self super
+  );
 
   octo-nvim = super.octo-nvim.overrideAttrs (old: {
     dependencies = with self; [ telescope-nvim plenary-nvim ];