#!/usr/bin/env ruby
require 'open-uri'
require 'open3'
require 'yaml'
require 'fileutils'
Dir.chdir(__dir__)
BASE_URL = 'https://raw.githubusercontent.com/github/linguist/master'
DIRS = {
default: %w(syntax indent compiler autoload ftplugin ctags after/syntax after/indent after/ftplugin),
all: %w(syntax indent compiler autoload ftplugin after extras ctags),
basic: %w(syntax compiler indent after/syntax after/indent),
syntax: %w(syntax indent after/syntax after/indent),
noafter: %w(syntax compiler indent autoload ftplugin),
javascript: %w(syntax indent compiler autoload ftplugin ctags after/syntax after/indent after/ftplugin extras),
go: %w(autoload/go/config.vim syntax compiler indent after/syntax after/indent)
}
PACKAGES = [
{
name: "acpiasl",
remote: "martinlroth/vim-acpi-asl",
},
{
name: "ansible",
remote: "pearofducks/ansible-vim",
},
{
name: "apiblueprint",
remote: "sheerun/apiblueprint.vim",
},
{
name: "applescript",
remote: "mityu/vim-applescript",
dirs: DIRS[:syntax]
},
{
name: "arduino",
remote: "sudar/vim-arduino-syntax",
},
{
name: "asciidoc",
remote: "asciidoc/vim-asciidoc",
},
{
name: "autohotkey",
remote: "hnamikaw/vim-autohotkey",
},
{
name: "blade",
remote: "jwalton512/vim-blade",
},
{
name: "brewfile",
remote: "bfontaine/Brewfile.vim",
},
{
name: "c++11",
remote: "octol/vim-cpp-enhanced-highlight",
},
{
name: "c/c++",
remote: "vim-jp/vim-cpp",
},
{
name: "caddyfile",
remote: "isobit/vim-caddyfile",
},
{
name: "carp",
remote: "hellerve/carp-vim",
},
{
name: "cjsx",
remote: "mtscout6/vim-cjsx",
},
{
name: "clojure",
remote: "guns/vim-clojure-static",
},
{
name: "cmake",
remote: "pboettch/vim-cmake-syntax",
},
{
name: "coffee-script",
remote: "kchmck/vim-coffee-script",
dirs: DIRS[:noafter]
},
{
name: "cql",
remote: "elubow/cql-vim",
},
{
name: "cryptol",
remote: "victoredwardocallaghan/cryptol.vim",
},
{
name: "crystal",
remote: "rhysd/vim-crystal",
},
{
name: "csv",
remote: "chrisbra/csv.vim",
},
{
name: "cucumber",
remote: "tpope/vim-cucumber",
},
{
name: "cue",
remote: "mgrabovsky/vim-cuesheet",
},
{
name: "dart",
remote: "dart-lang/dart-vim-plugin",
},
{
name: "dhall",
remote: "vmchale/dhall-vim",
},
{
name: "dlang",
remote: "JesseKPhillips/d.vim",
},
{
name: "dockerfile",
remote: "ekalinin/Dockerfile.vim",
},
{
name: "elixir",
remote: "elixir-lang/vim-elixir",
},
{
name: "elm",
remote: "andys8/vim-elm-syntax",
},
{
name: "emberscript",
remote: "yalesov/vim-ember-script",
},
{
name: "emblem",
remote: "yalesov/vim-emblem",
},
{
name: "erlang",
remote: "vim-erlang/vim-erlang-runtime",
},
{
name: "fennel",
remote: "bakpakin/fennel.vim",
},
{
name: "ferm",
remote: "vim-scripts/ferm.vim",
},
{
name: "fish",
remote: "georgewitteman/vim-fish",
},
{
name: "flatbuffers",
remote: "dcharbon/vim-flatbuffers",
},
{
name: "fsharp",
remote: "ionide/Ionide-vim",
dirs: DIRS[:basic]
},
{
name: "gdscript",
remote: "calviken/vim-gdscript3",
dirs: DIRS[:syntax]
},
{
name: "git",
remote: "tpope/vim-git",
},
{
name: "glsl",
remote: "tikhomirov/vim-glsl",
dirs: DIRS[:noafter]
},
{
name: "gmpl",
remote: "maelvalais/gmpl.vim",
},
{
name: "gnuplot",
remote: "vim-scripts/gnuplot-syntax-highlighting",
},
{
name: "go",
remote: "fatih/vim-go",
dirs: DIRS[:go]
},
{
name: "graphql",
remote: "jparise/vim-graphql",
dirs: DIRS[:all]
},
{
name: "gradle",
remote: "tfnico/vim-gradle",
},
{
name: "haml",
remote: "sheerun/vim-haml",
},
{
name: "handlebars",
remote: "sheerun/vim-mustache-handlebars",
},
{
name: "haproxy",
remote: "CH-DanReif/haproxy.vim",
},
{
name: "haskell",
remote: "neovimhaskell/haskell-vim",
},
{
name: "haxe",
remote: "yaymukund/vim-haxe",
},
{
name: "hcl",
remote: "b4b4r07/vim-hcl",
},
{
name: "helm",
remote: "towolf/vim-helm",
},
{
name: "hive",
remote: "zebradil/hive.vim",
},
{
name: "html5",
remote: "othree/html5.vim",
},
{
name: "i3",
remote: "mboughaba/i3config.vim",
},
{
name: "icalenadr",
remote: "chutzpah/icalendar.vim",
},
{
name: "idris",
remote: "idris-hackers/idris-vim",
},
{
name: "ion",
remote: "vmchale/ion-vim",
},
{
name: "javascript",
remote: "pangloss/vim-javascript",
dirs: DIRS[:javascript]
},
{
name: "jenkins",
remote: "martinda/Jenkinsfile-vim-syntax",
},
{
name: "jinja",
remote: "lepture/vim-jinja",
},
{
name: "jq",
remote: "vito-c/jq.vim",
},
{
name: "json5",
remote: "GutenYe/json5.vim",
},
{
name: "json",
remote: "elzr/vim-json",
},
{
name: "jsonnet",
remote: "google/vim-jsonnet",
},
{
name: "jst",
remote: "briancollins/vim-jst",
},
{
name: "jsx",
remote: "MaxMEllon/vim-jsx-pretty",
dirs: DIRS[:all]
},
{
name: "julia",
remote: "JuliaEditorSupport/julia-vim",
},
{
name: "kotlin",
remote: "udalov/kotlin-vim",
},
{
name: "ledger",
remote: "ledger/vim-ledger",
dirs: DIRS[:basic]
},
{
name: "less",
remote: "groenewege/vim-less",
dirs: DIRS[:noafter]
},
{
name: "lilypond",
remote: "anowlcalledjosh/vim-lilypond",
},
{
name: "livescript",
remote: "gkz/vim-ls",
},
{
name: "llvm",
remote: "rhysd/vim-llvm",
},
{
name: "log",
remote: "MTDL9/vim-log-highlighting",
},
{
name: "lua",
remote: "tbastos/vim-lua",
},
{
name: "mako",
remote: "sophacles/vim-bundle-mako",
},
{
name: "markdown",
remote: "plasticboy/vim-markdown",
dirs: DIRS[:noafter]
},
{
name: "mathematica",
remote: "voldikss/vim-mma",
},
{
name: "mdx",
remote: "jxnblk/vim-mdx-js",
},
{
name: "meson",
remote: "mesonbuild/meson:data/syntax-highlighting/vim",
dirs: DIRS[:all]
},
{
name: "moonscript",
remote: "leafo/moonscript-vim",
},
{
name: "nginx",
remote: "chr4/nginx.vim",
},
{
name: "nim",
remote: "zah/nim.vim",
dirs: DIRS[:basic]
},
{
name: "nix",
remote: "LnL7/vim-nix",
},
{
name: "objc",
remote: "b4winckler/vim-objc",
},
{
name: "ocaml",
remote: "rgrinberg/vim-ocaml",
},
{
name: "octave",
remote: "McSinyx/vim-octave",
},
{
name: "opencl",
remote: "petRUShka/vim-opencl",
},
{
name: "perl",
remote: "vim-perl/vim-perl",
},
{
name: "pgsql",
remote: "lifepillar/pgsql.vim",
},
{
name: "php",
remote: "StanAngeloff/php.vim",
},
{
name: "plantuml",
remote: "aklt/plantuml-syntax",
},
{
name: "pony",
remote: "jakwings/vim-pony",
},
{
name: "powershell",
remote: "PProvost/vim-ps1",
},
{
name: "protobuf",
remote: "uarun/vim-protobuf",
},
{
name: "pug",
remote: "digitaltoad/vim-pug",
},
{
name: "puppet",
remote: "rodjek/vim-puppet",
},
{
name: "purescript",
remote: "purescript-contrib/purescript-vim",
},
{
name: "python-compiler",
remote: "aliev/vim-compiler-python",
},
{
name: "python-indent",
remote: "Vimjas/vim-python-pep8-indent",
},
{
name: "python",
remote: "vim-python/python-syntax",
},
{
name: "qmake",
remote: "artoj/qmake-syntax-vim",
},
{
name: "qml",
remote: "peterhoeg/vim-qml",
},
{
name: "r-lang",
remote: "vim-scripts/R.vim",
dirs: DIRS[:basic]
},
{
name: "racket",
remote: "wlangstroth/vim-racket",
},
{
name: "ragel",
remote: "jneen/ragel.vim",
},
{
name: "raku",
remote: "Raku/vim-raku",
},
{
name: "raml",
remote: "IN3D/vim-raml",
},
{
name: "razor",
remote: "adamclerk/vim-razor",
},
{
name: "reason",
remote: "reasonml-editor/vim-reason-plus",
},
{
name: "requirements",
remote: "raimon49/requirements.txt.vim",
},
{
name: "rspec",
remote: "keith/rspec.vim",
},
{
name: "rst",
remote: "marshallward/vim-restructuredtext",
},
{
name: "ruby",
remote: "vim-ruby/vim-ruby",
},
{
name: "rust",
remote: "rust-lang/rust.vim",
},
{
name: "sbt",
remote: "derekwyatt/vim-sbt",
},
{
name: "scala",
remote: "derekwyatt/vim-scala",
},
{
name: "scss",
remote: "cakebaker/scss-syntax.vim",
},
{
name: "sh",
remote: "arzg/vim-sh",
},
{
name: "slim",
remote: "slim-template/vim-slim",
},
{
name: "slime",
remote: "slime-lang/vim-slime-syntax",
},
{
name: "smt2",
remote: "bohlender/vim-smt2",
},
{
name: "solidity",
remote: "tomlion/vim-solidity",
},
{
name: "sql",
remote: "shmup/vim-sql-syntax",
},
{
name: "stylus",
remote: "wavded/vim-stylus",
},
{
name: "svelte",
remote: "evanleck/vim-svelte@main",
},
{
name: "svg-indent",
remote: "jasonshell/vim-svg-indent",
},
{
name: "svg",
remote: "vim-scripts/svg.vim",
},
{
name: "swift",
remote: "keith/swift.vim",
},
{
name: "sxhkd",
remote: "baskerville/vim-sxhkdrc",
},
{
name: "systemd",
remote: "wgwoods/vim-systemd-syntax",
},
{
name: "terraform",
remote: "hashivim/vim-terraform",
},
{
name: "textile",
remote: "timcharper/textile.vim",
},
{
name: "thrift",
remote: "solarnz/thrift.vim",
},
{
name: "tmux",
remote: "ericpruitt/tmux.vim:vim",
dirs: DIRS[:all]
},
{
name: "tomdoc",
remote: "wellbredgrapefruit/tomdoc.vim",
},
{
name: "toml",
remote: "cespare/vim-toml",
},
{
name: "tptp",
remote: "c-cube/vim-tptp",
},
{
name: "twig",
remote: "lumiliet/vim-twig",
},
{
name: "typescript",
remote: "HerringtonDarkholme/yats.vim",
},
{
name: "unison",
remote: "unisonweb/unison@trunk:editor-support/vim",
dirs: DIRS[:all]
},
{
name: "v",
remote: "ollykel/v-vim",
},
{
name: "vala",
remote: "arrufat/vala.vim",
},
{
name: "vbnet",
remote: "vim-scripts/vbnet.vim",
},
{
name: "vcl",
remote: "smerrill/vcl-vim-plugin",
},
{
name: "vifm",
remote: "vifm/vifm.vim",
},
{
name: "vm",
remote: "lepture/vim-velocity",
},
{
name: "vue",
remote: "posva/vim-vue",
},
{
name: "xdc",
remote: "amal-khailtash/vim-xdc-syntax",
},
{
name: "xml",
remote: "amadeus/vim-xml",
},
{
name: "xls",
remote: "vim-scripts/XSLT-syntax",
},
{
name: "yaml",
remote: "stephpy/vim-yaml",
},
{
name: "yard",
remote: "sheerun/vim-yardoc",
},
{
name: "zephir",
remote: "xwsoul/vim-zephir",
},
{
name: "zig",
remote: "ziglang/zig.vim",
},
{
name: "zinit",
remote: "zinit-zsh/zplugin-vim-syntax",
}
]
def parallel(*procs)
threads = procs.map { |p| Thread.new { method(p).call } }
threads.map(&:join).map(&:value)
end
def read_strings(data, keys, print=false)
if data.is_a?(Hash)
data.flat_map do |key, val|
read_strings(val, keys, keys.include?(key))
end
elsif data.is_a?(Array)
data.flat_map { |d| read_strings(d, keys, print) }
elsif data.is_a?(String)
print ? [data] : []
else
[]
end
end
def transform_with(data, keys, transfrom=false, &block)
if data.is_a?(Hash)
Hash[data.map do |key, val|
[key, transform_with(val, keys, keys.include?(key), &block)]
end]
elsif data.is_a?(Array)
data.map { |d| transform_with(d, keys, transfrom, &block) }
elsif data.is_a?(String)
transfrom ? yield(data) : data
else
data
end
end
def each_hash(data, &block)
if data.is_a?(Hash)
yield data
data.each do |key, val|
each_hash(val, &block)
end
elsif data.is_a?(Array)
data.map { |d| each_hash(d, &block) }
end
end
def patterns_to_vim_patterns(patterns)
stdin, stdout, stderr = Open3.popen3('vim', '-V', '--clean', '/dev/stdin', '-es', '-c', "echo expand('%:p:h') | source #{__dir__}/eregex.vim", '-c', "for line in range(0, line('$')) | call setline(line, ExtendedRegex2VimRegex(getline(line))) | endfor", '-c', ':wq! /dev/stdout', chdir: __dir__)
stdin.write(patterns.join("\n"))
stdin.close
stdout.readlines
end
def transform_patterns(data)
patterns = read_strings(data, ["pattern", "patterns"])
patterns_mapping = Hash[patterns.zip(patterns_to_vim_patterns(patterns))]
transform_with(data, ["pattern", "patterns"]) { |a| patterns_mapping[a] }
end
def load_heuristics
url = "#{BASE_URL}/lib/linguist/heuristics.yml"
data = URI.open(url) { |io| YAML.load(io.read) }
each_hash(data["disambiguations"]) do |h|
if h.has_key?("named_pattern")
h["pattern"] = data["named_patterns"].fetch(h["named_pattern"])
h.delete("named_pattern")
end
end
transform_patterns(data["disambiguations"])
end
def load_languages
url = "#{BASE_URL}/lib/linguist/languages.yml"
data = URI.open(url) { |io| YAML.load(io.read) }
end
# heuristics, languages = parallel(:load_heuristics, :load_languages)
def parse_remote(remote)
match = remote.match(/(?<repo>[^@:]+)(?:@(?<branch>[^:]+))?(?::(?<path>.*))?/)
[match[:repo], match[:branch] || "master", match[:path]]
end
def copy_file(package, src, dest)
return unless [".vim", ".ctags", ".vital"].include?(File.extname(src))
FileUtils.mkdir_p(File.dirname(dest))
open(src, "r") do |input|
open(dest, "a+") do |output|
if package[:name] == "jsx"
output << "if !exists('g:polyglot_disabled') || !(index(g:polyglot_disabled, 'typescript') != -1 || index(g:polyglot_disabled, 'typescript') != -1 || index(g:polyglot_disabled, 'jsx') != -1)\n\n"
else
output << "if !exists('g:polyglot_disabled') || index(g:polyglot_disabled, '#{package[:name]}') == -1\n\n"
end
IO.copy_stream(input, output)
output << "\nendif\n"
end
end
end
def download
FileUtils.rm_rf('tmp')
PACKAGES.map do |package|
Thread.new do
repo, branch, path = parse_remote(package[:remote])
dir = "tmp/" + repo.split('/')[1]
FileUtils.mkdir_p(dir)
url = "https://codeload.github.com/#{repo}/tar.gz/#{branch}"
`curl --silent -fL #{url} | tar -zx -C "#{dir}" --strip 1`
progress
end
end.map(&:join)
end
$i = 0
LYRICS = "Never gonna give you up. Never gonna let you down. " +
"Never gonna run around and desert you. " +
"Never gonna make you cry. Never gonna say goodbye. " +
"Never gonna tell a lie and hurt you."
$mutex = Mutex.new
def progress
$mutex.synchronize do
$stdout.write(LYRICS[$i] || ".")
$i += 1
end
end
def extract
FileUtils.rm_rf(DIRS[:all])
output = []
PACKAGES.map do |package|
repo, branch, path = parse_remote(package[:remote])
dir = "tmp/" + repo.split('/')[1]
subdirs = []
for subdir in package.fetch(:dirs, DIRS[:default])
subtree = "#{dir}/#{path ? path + "/" : ""}"
subpath = "#{subtree}#{subdir}"
if FileTest.directory?(subpath)
Dir.glob("#{subdir}/**/*", base: subtree).each do |p|
next unless File.file?("#{subtree}/#{p}")
copy_file(package, "#{subtree}/#{p}", p)
end
subdirs << subdir.split("/").last
elsif File.exist?(subpath)
copy_file(package, subpath, subdir)
end
end
output << "- [#{package[:name]}](https://github.com/#{repo}) (#{subdirs.uniq.join(", ")})"
progress
end
readme = File.read('README.md')
readme.gsub!(
%r{(?<=<!--Package Count-->).*?(?=<!--/Package Count-->)},
output.size.to_s
)
readme.gsub!(
%r{(?<=<!--Language Packs-->).*?(?=<!--/Language Packs-->)}m,
"\n" + output.sort.join("\n") + "\n"
)
File.write('README.md', readme)
end
download
extract
puts(" Bye! Have a wonderful time!")
FileUtils.rm_rf("tmp")
