Extract non-generated code to polyglot.vim

6 files changed, 767 insertions, 142 deletions

diff --git a/Makefile b/Makefile
index fd5b5d84..cc1851b0 100644
--- a/Makefile
+++ b/Makefile
@@ -7,4 +7,4 @@ test:
 	@ scripts/test
 
 dev:
-	@ echo "packages.yaml\nheuristics.yaml\nscripts/test\nscripts/build\ntests/extensions.vim" | DEV=1 entr bash -c 'make && make test'
+	@ echo "polyglot.vim\npackages.yaml\nheuristics.yaml\nscripts/test\nscripts/build\ntests/extensions.vim" | DEV=1 entr bash -c 'make && make test'
diff --git a/autoload/polyglot.vim b/autoload/polyglot.vim
index 97a83752..ee03e989 100644
--- a/autoload/polyglot.vim
+++ b/autoload/polyglot.vim
@@ -1,16 +1,70 @@
+""" autoload/polyglot.vim
+
 " Line continuation is used here, remove 'C' from 'cpoptions'
 let s:cpo_save = &cpo
 set cpo&vim
 
+func! polyglot#ObserveShebang()
+  augroup polyglot-observer
+    au! CursorHold,CursorHoldI,BufWritePost <buffer>
+      \ if polyglot#Shebang() | au! polyglot-observer CursorHold,CursorHoldI,BufWritePost | endif
+  augroup END
+endfunc
+
 func! polyglot#Shebang()
-  " Try to detect filetype from shebang
-  let ft = polyglot#ShebangFiletype()
-  if ft != ""
-    let &ft = ft
-    return 1
+  if getline(1) =~# "^#!"
+    let ft = polyglot#ShebangFiletype()
+    if ft != ""
+      let &ft = ft
+    endif
+  endif
+
+  if &ft == ""
+    runtime! scripts.vim
   endif
 
-  return 0
+  return &ft != ""
+endfunc
+
+let s:r_hashbang = '^#!\s*\(\S\+\)\s*\(.*\)\s*'
+let s:r_envflag = '%(\S\+=\S\+\|-[iS]\|--ignore-environment\|--split-string\)'
+let s:r_env = '^\%(\' . s:r_envflag . '\s\+\)*\(\S\+\)'
+
+func! polyglot#ShebangFiletype()
+  let l:line1 = getline(1)
+
+  if l:line1 !~# "^#!"
+    return
+  endif
+
+  let l:pathrest = matchlist(l:line1, s:r_hashbang)
+
+  if len(l:pathrest) == 0
+    return
+  endif
+
+  let [_, l:path, l:rest; __] = l:pathrest
+
+  let l:script = split(l:path, "/")[-1]
+
+  if l:script == "env"
+    let l:argspath = matchlist(l:rest, s:r_env)
+    if len(l:argspath) == 0
+      return
+    endif
+
+    let l:script = l:argspath[1]
+  endif
+
+  if has_key(s:interpreters, l:script)
+    return s:interpreters[l:script]
+  endif
+
+  for interpreter in keys(s:interpreters)
+    if l:script =~# '^' . interpreter
+      return s:interpreters[interpreter]
+    endif
+  endfor
 endfunc
 
 let s:interpreters = {
@@ -83,48 +137,7 @@ let s:interpreters = {
   \ 'z3': 'smt2',
   \ 'deno': 'typescript',
   \ 'ts-node': 'typescript',
-  \ }
-
-let s:r_hashbang = '^#!\s*\(\S\+\)\s*\(.*\)\s*'
-let s:r_envflag = '%(\S\+=\S\+\|-[iS]\|--ignore-environment\|--split-string\)'
-let s:r_env = '^\%(\' . s:r_envflag . '\s\+\)*\(\S\+\)'
-
-func! polyglot#ShebangFiletype()
-  let l:line1 = getline(1)
-
-  if l:line1 !~# "^#!"
-    return
-  endif
-
-  let l:pathrest = matchlist(l:line1, s:r_hashbang)
-
-  if len(l:pathrest) == 0
-    return 
-  endif
-
-  let [_, l:path, l:rest; __] = l:pathrest
-
-  let l:script = split(l:path, "/")[-1]
-
-  if l:script == "env"
-    let l:argspath = matchlist(l:rest, s:r_env)
-    if len(l:argspath) == 0
-      return
-    endif
-
-    let l:script = l:argspath[1]
-  endif
-
-  if has_key(s:interpreters, l:script)
-    return s:interpreters[l:script]
-  endif
-
-  for interpreter in keys(s:interpreters)
-    if l:script =~# '^' . interpreter
-      return s:interpreters[interpreter]
-    endif
-  endfor
-endfunc
+\ }
 
 func! polyglot#DetectInpFiletype()
   let line = getline(nextnonblank(1))
@@ -382,6 +395,309 @@ func! polyglot#DetectHtmlFiletype()
   set ft=html | return
 endfunc
 
+
 " Restore 'cpoptions'
 let &cpo = s:cpo_save
 unlet s:cpo_save
+
+""" ftdetect/polyglot.vim
+
+" don't spam the user when Vim is started in Vi compatibility mode
+let s:cpo_save = &cpo
+set cpo&vim
+
+" Disable all native vim ftdetect
+if exists('g:polyglot_test')
+  autocmd!
+endif
+
+let s:disabled_packages = {}
+let s:new_polyglot_disabled = []
+
+if exists('g:polyglot_disabled')
+  for pkg in g:polyglot_disabled
+    let base = split(pkg, '\.')
+    if len(base) > 0
+      let s:disabled_packages[pkg] = 1
+      call add(s:new_polyglot_disabled, base[0]) 
+    endif
+  endfor
+else
+  let g:polyglot_disabled_not_set = 1
+endif
+
+function! s:SetDefault(name, value)
+  if !exists(a:name)
+    let {a:name} = a:value
+  endif
+endfunction
+
+call s:SetDefault('g:markdown_enable_spell_checking', 0)
+call s:SetDefault('g:markdown_enable_input_abbreviations', 0)
+call s:SetDefault('g:markdown_enable_mappings', 0)
+
+" Enable jsx syntax by default
+call s:SetDefault('g:jsx_ext_required', 0)
+
+" Needed for sql highlighting
+call s:SetDefault('g:javascript_sql_dialect', 'sql')
+
+" Make csv loading faster
+call s:SetDefault('g:csv_start', 1)
+call s:SetDefault('g:csv_end', 2)
+
+" Disable json concealing by default
+call s:SetDefault('g:vim_json_syntax_conceal', 0)
+
+call s:SetDefault('g:filetype_euphoria', 'elixir')
+
+if !exists('g:python_highlight_all')
+  call s:SetDefault('g:python_highlight_builtins', 1)
+  call s:SetDefault('g:python_highlight_builtin_objs', 1)
+  call s:SetDefault('g:python_highlight_builtin_types', 1)
+  call s:SetDefault('g:python_highlight_builtin_funcs', 1)
+  call s:SetDefault('g:python_highlight_builtin_funcs_kwarg', 1)
+  call s:SetDefault('g:python_highlight_exceptions', 1)
+  call s:SetDefault('g:python_highlight_string_formatting', 1)
+  call s:SetDefault('g:python_highlight_string_format', 1)
+  call s:SetDefault('g:python_highlight_string_templates', 1)
+  call s:SetDefault('g:python_highlight_indent_errors', 1)
+  call s:SetDefault('g:python_highlight_space_errors', 1)
+  call s:SetDefault('g:python_highlight_doctests', 1)
+  call s:SetDefault('g:python_highlight_func_calls', 1)
+  call s:SetDefault('g:python_highlight_class_vars', 1)
+  call s:SetDefault('g:python_highlight_operators', 1)
+  call s:SetDefault('g:python_highlight_file_headers_as_comments', 1)
+  call s:SetDefault('g:python_slow_sync', 1)
+endif
+
+" We need it because scripts.vim in vim uses "set ft=" which cannot be
+" overridden with setf (and we can't use set ft= so our scripts.vim work)
+func! s:Setf(ft)
+  if &filetype !~# '\<'.a:ft.'\>'
+    let &filetype = a:ft
+  endif
+endfunc
+
+" Function used for patterns that end in a star: don't set the filetype if the
+" file name matches ft_ignore_pat.
+" When using this, the entry should probably be further down below with the
+" other StarSetf() calls.
+func! s:StarSetf(ft)
+  if expand("<amatch>") !~ g:ft_ignore_pat && &filetype !~# '\<'.a:ft.'\>'
+    let &filetype = a:ft
+  endif
+endfunc
+
+augroup filetypedetect
+
+" scripts/build inserts here filetype detection autocommands
+
+au! BufNewFile,BufRead,StdinReadPost * if expand("<afile>") !~ g:ft_ignore_pat |
+  \ call polyglot#Shebang() | endif
+
+au BufEnter * if &ft == "" && expand("<afile>") !~ g:ft_ignore_pat |
+      \ call polyglot#ObserveShebang() | endif
+
+augroup END
+
+if !has_key(s:disabled_packages, 'autoindent')
+  " Code below re-implements sleuth for vim-polyglot
+  let g:loaded_sleuth = 1
+  let g:loaded_foobar = 1
+
+  " Makes shiftwidth to be synchronized with tabstop by default
+  if &shiftwidth == &tabstop
+    let &shiftwidth = 0
+  endif
+
+  function! s:guess(lines) abort
+    let options = {}
+    let ccomment = 0
+    let podcomment = 0
+    let triplequote = 0
+    let backtick = 0
+    let xmlcomment = 0
+    let heredoc = ''
+    let minindent = 10
+    let spaces_minus_tabs = 0
+    let i = 0
+
+    for line in a:lines
+      let i += 1
+
+      if !len(line) || line =~# '^\W*$'
+        continue
+      endif
+
+      if line =~# '^\s*/\*'
+        let ccomment = 1
+      endif
+      if ccomment
+        if line =~# '\*/'
+          let ccomment = 0
+        endif
+        continue
+      endif
+
+      if line =~# '^=\w'
+        let podcomment = 1
+      endif
+      if podcomment
+        if line =~# '^=\%(end\|cut\)\>'
+          let podcomment = 0
+        endif
+        continue
+      endif
+
+      if triplequote
+        if line =~# '^[^"]*"""[^"]*$'
+          let triplequote = 0
+        endif
+        continue
+      elseif line =~# '^[^"]*"""[^"]*$'
+        let triplequote = 1
+      endif
+
+      if backtick
+        if line =~# '^[^`]*`[^`]*$'
+          let backtick = 0
+        endif
+        continue
+      elseif &filetype ==# 'go' && line =~# '^[^`]*`[^`]*$'
+        let backtick = 1
+      endif
+
+      if line =~# '^\s*<\!--'
+        let xmlcomment = 1
+      endif
+      if xmlcomment
+        if line =~# '-->'
+          let xmlcomment = 0
+        endif
+        continue
+      endif
+
+      " This is correct order because both "<<EOF" and "EOF" matches end
+      if heredoc != ''
+        if line =~# heredoc
+          let heredoc = ''
+        endif
+        continue
+      endif
+      let herematch = matchlist(line, '\C<<\W*\([A-Z]\+\)\s*$')
+      if len(herematch) > 0
+        let heredoc = herematch[1] . '$'
+      endif
+
+      let spaces_minus_tabs += line[0] == "\t" ? 1 : -1
+
+      if line[0] == "\t"
+        setlocal noexpandtab
+        let &shiftwidth=&tabstop
+        let b:sleuth_culprit .= ':' . i
+        return 1
+      elseif line[0] == " "
+        let indent = len(matchstr(line, '^ *'))
+        if (indent % 2 == 0 || indent % 3 == 0) && indent < minindent
+          let minindent = indent
+        endif
+      endif
+    endfor
+
+    if minindent < 10
+      setlocal expandtab
+      let &shiftwidth=minindent
+      let b:sleuth_culprit .= ':' . i
+      return 1
+    endif
+
+    return 0
+  endfunction
+
+  function! s:detect_indent() abort
+    if &buftype ==# 'help'
+      return
+    endif
+
+    let b:sleuth_culprit = expand("<afile>:p")
+    if s:guess(getline(1, 32))
+      return
+    endif
+    let pattern = sleuth#GlobForFiletype(&filetype)
+    if len(pattern) == 0
+      return
+    endif
+    let pattern = '{' . pattern . ',.git,.svn,.hg}'
+    let dir = expand('%:p:h')
+    let level = 3
+    while isdirectory(dir) && dir !=# fnamemodify(dir, ':h') && level > 0
+      " Ignore files from homedir and root 
+      if dir == expand('~') || dir == '/'
+        unlet b:sleuth_culprit
+        return
+      endif
+      for neighbor in glob(dir . '/' . pattern, 0, 1)[0:level]
+        let b:sleuth_culprit = neighbor
+        " Do not consider directories above .git, .svn or .hg
+        if fnamemodify(neighbor, ":h:t")[0] == "."
+          let level = 0
+          continue
+        endif
+        if neighbor !=# expand('%:p') && filereadable(neighbor)
+          if s:guess(readfile(neighbor, '', 32))
+            return
+          endif
+        endif
+      endfor
+
+      let dir = fnamemodify(dir, ':h')
+      let level -= 1
+    endwhile
+
+    unlet b:sleuth_culprit
+  endfunction
+
+  setglobal smarttab
+
+  function! SleuthIndicator() abort
+    let sw = &shiftwidth ? &shiftwidth : &tabstop
+    if &expandtab
+      return 'sw='.sw
+    elseif &tabstop == sw
+      return 'ts='.&tabstop
+    else
+      return 'sw='.sw.',ts='.&tabstop
+    endif
+  endfunction
+
+  augroup polyglot-sleuth
+    au!
+    au FileType * call s:detect_indent()
+    au User Flags call Hoist('buffer', 5, 'SleuthIndicator')
+  augroup END
+
+  command! -bar -bang Sleuth call s:detect_indent()
+endif
+
+func! s:verify()
+  if exists("g:polyglot_disabled_not_set")
+    if exists("g:polyglot_disabled")
+      echohl WarningMsg
+      echo "vim-polyglot: g:polyglot_disabled should be defined before loading vim-polyglot"
+      echohl None
+    endif
+
+    unlet g:polyglot_disabled_not_set
+  endif
+endfunc
+
+au VimEnter * call s:verify()
+
+" Save polyglot_disabled without postfixes
+if exists('g:polyglot_disabled')
+  let g:polyglot_disabled = s:new_polyglot_disabled
+endif
+
+" restore Vi compatibility settings
+let &cpo = s:cpo_save
diff --git a/ftdetect/polyglot.vim b/ftdetect/polyglot.vim
index d6379ae2..cba8cf86 100644
--- a/ftdetect/polyglot.vim
+++ b/ftdetect/polyglot.vim
@@ -1,3 +1,5 @@
+""" ftdetect/polyglot.vim
+
 " don't spam the user when Vim is started in Vi compatibility mode
 let s:cpo_save = &cpo
 set cpo&vim
@@ -87,9 +89,6 @@ endfunc
 
 augroup filetypedetect
 
-au! filetypedetect BufRead,BufNewFile,StdinReadPost *
-
-" filetypes
 
 if !has_key(s:disabled_packages, '8th')
   au! BufRead,BufNewFile *.8th
@@ -2017,20 +2016,12 @@ if !has_key(s:disabled_packages, 'tads')
 endif
 
 
-" end filetypes
 
-func! s:PolyglotFallback() 
-  if expand("<afile>") !~ g:ft_ignore_pat
-    if getline(1) =~# "^#!"
-      call polyglot#Shebang()
-    endif
-    if &filetype == ''
-      runtime! scripts.vim
-    endif
-  endif
-endfunc
+au! BufNewFile,BufRead,StdinReadPost * if expand("<afile>") !~ g:ft_ignore_pat |
+  \ call polyglot#Shebang() | endif
 
-au BufNewFile,BufRead,StdinReadPost * call s:PolyglotFallback()
+au BufEnter * if &ft == "" && expand("<afile>") !~ g:ft_ignore_pat |
+      \ call polyglot#ObserveShebang() | endif
 
 augroup END
 
@@ -2227,17 +2218,6 @@ endfunc
 
 au VimEnter * call s:verify()
 
-func! s:observe_filetype()
-  augroup polyglot-observer
-    au! CursorHold,CursorHoldI <buffer>
-      \ if polyglot#Shebang() | au! polyglot-observer CursorHold,CursorHoldI | endif
-  augroup END
-endfunc
-
-au BufEnter * if &ft == "" && expand("<afile>") !~ g:ft_ignore_pat
-      \ | call s:observe_filetype() | endif
-
-
 " Save polyglot_disabled without postfixes
 if exists('g:polyglot_disabled')
   let g:polyglot_disabled = s:new_polyglot_disabled
@@ -2245,4 +2225,3 @@ endif
 
 " restore Vi compatibility settings
 let &cpo = s:cpo_save
-unlet s:cpo_save
diff --git a/heuristics.yaml b/heuristics.yaml
index c53a704c..ee1b21af 100644
--- a/heuristics.yaml
+++ b/heuristics.yaml
@@ -69,6 +69,7 @@ rules:
     filetype: glsl
 - override: true
 - filetype: forth
+  fallback: true
 ---
 extensions: [re]
 rules:
diff --git a/polyglot.vim b/polyglot.vim
new file mode 100644
index 00000000..9199a207
--- /dev/null
+++ b/polyglot.vim
@@ -0,0 +1,378 @@
+""" autoload/polyglot.vim
+
+" Line continuation is used here, remove 'C' from 'cpoptions'
+let s:cpo_save = &cpo
+set cpo&vim
+
+func! polyglot#ObserveShebang()
+  augroup polyglot-observer
+    au! CursorHold,CursorHoldI,BufWritePost <buffer>
+      \ if polyglot#Shebang() | au! polyglot-observer CursorHold,CursorHoldI,BufWritePost | endif
+  augroup END
+endfunc
+
+func! polyglot#Shebang()
+  if getline(1) =~# "^#!"
+    let ft = polyglot#ShebangFiletype()
+    if ft != ""
+      let &ft = ft
+    endif
+  endif
+
+  if &ft == ""
+    runtime! scripts.vim
+  endif
+
+  return &ft != ""
+endfunc
+
+let s:r_hashbang = '^#!\s*\(\S\+\)\s*\(.*\)\s*'
+let s:r_envflag = '%(\S\+=\S\+\|-[iS]\|--ignore-environment\|--split-string\)'
+let s:r_env = '^\%(\' . s:r_envflag . '\s\+\)*\(\S\+\)'
+
+func! polyglot#ShebangFiletype()
+  let l:line1 = getline(1)
+
+  if l:line1 !~# "^#!"
+    return
+  endif
+
+  let l:pathrest = matchlist(l:line1, s:r_hashbang)
+
+  if len(l:pathrest) == 0
+    return
+  endif
+
+  let [_, l:path, l:rest; __] = l:pathrest
+
+  let l:script = split(l:path, "/")[-1]
+
+  if l:script == "env"
+    let l:argspath = matchlist(l:rest, s:r_env)
+    if len(l:argspath) == 0
+      return
+    endif
+
+    let l:script = l:argspath[1]
+  endif
+
+  if has_key(s:interpreters, l:script)
+    return s:interpreters[l:script]
+  endif
+
+  for interpreter in keys(s:interpreters)
+    if l:script =~# '^' . interpreter
+      return s:interpreters[interpreter]
+    endif
+  endfor
+endfunc
+
+" scripts/build generates heuristics functions here
+
+" Restore 'cpoptions'
+let &cpo = s:cpo_save
+unlet s:cpo_save
+
+""" ftdetect/polyglot.vim
+
+" don't spam the user when Vim is started in Vi compatibility mode
+let s:cpo_save = &cpo
+set cpo&vim
+
+" Disable all native vim ftdetect
+if exists('g:polyglot_test')
+  autocmd!
+endif
+
+let s:disabled_packages = {}
+let s:new_polyglot_disabled = []
+
+if exists('g:polyglot_disabled')
+  for pkg in g:polyglot_disabled
+    let base = split(pkg, '\.')
+    if len(base) > 0
+      let s:disabled_packages[pkg] = 1
+      call add(s:new_polyglot_disabled, base[0]) 
+    endif
+  endfor
+else
+  let g:polyglot_disabled_not_set = 1
+endif
+
+function! s:SetDefault(name, value)
+  if !exists(a:name)
+    let {a:name} = a:value
+  endif
+endfunction
+
+call s:SetDefault('g:markdown_enable_spell_checking', 0)
+call s:SetDefault('g:markdown_enable_input_abbreviations', 0)
+call s:SetDefault('g:markdown_enable_mappings', 0)
+
+" Enable jsx syntax by default
+call s:SetDefault('g:jsx_ext_required', 0)
+
+" Needed for sql highlighting
+call s:SetDefault('g:javascript_sql_dialect', 'sql')
+
+" Make csv loading faster
+call s:SetDefault('g:csv_start', 1)
+call s:SetDefault('g:csv_end', 2)
+
+" Disable json concealing by default
+call s:SetDefault('g:vim_json_syntax_conceal', 0)
+
+call s:SetDefault('g:filetype_euphoria', 'elixir')
+
+if !exists('g:python_highlight_all')
+  call s:SetDefault('g:python_highlight_builtins', 1)
+  call s:SetDefault('g:python_highlight_builtin_objs', 1)
+  call s:SetDefault('g:python_highlight_builtin_types', 1)
+  call s:SetDefault('g:python_highlight_builtin_funcs', 1)
+  call s:SetDefault('g:python_highlight_builtin_funcs_kwarg', 1)
+  call s:SetDefault('g:python_highlight_exceptions', 1)
+  call s:SetDefault('g:python_highlight_string_formatting', 1)
+  call s:SetDefault('g:python_highlight_string_format', 1)
+  call s:SetDefault('g:python_highlight_string_templates', 1)
+  call s:SetDefault('g:python_highlight_indent_errors', 1)
+  call s:SetDefault('g:python_highlight_space_errors', 1)
+  call s:SetDefault('g:python_highlight_doctests', 1)
+  call s:SetDefault('g:python_highlight_func_calls', 1)
+  call s:SetDefault('g:python_highlight_class_vars', 1)
+  call s:SetDefault('g:python_highlight_operators', 1)
+  call s:SetDefault('g:python_highlight_file_headers_as_comments', 1)
+  call s:SetDefault('g:python_slow_sync', 1)
+endif
+
+" We need it because scripts.vim in vim uses "set ft=" which cannot be
+" overridden with setf (and we can't use set ft= so our scripts.vim work)
+func! s:Setf(ft)
+  if &filetype !~# '\<'.a:ft.'\>'
+    let &filetype = a:ft
+  endif
+endfunc
+
+" Function used for patterns that end in a star: don't set the filetype if the
+" file name matches ft_ignore_pat.
+" When using this, the entry should probably be further down below with the
+" other StarSetf() calls.
+func! s:StarSetf(ft)
+  if expand("<amatch>") !~ g:ft_ignore_pat && &filetype !~# '\<'.a:ft.'\>'
+    let &filetype = a:ft
+  endif
+endfunc
+
+augroup filetypedetect
+
+" scripts/build inserts here filetype detection autocommands
+
+au! BufNewFile,BufRead,StdinReadPost * if expand("<afile>") !~ g:ft_ignore_pat |
+  \ call polyglot#Shebang() | endif
+
+au BufEnter * if &ft == "" && expand("<afile>") !~ g:ft_ignore_pat |
+      \ call polyglot#ObserveShebang() | endif
+
+augroup END
+
+if !has_key(s:disabled_packages, 'autoindent')
+  " Code below re-implements sleuth for vim-polyglot
+  let g:loaded_sleuth = 1
+  let g:loaded_foobar = 1
+
+  " Makes shiftwidth to be synchronized with tabstop by default
+  if &shiftwidth == &tabstop
+    let &shiftwidth = 0
+  endif
+
+  function! s:guess(lines) abort
+    let options = {}
+    let ccomment = 0
+    let podcomment = 0
+    let triplequote = 0
+    let backtick = 0
+    let xmlcomment = 0
+    let heredoc = ''
+    let minindent = 10
+    let spaces_minus_tabs = 0
+    let i = 0
+
+    for line in a:lines
+      let i += 1
+
+      if !len(line) || line =~# '^\W*$'
+        continue
+      endif
+
+      if line =~# '^\s*/\*'
+        let ccomment = 1
+      endif
+      if ccomment
+        if line =~# '\*/'
+          let ccomment = 0
+        endif
+        continue
+      endif
+
+      if line =~# '^=\w'
+        let podcomment = 1
+      endif
+      if podcomment
+        if line =~# '^=\%(end\|cut\)\>'
+          let podcomment = 0
+        endif
+        continue
+      endif
+
+      if triplequote
+        if line =~# '^[^"]*"""[^"]*$'
+          let triplequote = 0
+        endif
+        continue
+      elseif line =~# '^[^"]*"""[^"]*$'
+        let triplequote = 1
+      endif
+
+      if backtick
+        if line =~# '^[^`]*`[^`]*$'
+          let backtick = 0
+        endif
+        continue
+      elseif &filetype ==# 'go' && line =~# '^[^`]*`[^`]*$'
+        let backtick = 1
+      endif
+
+      if line =~# '^\s*<\!--'
+        let xmlcomment = 1
+      endif
+      if xmlcomment
+        if line =~# '-->'
+          let xmlcomment = 0
+        endif
+        continue
+      endif
+
+      " This is correct order because both "<<EOF" and "EOF" matches end
+      if heredoc != ''
+        if line =~# heredoc
+          let heredoc = ''
+        endif
+        continue
+      endif
+      let herematch = matchlist(line, '\C<<\W*\([A-Z]\+\)\s*$')
+      if len(herematch) > 0
+        let heredoc = herematch[1] . '$'
+      endif
+
+      let spaces_minus_tabs += line[0] == "\t" ? 1 : -1
+
+      if line[0] == "\t"
+        setlocal noexpandtab
+        let &shiftwidth=&tabstop
+        let b:sleuth_culprit .= ':' . i
+        return 1
+      elseif line[0] == " "
+        let indent = len(matchstr(line, '^ *'))
+        if (indent % 2 == 0 || indent % 3 == 0) && indent < minindent
+          let minindent = indent
+        endif
+      endif
+    endfor
+
+    if minindent < 10
+      setlocal expandtab
+      let &shiftwidth=minindent
+      let b:sleuth_culprit .= ':' . i
+      return 1
+    endif
+
+    return 0
+  endfunction
+
+  function! s:detect_indent() abort
+    if &buftype ==# 'help'
+      return
+    endif
+
+    let b:sleuth_culprit = expand("<afile>:p")
+    if s:guess(getline(1, 32))
+      return
+    endif
+    let pattern = sleuth#GlobForFiletype(&filetype)
+    if len(pattern) == 0
+      return
+    endif
+    let pattern = '{' . pattern . ',.git,.svn,.hg}'
+    let dir = expand('%:p:h')
+    let level = 3
+    while isdirectory(dir) && dir !=# fnamemodify(dir, ':h') && level > 0
+      " Ignore files from homedir and root 
+      if dir == expand('~') || dir == '/'
+        unlet b:sleuth_culprit
+        return
+      endif
+      for neighbor in glob(dir . '/' . pattern, 0, 1)[0:level]
+        let b:sleuth_culprit = neighbor
+        " Do not consider directories above .git, .svn or .hg
+        if fnamemodify(neighbor, ":h:t")[0] == "."
+          let level = 0
+          continue
+        endif
+        if neighbor !=# expand('%:p') && filereadable(neighbor)
+          if s:guess(readfile(neighbor, '', 32))
+            return
+          endif
+        endif
+      endfor
+
+      let dir = fnamemodify(dir, ':h')
+      let level -= 1
+    endwhile
+
+    unlet b:sleuth_culprit
+  endfunction
+
+  setglobal smarttab
+
+  function! SleuthIndicator() abort
+    let sw = &shiftwidth ? &shiftwidth : &tabstop
+    if &expandtab
+      return 'sw='.sw
+    elseif &tabstop == sw
+      return 'ts='.&tabstop
+    else
+      return 'sw='.sw.',ts='.&tabstop
+    endif
+  endfunction
+
+  augroup polyglot-sleuth
+    au!
+    au FileType * call s:detect_indent()
+    au User Flags call Hoist('buffer', 5, 'SleuthIndicator')
+  augroup END
+
+  command! -bar -bang Sleuth call s:detect_indent()
+endif
+
+func! s:verify()
+  if exists("g:polyglot_disabled_not_set")
+    if exists("g:polyglot_disabled")
+      echohl WarningMsg
+      echo "vim-polyglot: g:polyglot_disabled should be defined before loading vim-polyglot"
+      echohl None
+    endif
+
+    unlet g:polyglot_disabled_not_set
+  endif
+endfunc
+
+au VimEnter * call s:verify()
+
+" Save polyglot_disabled without postfixes
+if exists('g:polyglot_disabled')
+  let g:polyglot_disabled = s:new_polyglot_disabled
+endif
+
+" restore Vi compatibility settings
+let &cpo = s:cpo_save
+
+""" end
diff --git a/scripts/build b/scripts/build
index 260beddb..0ed76045 100755
--- a/scripts/build
+++ b/scripts/build
@@ -12,6 +12,10 @@ Dir.chdir(File.dirname(__dir__))
 
 BASE_URL = 'https://raw.githubusercontent.com/github/linguist/master'
 
+def read_section(name)
+  File.read('polyglot.vim').match(/""" #{Regexp.escape(name)}.*\n(?=\n""")/m)[0]
+end
+
 def camelize(str)
   str.split(/[-_\.]/).map { |a| a.capitalize }.join("")
 end
@@ -121,6 +125,10 @@ def transform_patterns(heuristics)
   patterns_mapping = Hash[patterns.zip(patterns_to_vim_patterns(patterns))]
   each_hash(heuristics) do |h|
     if h.has_key?("pattern")
+      if h["pattern"].include?("#import")
+        puts h["pattern"]
+        puts patterns_mapping.fetch(h["pattern"])
+      end
       h["pattern"] = patterns_mapping.fetch(h["pattern"])
     end
   end
@@ -554,25 +562,14 @@ def generate_ftdetect(packages, heuristics)
     puts "Missing filename for #{name}: #{e}"
   end
 
-  ftdetect = File.read('ftdetect/polyglot.vim')
-  File.write('ftdetect/polyglot.vim', ftdetect.gsub(/(?<=" filetypes\n).*(?=\n" end filetypes)/m, output))
+  ftdetect = read_section('ftdetect/polyglot.vim')
 
-  output = <<~EOS
-    " Line continuation is used here, remove 'C' from 'cpoptions'
-    let s:cpo_save = &cpo
-    set cpo&vim
-
-    func! polyglot#Shebang()
-      " Try to detect filetype from shebang
-      let ft = polyglot#ShebangFiletype()
-      if ft != ""
-        let &ft = ft
-        return 1
-      endif
+  File.write(
+    'ftdetect/polyglot.vim',
+     ftdetect.gsub('" scripts/build inserts here filetype detection autocommands') { output }
+  )
 
-      return 0
-    endfunc
-    
+  output = <<~EOS
     let s:interpreters = {
     EOS
 
@@ -585,47 +582,6 @@ def generate_ftdetect(packages, heuristics)
   output << <<~EOS
       \\ }
 
-    let s:r_hashbang = '^#!\\s*\\(\\S\\+\\)\\s*\\(.*\\)\\s*'
-    let s:r_envflag = '%(\\S\\+=\\S\\+\\|-[iS]\\|--ignore-environment\\|--split-string\\)'
-    let s:r_env = '^\\%(\\' . s:r_envflag . '\\s\\+\\)*\\(\\S\\+\\)'
-
-    func! polyglot#ShebangFiletype()
-      let l:line1 = getline(1)
-
-      if l:line1 !~# "^#!"
-        return
-      endif
-
-      let l:pathrest = matchlist(l:line1, s:r_hashbang)
-
-      if len(l:pathrest) == 0
-        return 
-      endif
-
-      let [_, l:path, l:rest; __] = l:pathrest
-
-      let l:script = split(l:path, "/")[-1]
-
-      if l:script == "env"
-        let l:argspath = matchlist(l:rest, s:r_env)
-        if len(l:argspath) == 0
-          return
-        endif
-
-        let l:script = l:argspath[1]
-      endif
-
-      if has_key(s:interpreters, l:script)
-        return s:interpreters[l:script]
-      endif
-
-      for interpreter in keys(s:interpreters)
-        if l:script =~# '^' . interpreter
-          return s:interpreters[interpreter]
-        endif
-      endfor
-    endfunc
-
   EOS
 
   for heuristic in heuristics
@@ -637,13 +593,10 @@ def generate_ftdetect(packages, heuristics)
     EOS
   end
 
-  output << <<~EOS
-    " Restore 'cpoptions'
-    let &cpo = s:cpo_save
-    unlet s:cpo_save
-  EOS
+  autoload_script = read_section('autoload/polyglot.vim')
+  autoload_script["\" scripts/build generates heuristics functions here\n"] = output
 
-  File.write('autoload/polyglot.vim', output)
+  File.write('autoload/polyglot.vim', autoload_script)
 end
 
 def generate_tests(packages)
@@ -742,8 +695,6 @@ def detect_filetypes(glob)
 end
 
 def generate_plugins(packages)
-  FileUtils.mkdir_p('autoload/polyglot')
-
   output = "let s:globs = {\n"
 
   patterns = Hash.new { |h, k| h[k] = [] }