Introduce byte offset

pos.i is actually "character offset" and it doesn't handle
line-continuation, so we cannot use pos.i to extract node from the
source with it.

Introduce pos.offset which represents byte offset which can be used
above cases.

FIX #65

Author: haya14busa

.gitignore

@@ -1 +1 @@
-test/*.out
+*.out

Makefile

@@ -25,4 +25,8 @@ js/test: js/vimlparser.js
 py/test: py/vimlparser.py
 	test/run_command.sh python py/vimlparser.py
 
+test/node_position/test_position.out: test/node_position/test_position.vim test/node_position/test_position.ok
+	vim -u NONE -N --cmd "let &rtp .= ',' . getcwd()" -S test/node_position/test_position.vim
+	diff -u test/node_position/test_position.ok test/node_position/test_position.out
+
 .PHONY: all clean_compiled check test js/test py/test

autoload/vimlparser.vim

@@ -3768,13 +3768,17 @@ endfunction
 function! s:StringReader.__init__(lines)
   let self.buf = []
   let self.pos = []
+  let self.offset = []
   let lnum = 0
+  let offset = 0
   while lnum < len(a:lines)
     let col = 0
     for c in split(a:lines[lnum], '\zs')
       call add(self.buf, c)
       call add(self.pos, [lnum + 1, col + 1])
+      call add(self.offset, offset)
       let col += len(c)
+      let offset += len(c)
     endfor
     while lnum + 1 < len(a:lines) && a:lines[lnum + 1] =~# '^\s*\\'
       let skip = s:TRUE
@@ -3787,17 +3791,23 @@ function! s:StringReader.__init__(lines)
         else
           call add(self.buf, c)
           call add(self.pos, [lnum + 2, col + 1])
+          call add(self.offset, offset)
         endif
         let col += len(c)
+        let offset += len(c)
       endfor
       let lnum += 1
+      let offset += 1
     endwhile
     call add(self.buf, '<EOL>')
     call add(self.pos, [lnum + 1, col + 1])
+    call add(self.offset, offset)
     let lnum += 1
+    let offset += 1
   endwhile
   " for <EOF>
   call add(self.pos, [lnum + 1, 0])
+  call add(self.offset, offset)
   let self.i = 0
 endfunction
 
@@ -3892,7 +3902,7 @@ endfunction
 
 function! s:StringReader.getpos()
   let [lnum, col] = self.pos[self.i]
-  return {'i': self.i, 'lnum': lnum, 'col': col}
+  return {'i': self.i, 'lnum': lnum, 'col': col, 'offset': self.offset[self.i]}
 endfunction
 
 function! s:StringReader.setpos(pos)

test/node_position/test_position.ok

@@ -0,0 +1,9 @@
+function! F()
+  let x =
+\ 1
+
+  let x = "
+  \1
+	\2 <- tab
+  \3"
+endfunction

test/node_position/test_position.vim

@@ -0,0 +1,44 @@
+let s:vimlparser = vimlparser#import()
+
+function! s:run()
+  let src = [
+  \  '',
+  \  'function! F()',
+  \  '  let x =',
+  \  '\ 1',
+  \  '',
+  \  '  let x = "',
+  \  '  \1',
+  \  '	\2 <- tab',
+  \  '  \3"',
+  \  'endfunction',
+  \  '',
+  \  '" END',
+  \]
+  let r = s:vimlparser.StringReader.new(src)
+  let p = s:vimlparser.VimLParser.new(0)
+  let c = s:vimlparser.Compiler.new()
+  let toplevel = p.parse(r)
+  let func = toplevel.body[0]
+  let body = s:extract_body(func, src)
+  echom string(split(body, "\n"))
+  call writefile(split(body, "\n"), 'test/node_position/test_position.out')
+  qall!
+endfunction
+
+function! s:extract_body(func, src)
+  let pos = a:func.pos
+
+  " FIXME calculating endpos is workaround. Ideally, it should have the end
+  " position of the node.
+
+  let endpos = a:func.endfunction.pos
+  let endfunc = a:func.endfunction.ea
+  let cmdlen = endfunc.argpos.i - endfunc.cmdpos.i
+  let endpos.i += cmdlen
+  let endpos.offset += cmdlen
+
+  return join(a:src, "\n")[pos.offset : endpos.offset]
+endfunction
+
+call s:run()