Merge pull request #69 from haya14busa/fix-pos-i

Introduce byte offset

Author: mattn

.gitignore

@@ -1 +1 @@
-test/*.out
+*.out

.travis.yml

@@ -16,7 +16,7 @@ matrix:
       - MAKE_TARGET=test
     - env:
       - VIM_VERSION=installed
-      - MAKE_TARGET="clean_compiled check js/test py/test"
+      - MAKE_TARGET="clean_compiled check js/test py/test test/node_position/test_position.out"
 
 install:
   - |

Makefile

@@ -25,4 +25,8 @@ js/test: js/vimlparser.js
 py/test: py/vimlparser.py
 	test/run_command.sh python py/vimlparser.py
 
+test/node_position/test_position.out: test/node_position/test_position.vim test/node_position/test_position.ok
+	vim -u NONE -N --cmd "let &rtp .= ',' . getcwd()" -S test/node_position/test_position.vim
+	diff -u test/node_position/test_position.ok test/node_position/test_position.out
+
 .PHONY: all clean_compiled check test js/test py/test

autoload/vimlparser.vim

@@ -3769,12 +3769,14 @@ function! s:StringReader.__init__(lines)
   let self.buf = []
   let self.pos = []
   let lnum = 0
+  let offset = 0
   while lnum < len(a:lines)
     let col = 0
     for c in split(a:lines[lnum], '\zs')
       call add(self.buf, c)
-      call add(self.pos, [lnum + 1, col + 1])
+      call add(self.pos, [lnum + 1, col + 1, offset])
       let col += len(c)
+      let offset += len(c)
     endfor
     while lnum + 1 < len(a:lines) && a:lines[lnum + 1] =~# '^\s*\\'
       let skip = s:TRUE
@@ -3786,18 +3788,21 @@ function! s:StringReader.__init__(lines)
           endif
         else
           call add(self.buf, c)
-          call add(self.pos, [lnum + 2, col + 1])
+          call add(self.pos, [lnum + 2, col + 1, offset])
         endif
         let col += len(c)
+        let offset += len(c)
       endfor
       let lnum += 1
+      let offset += 1
     endwhile
     call add(self.buf, '<EOL>')
-    call add(self.pos, [lnum + 1, col + 1])
+    call add(self.pos, [lnum + 1, col + 1, offset])
     let lnum += 1
+    let offset += 1
   endwhile
   " for <EOF>
-  call add(self.pos, [lnum + 1, 0])
+  call add(self.pos, [lnum + 1, 0, offset])
   let self.i = 0
 endfunction
 
@@ -3891,8 +3896,8 @@ function! s:StringReader.getstr(begin, end)
 endfunction
 
 function! s:StringReader.getpos()
-  let [lnum, col] = self.pos[self.i]
-  return {'i': self.i, 'lnum': lnum, 'col': col}
+  let [lnum, col, offset] = self.pos[self.i]
+  return {'i': self.i, 'lnum': lnum, 'col': col, 'offset': offset}
 endfunction
 
 function! s:StringReader.setpos(pos)

js/vimlfunc.js

@@ -151,6 +151,9 @@ function viml_keys(obj) {
 }
 
 function viml_len(obj) {
+    if (typeof obj === 'string') {
+      return encodeURIComponent(obj).replace(/%../g, ' ').length;
+    }
     return obj.length;
 }
 

js/vimlparser.js

@@ -151,6 +151,9 @@ function viml_keys(obj) {
 }
 
 function viml_len(obj) {
+    if (typeof obj === 'string') {
+      return encodeURIComponent(obj).replace(/%../g, ' ').length;
+    }
     return obj.length;
 }
 
@@ -3660,14 +3663,16 @@ StringReader.prototype.__init__ = function(lines) {
     this.buf = [];
     this.pos = [];
     var lnum = 0;
+    var offset = 0;
     while (lnum < viml_len(lines)) {
         var col = 0;
         var __c7 = viml_split(lines[lnum], "\\zs");
         for (var __i7 = 0; __i7 < __c7.length; ++__i7) {
             var c = __c7[__i7];
             viml_add(this.buf, c);
-            viml_add(this.pos, [lnum + 1, col + 1]);
+            viml_add(this.pos, [lnum + 1, col + 1, offset]);
             col += viml_len(c);
+            offset += viml_len(c);
         }
         while (lnum + 1 < viml_len(lines) && viml_eqregh(lines[lnum + 1], "^\\s*\\\\")) {
             var skip = TRUE;
@@ -3682,18 +3687,21 @@ StringReader.prototype.__init__ = function(lines) {
                 }
                 else {
                     viml_add(this.buf, c);
-                    viml_add(this.pos, [lnum + 2, col + 1]);
+                    viml_add(this.pos, [lnum + 2, col + 1, offset]);
                 }
                 col += viml_len(c);
+                offset += viml_len(c);
             }
             lnum += 1;
+            offset += 1;
         }
         viml_add(this.buf, "<EOL>");
-        viml_add(this.pos, [lnum + 1, col + 1]);
+        viml_add(this.pos, [lnum + 1, col + 1, offset]);
         lnum += 1;
+        offset += 1;
     }
     // for <EOF>
-    viml_add(this.pos, [lnum + 1, 0]);
+    viml_add(this.pos, [lnum + 1, 0, offset]);
     this.i = 0;
 }
 
@@ -3792,7 +3800,8 @@ StringReader.prototype.getpos = function() {
     var __tmp = this.pos[this.i];
     var lnum = __tmp[0];
     var col = __tmp[1];
-    return {"i":this.i, "lnum":lnum, "col":col};
+    var offset = __tmp[2];
+    return {"i":this.i, "lnum":lnum, "col":col, "offset":offset};
 }
 
 StringReader.prototype.setpos = function(pos) {

py/vimlfunc.py

@@ -128,6 +128,8 @@ def viml_keys(obj):
     return obj.keys()
 
 def viml_len(obj):
+    if type(obj) is str:
+        return len(obj.encode('utf-8'))
     return len(obj)
 
 def viml_printf(*args):

py/vimlparser.py

@@ -128,6 +128,8 @@ def viml_keys(obj):
     return obj.keys()
 
 def viml_len(obj):
+    if type(obj) is str:
+        return len(obj.encode('utf-8'))
     return len(obj)
 
 def viml_printf(*args):
@@ -2921,12 +2923,14 @@ def __init__(self, lines):
         self.buf = []
         self.pos = []
         lnum = 0
+        offset = 0
         while lnum < viml_len(lines):
             col = 0
             for c in viml_split(lines[lnum], "\\zs"):
                 viml_add(self.buf, c)
-                viml_add(self.pos, [lnum + 1, col + 1])
+                viml_add(self.pos, [lnum + 1, col + 1, offset])
                 col += viml_len(c)
+                offset += viml_len(c)
             while lnum + 1 < viml_len(lines) and viml_eqregh(lines[lnum + 1], "^\\s*\\\\"):
                 skip = TRUE
                 col = 0
@@ -2936,14 +2940,17 @@ def __init__(self, lines):
                             skip = FALSE
                     else:
                         viml_add(self.buf, c)
-                        viml_add(self.pos, [lnum + 2, col + 1])
+                        viml_add(self.pos, [lnum + 2, col + 1, offset])
                     col += viml_len(c)
+                    offset += viml_len(c)
                 lnum += 1
+                offset += 1
             viml_add(self.buf, "<EOL>")
-            viml_add(self.pos, [lnum + 1, col + 1])
+            viml_add(self.pos, [lnum + 1, col + 1, offset])
             lnum += 1
+            offset += 1
         # for <EOF>
-        viml_add(self.pos, [lnum + 1, 0])
+        viml_add(self.pos, [lnum + 1, 0, offset])
         self.i = 0
 
     def eof(self):
@@ -3015,8 +3022,8 @@ def getstr(self, begin, end):
         return r
 
     def getpos(self):
-        lnum, col = self.pos[self.i]
-        return AttributeDict({"i":self.i, "lnum":lnum, "col":col})
+        lnum, col, offset = self.pos[self.i]
+        return AttributeDict({"i":self.i, "lnum":lnum, "col":col, "offset":offset})
 
     def setpos(self, pos):
         self.i = pos.i

test/node_position/test_position.ok

@@ -0,0 +1,10 @@
+function! F()
+  let x =
+\ 1
+
+  let x = "
+  \1
+	\2 <- tab
+  \3 マルチバイト
+  \4"
+endfunction

test/node_position/test_position.vim

@@ -0,0 +1,43 @@
+let s:vimlparser = vimlparser#import()
+
+function! s:run()
+  let src = [
+  \  '',
+  \  'function! F()',
+  \  '  let x =',
+  \  '\ 1',
+  \  '',
+  \  '  let x = "',
+  \  '  \1',
+  \  '	\2 <- tab',
+  \  '  \3 マルチバイト',
+  \  '  \4"',
+  \  'endfunction',
+  \  '',
+  \  '" END',
+  \]
+  let r = s:vimlparser.StringReader.new(src)
+  let p = s:vimlparser.VimLParser.new(0)
+  let c = s:vimlparser.Compiler.new()
+  let toplevel = p.parse(r)
+  let func = toplevel.body[0]
+  let body = s:extract_body(func, src)
+  call writefile(split(body, "\n"), 'test/node_position/test_position.out')
+  qall!
+endfunction
+
+function! s:extract_body(func, src)
+  let pos = a:func.pos
+
+  " FIXME calculating endpos is workaround. Ideally, it should have the end
+  " position of the node.
+
+  let endpos = a:func.endfunction.pos
+  let endfunc = a:func.endfunction.ea
+  let cmdlen = endfunc.argpos.offset - endfunc.cmdpos.offset
+  let endpos.offset += cmdlen
+
+  return join(a:src, "\n")[pos.offset : endpos.offset]
+endfunction
+
+call s:run()