refactor codeAsInSource to not require command + fix nbJs bug fixes (#163)

* refactor codeAsInSource to not require command

* fix infix

* add more fixes

* implement nb.sourceFiles

* fix typesection and add discard test

* fix for loops

* let try pietro's idea of taking the minimum

* fix parallel bug

* add template test

* fix type bug

* update changelog

* bump nimble version

* clean sources

* add Hugo as co-author

* add filename assert in Pos comparision

* assert -> doAssert. Improved error message and added an assert for the endPos as well

Author: HugoGranstrom

changelog.md

@@ -10,6 +10,11 @@ When contributing a fix, feature or example please add a new line to briefly exp
 ## 0.3.x
 * _add next change here_
 
+## 0.3.5
+* codeAsInSource has been reworked to work better with templates and uses of `nbCode` in different files.
+* If you don't use any nbJs, now nimib won't build an empty nim file in those cases.
+* The temporary js files generated by nbJs now has unique names to allow parallel builds.
+
 ## 0.3.4
 
 * added `nbCodeDisplay` and `nbCodeAnd` (#158).

nimib.nimble

@@ -1,7 +1,7 @@
 # Package
 
-version       = "0.3.4"
-author        = "Pietro Peterlongo"
+version       = "0.3.5"
+author        = "Pietro Peterlongo & Hugo Granström"
 description   = "nimib 🐳 - nim 👑 driven ⛵ publishing ✍"
 license       = "MIT"
 srcDir        = "src"

src/nimib/jsutils.nim

@@ -1,4 +1,4 @@
-import std / [macros, macrocache, tables, strutils, strformat, sequtils, sugar, os]
+import std / [macros, macrocache, tables, strutils, strformat, sequtils, sugar, os, hashes]
 import ./types
 
 proc contains(tab: CacheTable, keyToCheck: string): bool =
@@ -142,7 +142,7 @@ proc compileNimToJs*(doc: var NbDoc, blk: var NbBlock) =
   createDir(tempdir)
   let (dir, filename, ext) = doc.thisFile.splitFile()
   let nimfile = dir / (filename & "_nbCodeToJs_" & $doc.newId() & ext).RelativeFile
-  let jsfile = tempdir / "out.js"
+  let jsfile = tempdir / &"out{hash(doc.thisFile)}.js"
   var codeText = blk.context["transformedCode"].vString
   let nbJsCounter = doc.nbJsCounter
   doc.nbJsCounter += 1
@@ -178,8 +178,9 @@ proc nbCollectAllNbJs*(doc: var NbDoc) =
         code.add "\n" & blk.context["transformedCode"].vString
   code = topCode & "\n" & code
 
-  # Create block which which will compile the code when rendered (nbJsFromJsOwnFile)
-  var blk = NbBlock(command: "nbJsFromCodeOwnFile", code: code, context: newContext(searchDirs = @[], partials = doc.partials), output: "")
-  blk.context["transformedCode"] = code
-  doc.blocks.add blk
-  doc.blk = blk
+  if not code.isEmptyOrWhitespace:
+    # Create block which which will compile the code when rendered (nbJsFromJsOwnFile)
+    var blk = NbBlock(command: "nbJsFromCodeOwnFile", code: code, context: newContext(searchDirs = @[], partials = doc.partials), output: "")
+    blk.context["transformedCode"] = code
+    doc.blocks.add blk
+    doc.blk = blk

src/nimib/sources.nim

@@ -6,29 +6,40 @@ import std/[
 import types
 
 
-
 # Credits to @haxscramper for sharing his code on reading the line info
 # And credits to @Yardanico for making a previous attempt which @hugogranstrom have taken much inspiration from
 # when implementing this.
 
 type
   Pos* = object
+    filename*: string
     line*: int
     column*: int
 
+proc `<`*(p1, p2: Pos): bool =
+  doAssert p1.filename == p2.filename, """
+  Code from two different files were found in the same nbCode!
+  If you want to mix code from different files in nbCode, use -d:nimibCodeFromAst instead. 
+  If you are not mixing code from different files, please open an issue on nimib's Github with a minimal reproducible example."""
+  (p1.line, p1.column) < (p2.line, p2.column)
+
 proc toPos*(info: LineInfo): Pos =
-  Pos(line: info.line, column: info.column)
+  Pos(line: info.line, column: info.column, filename: info.filename)
+
+proc startPos(node: NimNode): Pos =
+  case node.kind
+  of nnkStmtList:
+    return node[0].startPos()
+  else:
+    result = toPos(node.lineInfoObj())
+    for child in node.children:
+      let childPos = child.startPos()
+      # If we can't get the line info for some reason, skip it!
+      if childPos.line == 0: continue
+      
+      if childPos < result:
+        result = childPos
 
-proc startPos*(node: NimNode): Pos =
-  ## Get the starting position of a NimNode. Corrections will be needed for certains cases though.
-  # Has column info
-  case node.kind:
-    of nnkNone .. nnkNilLit, nnkDiscardStmt, nnkCommentStmt:
-      result = toPos(node.lineInfoObj())
-    of nnkBlockStmt:
-      result = node[1].startPos()
-    else:
-      result = node[0].startPos()
 
 proc finishPos*(node: NimNode): Pos =
   ## Get the ending position of a NimNode. Corrections will be needed for certains cases though.
@@ -56,18 +67,24 @@ proc finishPos*(node: NimNode): Pos =
 proc isCommandLine*(s: string, command: string): bool =
   nimIdentNormalize(s.strip()).startsWith(nimIdentNormalize(command))
 
+proc isCommentLine*(s: string): bool =
+  s.strip.startsWith('#')
+
+proc findStartLine*(source: seq[string], startPos: Pos): int =
+  let line = source[startPos.line - 1]
+  let preline = line[0 ..< startPos.column - 1]
+  # Multiline, we need to check further up for comments
+  if preline.isEmptyOrWhitespace:
+    result = startPos.line - 1
+    # Make sure we catch all comments
+    while source[result-1].isCommentLine() or source[result-1].isEmptyOrWhitespace() or source[result-1].nimIdentNormalize.strip() == "type":
+      dec result
+    # Now remove all empty lines
+    while source[result].isEmptyOrWhitespace():
+      inc result
+  else: # starts on same line as command
+    return startPos.line - 1
 
-proc findStartLine*(source: seq[string], command: string, startPos: int): int =
-  if source[startPos].isCommandLine(command):
-    return startPos
-  # The code is starting on a line below the command
-  # Decrease result until it is on the line below the command
-  result = startPos
-  while not source[result-1].isCommandLine(command):
-    dec result
-  # Remove empty lines at the beginning of the block
-  while source[result].isEmptyOrWhitespace:
-    inc result
 
 proc findEndLine*(source: seq[string], command: string, startLine, endPos: int): int =
   result = endPos
@@ -90,49 +107,32 @@ proc findEndLine*(source: seq[string], command: string, startLine, endPos: int):
     while result < source.high and (source[result+1].startsWith(baseIndentStr) or source[result+1].isEmptyOrWhitespace):
       inc result
 
-
 proc getCodeBlock*(source, command: string, startPos, endPos: Pos): string =
   ## Extracts the code in source from startPos to endPos with additional processing to get the entire code block.
-  let rawLines = source.split("\n")
-  var startLine = findStartLine(rawLines, command, startPos.line - 1)
+  let rawLines = source.splitLines()
+  let rawStartLine = startPos.line - 1
+  let rawStartCol = startPos.column - 1
+  var startLine = findStartLine(rawLines, startPos)
   var endLine = findEndLine(rawLines, command, startLine, endPos.line - 1)
 
   var lines = rawLines[startLine .. endLine]
 
-  let baseIndent = skipWhile(rawLines[startLine], {' '})
-
-  let startsOnCommandLine = lines[0].isCommandLine(command) # is it nbCode: code or nbCode: <enter> code
-  if startsOnCommandLine: # remove the command
-    var startColumn = startPos.column
-    # the "import"-part is not included in the startPos
-    let startsWithImport = lines[0].find("import")
-    if startsWithImport != -1:
-      startColumn = startsWithImport
-    lines[0] = lines[0][startColumn .. ^1].strip()
-
-  var codeText: string
-  if startLine == endLine and startsOnCommandLine: # single-line expression 
-    # remove eventual unneccerary parenthesis
-    let line = rawLines[startLine] # includes command and eventual opening parethesises
-    var extractedLine = lines[0] # doesn't include command
-    if extractedLine.endsWith(")"):
-      # check if the ending ")" has a matching "(", otherwise remove it.
-      var nOpen: int
-      var i = startPos.column
-      # count the number of opening brackets before code starts.
-      while line[i-1] in Whitespace or line[i-1] == '(':
-        if line[i-1] == '(':
-          nOpen += 1
-        i -= 1
-      var nRemoved: int
-      while nRemoved < nOpen: # remove last char until we have removed correct number of parentesis
-                              # We assume we are given correct Nim code and thus won't have to check what we remove, it should either be Whitespace or ')'
-        assert extractedLine[^1] in Whitespace or extractedLine[^1] == ')', "Unexpected ending of string during parsing. Single line expression ended with character that wasn't whitespace of ')'."
-        if extractedLine[^1] == ')':
-          nRemoved += 1
-        extractedLine.setLen(extractedLine.len-1)
-    codeText = extractedLine
+  let startsOnCommandLine = block:
+    let preline = lines[0][0 ..< rawStartCol]
+    startLine == rawStartLine and (not preline.isEmptyOrWhitespace) and (not (preline.nimIdentNormalize.strip() in ["for", "type"]))
+    
+  if startsOnCommandLine:
+    lines[0] = lines[0][rawStartCol .. ^1].strip()
+  
+  if startLine == endLine and startsOnCommandLine:
+    # single line expression
+    var line = lines[0] # doesn't include command, but includes opening parenthesis
+    while line.startsWith('(') and line.endsWith(')'):
+      line = line[1 .. ^2].strip()
+
+    result = line
   else: # multi-line expression
+    let baseIndent = skipWhile(rawLines[startLine], {' '})
     var preserveIndent: bool = false
     for i in 0 .. lines.high:
       let line = lines[i]
@@ -141,93 +141,23 @@ proc getCodeBlock*(source, command: string, startPos, endPos: Pos): string =
         lines[i] = line.substr(baseIndent)
       if nonMatching: # there is a non-matching triple-quote string
         preserveIndent = not preserveIndent
-    codeText = lines.join("\n")
-  result = codeText
-
-
-  
-
-func getCodeBlockOld*(source: string, command: string, startPos, endPos: Pos): string =
-  ## Extracts the code in source from startPos to endPos with additional processing to get the entire code block.
-  let lines = source.split("\n")
-  var startLine = startPos.line - 1
-  var endLine = endPos.line - 1
-  debugecho "Start line: ", startLine + 1, startPos
-  debugecho "End line: ", endLine + 1, endPos
-  var codeText: string
-  if not lines[startLine].isCommandLine(command): # multiline case
-    while 0 < startLine and not lines[startLine-1].isCommandLine(command):
-      #[ cases like this reports the third line instead of the second line:
-        nbCode:
-          let # this is the line we want
-            x = 1 # but this is the one we get
-      ]#
-      dec startLine
-
-    let indent = skipWhile(lines[startLine], {' '})
-    let indentStr = " ".repeat(indent)
-
-    if lines[endLine].count("\"\"\"") == 1: # only opening of triple quoted string found. Rest is below it. 
-      inc endLine # bump it to not trigger the loop to immediately break
-      while endLine < lines.high and "\"\"\"" notin lines[endLine]:
-        inc endLine
-        debugecho "Triple quote: ", lines[endLine]
-
-    while endLine < lines.high and (lines[endLine+1].startsWith(indentStr) or lines[endLine+1].isEmptyOrWhitespace):# and lines[endLine+1].strip().startsWith("#"):
-      # Ending Comments should be included as well, but they won't be included in the AST -> endLine doesn't take them into account.
-      # Block comments must be properly indented (including the content)
-      inc endLine
-
-    var codeLines = lines[startLine .. endLine]
-
-    var notIndentLines: seq[int] # these lines are not to be adjusted for indentation. Eg content of triple quoted strings.
-    var i: int
-    while i < codeLines.len:
-      if codeLines[i].count("\"\"\"") == 1:
-        # We must do the identification of triple quoted string separatly from the endLine bumping because the triple strings
-        # might not be the last expression in the code block.
-        inc i # bump it to not trigger the loop to immediately break on the initial """
-        notIndentLines.add i
-        while i < codeLines.len and "\"\"\"" notin codeLines[i]:
-          inc i
-          notIndentLines.add i
-      inc i
-      
-    let parsedLines = collect(newSeqOfCap(codeLines.len)):
-      for i in 0 .. codeLines.high:
-        if i in notIndentLines:
-          codeLines[i]
-        else:
-          codeLines[i].substr(indent)
-    codeText = parsedLines.join("\n")
-  elif lines[startLine].isCommandLine(command) and "\"\"\"" in lines[startLine]: # potentially multiline string 
-    discard
-  else: # single line case, eg `nbCode: echo "Hello World"`
-    let line = lines[startLine]
-    var extractedLine = line[startPos.column .. ^1].strip()
-    if extractedLine.strip().endsWith(")"):
-      # check if the ending ")" has a matching "(", otherwise remove it.
-      var nOpen: int
-      var i = startPos.column
-      # count the number of opening brackets before code starts.
-      while line[i-1] in Whitespace or line[i-1] == '(':
-        if line[i-1] == '(':
-          nOpen += 1
-        i -= 1
-      var nRemoved: int
-      while nRemoved < nOpen: # remove last char until we have removed correct number of parentesis
-                              # We assume we are given correct Nim code and thus won't have to check what we remove, it should either be Whitespace or ')'
-        assert extractedLine[^1] in Whitespace or extractedLine[^1] == ')', "Unexpected ending of string during parsing. Single line expression ended with character that wasn't whitespace of ')'."
-        if extractedLine[^1] == ')':
-          nRemoved += 1
-        extractedLine.setLen(extractedLine.len-1)
-    codeText = extractedLine
-  return codeText
+    result = lines.join("\n")
 
 macro getCodeAsInSource*(source: string, command: static string, body: untyped): string =
   ## Returns string for the code in body from source. 
   # substitute for `toStr` in blocks.nim
   let startPos = startPos(body)
+  let filename = startPos.filename.newLit
   let endPos = finishPos(body)
+  let endFilename = endPos.filename.newLit
+
   result = quote do:
-    getCodeBlock(`source`, `command`, `startPos`, `endPos`)
+    if `filename` notin nb.sourceFiles:
+      nb.sourceFiles[`filename`] = readFile(`filename`)
+
+    doAssert `endFilename` == `filename`, """
+    Code from two different files were found in the same nbCode!
+    If you want to mix code from different files in nbCode, use -d:nimibCodeFromAst instead. 
+    If you are not mixing code from different files, please open an issue on nimib's Github with a minimal reproducible example."""
+
+    getCodeBlock(nb.sourceFiles[`filename`], `command`, `startPos`, `endPos`)

src/nimib/types.nim

@@ -20,6 +20,7 @@ type
     thisFile*: AbsoluteFile
     filename*: string
     source*: string
+    sourceFiles*: Table[string, string]
     initDir*: AbsoluteDir
     options*: NbOptions
     cfg*: NbConfig

tests/tsources.nim

@@ -7,8 +7,9 @@ suite "test sources":
     # the replace stuff needed on windows where the lines read from file will have windows native new lines
     test $currentTest:
       actual = nbBlock.code
-      echo &"===\n---actual:\n{actual.repr}\n---expected\n{expected.repr}\n---\n==="
       check actual.nbNormalize == expected.nbNormalize
+      if actual.nbNormalize != expected.nbNormalize:
+        echo &"===\n---actual:\n{actual.repr}\n---expected\n{expected.repr}\n---\n==="
     currentTest += 1
 
   var currentTest: int
@@ -107,4 +108,59 @@ end"""
     expected = "echo y"
     check
 
+    nbCode:
+      block:
+        let
+          b = 1
+
+    expected = "block:\n  let\n    b = 1"
+    check
+
+    template notNbCode(body: untyped) =
+      nbCode:
+        body
+
+    notNbCode:
+      echo y
+
+    expected = "echo y"
+    check
+
+    template `&`(a,b: int) = discard
+
+    nbCode:
+      1 &
+        2
+
+    expected = "1 &\n  2"
+    check
+
+    nbCode:
+      nb.context["no_source"] = true
+
+    expected = "nb.context[\"no_source\"] = true"
+    check
+
+    nbCode: discard
+    expected = "discard"
+    check
+
+    nbCode:
+      for n in 0 .. 1:
+        discard
+    expected = "for n in 0 .. 1:\n  discard"
+    check
+
+    template nbCodeInTemplate =
+      nbCode:
+        nb.renderPlans["nbText"] = @["mdOutputToHtml"]
+
+    nbCodeInTemplate()
+    expected = """nb.renderPlans["nbText"] = @["mdOutputToHtml"]"""
+    check
+
+    nbCode:
+      type A = object
+    expected = "type A = object"
+    check