refactor: self-contain source plugin

autocomplete now fully uses that codec to update the autocomplete shell script on system when using TTY session
crashappsec · Feb 25, 2025 · b629923 · b629923
1 parent 8741a25
commit b629923
Show file tree

Hide file tree

Showing 22 changed files with 535 additions and 489 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,12 @@
   configurations as well as keys.
   ([#352](https://github.com/crashappsec/chalk/pull/352))
 
+### Fixes
+
+- In interactive shell, autocomplete script is now only updated
+  when its content is changed.
+  ([#493](https://github.com/crashappsec/chalk/pull/493))
+
 ## 0.5.4
 
 **Feb 19, 2025**

diff --git a/chalk.nimble b/chalk.nimble
@@ -50,26 +50,3 @@ task performance, "Get a build that adds execution profiling support":
 
 task memprofile, "Get a build that adds memory profiling to the binary":
   exec "nimble build --profiler:off --stackTrace:on -d:memProfiler -d:cprofiling"
-
-let completion_script_version = version
-
-task mark_completion, "Replace the chalk mark in a completion script, including the articact version":
-
-  exec """cat > tmpcfg.c4m << EOF
-keyspec.ARTIFACT_VERSION.value = "REPLACE_ME"
-keyspec.CHALK_PTR.value = ("This mark determines when to update the script." +
-" If there is no mark, or the mark is invalid it will be replaced. " +
-" To customize w/o Chalk disturbing it when it can update, add a valid " +
-" mark with a version key higher than the current chalk verison, or " +
-" use version 0.0.0 to prevent updates")
-
-mark_template.mark_default.key.BRANCH.use = false
-mark_template.mark_default.key.CHALK_RAND.use = false
-mark_template.mark_default.key.CODE_OWNERS.use = false
-mark_template.mark_default.key.COMMIT_ID.use = false
-mark_template.mark_default.key.PLATFORM_WHEN_CHALKED.use = false
-EOF
-""".replace("REPLACE_ME", completion_script_version)
-
-  exec "./chalk --use-external-config --config-file=./tmpcfg.c4m --no-use-embedded-config --skip-command-report insert src/autocomplete"
-  exec "rm ./tmpcfg.c4m"
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -24,6 +24,7 @@ services:
     environment:
       DEBUG: ${DEBUG:-}
       CHALK_PASSWORD: ${CHALK_PASSWORD:-}
+      CHALK_BUILD: ${CHALK_BUILD:-release}
 
   # --------------------------------------------------------------------------
   # TESTS

diff --git a/src/attestation_api.nim b/src/attestation_api.nim
@@ -174,10 +174,10 @@ proc willSignByHash*(chalk: ChalkObj): bool =
   # If there's no associated fs ref, it's either a container or
   # something we don't have permission to read; either way, it's not
   # getting signed in this flow.
-  return willSign() and chalk.canVerifyByHash()
+  return willSign() and not chalk.noAttestation and chalk.canVerifyByHash()
 
 proc willSignBySigStore*(chalk: ChalkObj): bool =
-  return willSign() and chalk.canVerifyBySigStore()
+  return willSign() and not chalk.noAttestation and chalk.canVerifyBySigStore()
 
 proc verifyBySigStore(chalk: ChalkObj, key: AttestationKey, image: DockerImage): (ValidateResult, ChalkDict) =
   let

diff --git a/src/autocomplete.nim b/src/autocomplete.nim
@@ -5,8 +5,17 @@
 ## (see https://crashoverride.com/docs/chalk)
 ##
 
-import std/[posix]
-import "."/[config, subscan, fd_cache, config_version, semver]
+import std/[posix, options]
+import "."/[
+  chalkjson,
+  collect,
+  config,
+  config_version,
+  fd_cache,
+  plugin_api,
+  selfextract,
+  subscan,
+]
 
 when hostOs == "macosx":
   const staticScriptLoc = "autocomplete/mac.bash"
@@ -95,6 +104,9 @@ proc validateMetaData*(obj: ChalkObj): ValidateResult {.importc.}
 proc autocompleteFileCheck*() =
   if isatty(0) == 0 or attrGet[bool]("install_completion_script") == false:
     return
+  # compiling chalk itself
+  if existsEnv("CHALK_BUILD"):
+    return
 
   var dst = ""
   try:
@@ -111,21 +123,26 @@ proc autocompleteFileCheck*() =
 
     if len(allChalks) != 0 and allChalks[0].extract != nil:
       if (
-        "ARTIFACT_VERSION" in allChalks[0].extract and
+        "HASH" in allChalks[0].extract and
+        "CHALK_VERSION" in allChalks[0].extract and
         allChalks[0].validateMetaData() == vOk
       ):
         const chalkVersion = getChalkVersion()
         let
-          boxedVersion   = allChalks[0].extract["ARTIFACT_VERSION"]
-          foundVersion   = parseVersion(unpack[string](boxedVersion))
-          currentVersion = parseVersion(chalkVersion)
-
-        trace("Extracted semver string from existing autocomplete file: " & $foundVersion)
-
-        if foundVersion < currentVersion:
-          info("Updating autocomplete script to version: " & $currentVersion)
+          boxedVersion   = allChalks[0].extract["CHALK_VERSION"]
+          foundVersion   = unpack[string](boxedVersion)
+          boxedHash      = allChalks[0].extract["HASH"]
+          foundHash      = unpack[string](boxedHash)
+          embedHash      = bashScript.sha256Hex()
+
+        trace("Extracted semver string from existing autocomplete file: " & foundVersion)
+
+        # compare if the autocomplete script actually changed
+        # vs comparing chalk version
+        if foundHash != embedHash:
+          info("Updating autocomplete script to current version: " & chalkVersion)
         else:
-          trace("Autocomplete script does not need updating.")
+          trace("Autocomplete script is up to date. Skipping.")
           return
     else:
       info("Autocomplete file exists but is missing chalkmark. Updating.")
@@ -141,9 +158,25 @@ proc autocompleteFileCheck*() =
   if not tryToWriteFile(dst, bashScript):
     warn("Could not write to auto-completion file: " & dst)
     return
-  else:
-    info("Installed bash auto-completion file to: " & dst)
 
+  let selfChalkOpt = getSelfExtraction()
+  if selfChalkOpt.isSome():
+    let
+      selfChalk = selfChalkOpt.get()
+      autoCompleteChalk = newChalk(
+        dst,
+        fsRef         = dst,
+        codec         = getPluginByName("source"),
+        noAttestation = true,
+      ).copyCollectedDataFrom(selfChalk)
+    withSuspendChalkCollectionFor(autoCompleteChalk.getRequiredPlugins()):
+      initCollection()
+      collectChalkTimeHostInfo()
+      collectChalkTimeArtifactInfo(autoCompleteChalk, override = true)
+    let chalkMark = autoCompleteChalk.getChalkMarkAsStr()
+    autoCompleteChalk.callHandleWrite(some(chalkMark))
+
+  info("Installed bash auto-completion file to: " & dst)
   if not alreadyExists:
     makeCompletionAutoSource(dst)
 
@@ -152,3 +185,4 @@ proc setupAutocomplete*() =
     autocompleteFileCheck()
   except:
     warn("could not check autocomplete file due to: " & getCurrentExceptionMsg())
+    dumpExOnDebug()
diff --git a/src/autocomplete/default.bash b/src/autocomplete/default.bash
@@ -168,4 +168,3 @@ function _chalk_completions {
 }
 
 complete -F _chalk_completions chalk
-# { "MAGIC" : "dadfedabbadabbed", "CHALK_ID" : "C4TPCR-SSCH-GKJR-V46DGK", "CHALK_VERSION" : "0.3.3", "TIMESTAMP_WHEN_CHALKED" : 1710356599977, "DATETIME_WHEN_CHALKED" : "2024-03-13T15:03:18.901-04:00", "ARTIFACT_TYPE" : "bash", "AUTHOR" : "Miroslav Shubernetskiy <[email protected]> 1710352293 -0400", "BRANCH" : "ms", "CHALK_RAND" : "9f75af21a1bf7665", "CODE_OWNERS" : "* @viega\n", "COMMITTER" : "Miroslav Shubernetskiy <[email protected]> 1710352893 -0400", "COMMIT_ID" : "5f95367b955256bb92254a5bddaea6e5285a29f6", "COMMIT_MESSAGE" : "build: bumping con4m to include get(dict, \"key\")\n\nits used in connect.c4m", "COMMIT_SIGNED" : true, "DATE_AUTHORED" : "Wed Mar 13 13:51:33 2024 -0400", "DATE_COMMITTED" : "Wed Mar 13 14:01:33 2024 -0400", "HASH" : "a5fc9da9cd3a291f4758e8e19028efeb4ba984dda9e4db7e8762215521767f83", "INJECTOR_COMMIT_ID" : "e96746336a5dd6618d4a8d5eae7a5542d048f301", "ORIGIN_URI" : "[email protected]:crashappsec/chalk.git", "PLATFORM_WHEN_CHALKED" : "GNU/Linux x86_64", "METADATA_ID" : "04CB0D-M6D6-ZP5Q-XN6FHA" }
diff --git a/src/autocomplete/mac.bash b/src/autocomplete/mac.bash
@@ -146,4 +146,3 @@ function _chalk_completions {
 }
 
 complete -F _chalk_completions chalk
-# { "MAGIC" : "dadfedabbadabbed", "CHALK_ID" : "CCVP2S-HSCR-VK6C-3569H3", "CHALK_VERSION" : "0.3.3", "TIMESTAMP_WHEN_CHALKED" : 1710356599979, "DATETIME_WHEN_CHALKED" : "2024-03-13T15:03:18.901-04:00", "ARTIFACT_TYPE" : "bash", "AUTHOR" : "Miroslav Shubernetskiy <[email protected]> 1710352293 -0400", "BRANCH" : "ms", "CHALK_RAND" : "63b2eb8e1ecca0bc", "CODE_OWNERS" : "* @viega\n", "COMMITTER" : "Miroslav Shubernetskiy <[email protected]> 1710352893 -0400", "COMMIT_ID" : "5f95367b955256bb92254a5bddaea6e5285a29f6", "COMMIT_MESSAGE" : "build: bumping con4m to include get(dict, \"key\")\n\nits used in connect.c4m", "COMMIT_SIGNED" : true, "DATE_AUTHORED" : "Wed Mar 13 13:51:33 2024 -0400", "DATE_COMMITTED" : "Wed Mar 13 14:01:33 2024 -0400", "HASH" : "c7af9f730e2b519880efe4be912469fa3bf7276a13a8dc84f2b4b6d4abad2ac7", "INJECTOR_COMMIT_ID" : "e96746336a5dd6618d4a8d5eae7a5542d048f301", "ORIGIN_URI" : "[email protected]:crashappsec/chalk.git", "PLATFORM_WHEN_CHALKED" : "GNU/Linux x86_64", "METADATA_ID" : "2BNPPM-H38H-SWDQ-19XTT8" }
diff --git a/src/chalk.nim b/src/chalk.nim
@@ -15,22 +15,24 @@ import "."/[config, confload, commands, norecurse, sinks,
             attestation_api, util, autocomplete]
 
 when isMainModule:
-  setupSignalHandlers() # util.nim
-  setupTerminal()       # util.nim
-  ioSetup()             # sinks.nim
-  loadAllConfigs()      # confload.nim
-  recursionCheck()      # norecurse.nim
-  otherSetupTasks()     # util.nim
-  setupAutocomplete()   # autocomplete.nim
+  setupSignalHandlers()    # util.nim
+  setupTerminal()          # util.nim
+  ioSetup()                # sinks.nim
+  loadAllConfigs()         # confload.nim
+  recursionCheck()         # norecurse.nim
+  otherSetupTasks()        # util.nim
+
   # Wait for this warning until after configs load.
   if not canSelfInject:
     warn("No working codec is available for the native executable type")
 
   if passedHelpFlag:
     runChalkHelp(getCommandName()) # no return; in cmd_help.nim
 
+  setupAutocomplete()      # autocomplete.nim
   setupDefaultLogConfigs() # src/sinks.nim
   loadAttestation()        # attestation.nim
+
   case getCommandName()    # config.nim
   of "extract":            runCmdExtract(attrGet[seq[string]]("artifact_search_path"))
   of "extract.containers": runCmdExtractContainers()

diff --git a/src/chalk_common.nim b/src/chalk_common.nim
@@ -32,9 +32,6 @@ type
     collectedData*: ChalkDict        ## What we're adding during insertion.
     extract*:       ChalkDict        ## What we extracted, or nil if no extract.
     cachedMark*:    string           ## Cached chalk mark.
-    commentPrefix*: string           ## For scripting languages only, the comment
-                                     ## prefix we use when adding / rming marks
-    detectedLang*:  string           ## Currently only used in codecSource.
     opFailed*:      bool
     marked*:        bool
     embeds*:        seq[ChalkObj]
@@ -63,11 +60,10 @@ type
     repos*:         OrderedTableRef[string, DockerImageRepo] ## all images where image was tagged/pushed
     imageId*:       string           ## Image ID if this is a docker image
     containerId*:   string           ## Container ID if this is a container
+    noAttestation*: bool             ## Whether to skip attestation for chalkmark
     noCosign*:      bool             ## When we know image is not in registry. skips validation
     signed*:        bool             ## True on the insert path once signed,
                                      ## and once we've seen an attestation otherwise
-    inspected*:     bool             ## True for images once inspected; we don't
-                                     ## need to inspect twice when we build + push.
     resourceType*:  set[ResourceType]
 
   PluginClearCb*       = proc (a: Plugin) {.cdecl.}

diff --git a/src/chalkjson.nim b/src/chalkjson.nim
@@ -17,7 +17,7 @@ when (NimMinor, NimPatch) >= (6, 14):
   {.warning[CastSizes]: off.}
 
 import std/[unicode, parseutils, algorithm]
-import "."/config
+import "."/[config, util]
 
 const
   jsonWSChars      = ['\x20', '\x0a', '\x0d', '\x09']
@@ -497,25 +497,20 @@ proc forcePrivateKeys() =
   forceChalkKeys(toForce)
 
 proc getChalkMark*(obj: ChalkObj): ChalkDict =
-  trace("Creating mark using template: " & attrGet[string](getOutputConfig() & ".mark_template"))
+  let templ = getMarkTemplate()
+  trace("Creating mark using template: " & templ)
 
   forcePrivateKeys()
 
-  let templ = getMarkTemplate()
-
   assert "CHALK_VERSION" in hostInfo or "CHALK_VERSION" in obj.collectedData
 
   result              = hostInfo.filterByTemplate(templ)
-  let artifactResults = obj.collectedData.filterByTemplate(templ)
-
-  for k, v in artifactResults:
-    result[k] = v
+  result.merge(obj.collectedData.filterByTemplate(templ))
 
 proc getChalkMarkAsStr*(obj: ChalkObj): string =
   if obj.cachedMark != "":
     trace("Chalk cachemark " & obj.cachedMark)
     return obj.cachedMark
-  trace("Converting Mark to JSON. Mark template is: " & getMarkTemplate())
   let mark = obj.getChalkMark()
   result = mark.toJson()
   obj.cachedMark = result

diff --git a/src/collect.nim b/src/collect.nim
@@ -9,9 +9,6 @@ import std/re
 import "./docker"/[scan]
 import "."/[config, plugin_api]
 
-proc isSystem*(p: Plugin): bool =
-  return p.name in ["system", "attestation", "metsys"]
-
 proc hasSubscribedKey(p: Plugin, keys: seq[string], dict: ChalkDict): bool =
   # Decides whether to run a given plugin... does it export any key we
   # are subscribed to, that hasn't already been provided?
@@ -75,16 +72,8 @@ proc registerOutconfKeys() =
   #
   # TODO: The config should hand us a list of keys to force.
   subscribedKeys["_VALIDATED"] = true
-
-  let outconf = getOutputConfig()
-
-  let markTemplate = attrGet[string](outconf & ".mark_template")
-  if markTemplate != "":
-    registerKeys("mark_template." & markTemplate)
-
-  let reportTemplate = attrGet[string](outconf & ".report_template")
-  if reportTemplate != "":
-    registerKeys("report_template." & reportTemplate)
+  registerKeys(getMarkTemplate())
+  registerKeys(getReportTemplate())
 
 proc collectChalkTimeHostInfo*() =
   if hostCollectionSuspended():
@@ -103,6 +92,7 @@ proc collectChalkTimeHostInfo*() =
 
       for k, v in dict:
         if not plugin.canWrite(k, attrGet[seq[string]]("plugin." & plugin.name & ".pre_run_keys")):
+          trace(plugin.name & ": cannot write " & k & ". skipping")
           continue
         if k notin hostInfo or
             k in attrGet[seq[string]]("plugin." & plugin.name & ".overrides") or
@@ -113,7 +103,7 @@ proc collectChalkTimeHostInfo*() =
             plugin.name & " threw an exception it didn't handle: " & getCurrentExceptionMsg())
       dumpExOnDebug()
 
-proc initCollection*() =
+proc initCollection*(collectHost = true) =
   ## Chalk commands that report call this to initialize the collection
   ## system.  It looks at any reports that are currently configured,
   ## and 'registers' the keys, so that we don't waste time trying to
@@ -123,7 +113,13 @@ proc initCollection*() =
 
   trace("Collecting host-level chalk-time data")
 
-  forceChalkKeys(["MAGIC", "CHALK_VERSION", "CHALK_ID", "METADATA_ID"])
+  forceChalkKeys([
+    "MAGIC",
+    "CHALK_VERSION",
+    "CHALK_ID",
+    "METADATA_ID",
+    "HASH",
+  ])
   registerOutconfKeys()
 
   # Next, register for any custom reports.
@@ -134,14 +130,9 @@ proc initCollection*() =
       let useWhen = useWhenOpt.get()
       if (getBaseCommandName() notin useWhen and "*" notin useWhen):
         continue
+    registerKeys(getReportTemplate(report))
 
-    let templNameOpt = attrGetOpt[string](report & ".report_template")
-    if templNameOpt.isSome():
-      let templName = templNameOpt.get()
-      if templName != "":
-        registerKeys("report_template." & templName)
-
-  if isChalkingOp():
+  if isChalkingOp() and collectHost:
       collectChalkTimeHostInfo()
 
 proc collectRunTimeArtifactInfo*(artifact: ChalkObj) =
@@ -195,9 +186,7 @@ proc collectChalkTimeArtifactInfo*(obj: ChalkObj, override = false) =
         trace("Filling in codec info")
         if "CHALK_ID" notin data:
           data["CHALK_ID"]      = pack(obj.callGetChalkId())
-        let preHashOpt = obj.callGetUnchalkedHash()
-        if preHashOpt.isSome():
-          data["HASH"]          = pack(preHashOpt.get())
+        data.setIfNeeded("HASH", obj.callGetUnchalkedHash())
         if obj.fsRef != "":
           data["PATH_WHEN_CHALKED"] = pack(resolvePath(obj.fsRef))
 
@@ -219,7 +208,9 @@ proc collectChalkTimeArtifactInfo*(obj: ChalkObj, override = false) =
           continue
 
         for k, v in dict:
-          if not plugin.canWrite(k, attrGet[seq[string]]("plugin." & plugin.name & ".pre_chalk_keys")): continue
+          if not plugin.canWrite(k, attrGet[seq[string]]("plugin." & plugin.name & ".pre_chalk_keys")):
+            trace(plugin.name & ": cannot write " & k & ". skipping")
+            continue
           if k notin obj.collectedData or
               k in attrGet[seq[string]]("plugin." & plugin.name & ".overrides") or
               plugin.isSystem() or
Original file line number	Diff line number	Diff line change
Expand Up		@@ -168,4 +168,3 @@ function _chalk_completions {
		}

		complete -F _chalk_completions chalk
		# { "MAGIC" : "dadfedabbadabbed", "CHALK_ID" : "C4TPCR-SSCH-GKJR-V46DGK", "CHALK_VERSION" : "0.3.3", "TIMESTAMP_WHEN_CHALKED" : 1710356599977, "DATETIME_WHEN_CHALKED" : "2024-03-13T15:03:18.901-04:00", "ARTIFACT_TYPE" : "bash", "AUTHOR" : "Miroslav Shubernetskiy <[email protected]> 1710352293 -0400", "BRANCH" : "ms", "CHALK_RAND" : "9f75af21a1bf7665", "CODE_OWNERS" : "* @viega\n", "COMMITTER" : "Miroslav Shubernetskiy <[email protected]> 1710352893 -0400", "COMMIT_ID" : "5f95367b955256bb92254a5bddaea6e5285a29f6", "COMMIT_MESSAGE" : "build: bumping con4m to include get(dict, \"key\")\n\nits used in connect.c4m", "COMMIT_SIGNED" : true, "DATE_AUTHORED" : "Wed Mar 13 13:51:33 2024 -0400", "DATE_COMMITTED" : "Wed Mar 13 14:01:33 2024 -0400", "HASH" : "a5fc9da9cd3a291f4758e8e19028efeb4ba984dda9e4db7e8762215521767f83", "INJECTOR_COMMIT_ID" : "e96746336a5dd6618d4a8d5eae7a5542d048f301", "ORIGIN_URI" : "[email protected]:crashappsec/chalk.git", "PLATFORM_WHEN_CHALKED" : "GNU/Linux x86_64", "METADATA_ID" : "04CB0D-M6D6-ZP5Q-XN6FHA" }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -146,4 +146,3 @@ function _chalk_completions {
		}

		complete -F _chalk_completions chalk
		# { "MAGIC" : "dadfedabbadabbed", "CHALK_ID" : "CCVP2S-HSCR-VK6C-3569H3", "CHALK_VERSION" : "0.3.3", "TIMESTAMP_WHEN_CHALKED" : 1710356599979, "DATETIME_WHEN_CHALKED" : "2024-03-13T15:03:18.901-04:00", "ARTIFACT_TYPE" : "bash", "AUTHOR" : "Miroslav Shubernetskiy <[email protected]> 1710352293 -0400", "BRANCH" : "ms", "CHALK_RAND" : "63b2eb8e1ecca0bc", "CODE_OWNERS" : "* @viega\n", "COMMITTER" : "Miroslav Shubernetskiy <[email protected]> 1710352893 -0400", "COMMIT_ID" : "5f95367b955256bb92254a5bddaea6e5285a29f6", "COMMIT_MESSAGE" : "build: bumping con4m to include get(dict, \"key\")\n\nits used in connect.c4m", "COMMIT_SIGNED" : true, "DATE_AUTHORED" : "Wed Mar 13 13:51:33 2024 -0400", "DATE_COMMITTED" : "Wed Mar 13 14:01:33 2024 -0400", "HASH" : "c7af9f730e2b519880efe4be912469fa3bf7276a13a8dc84f2b4b6d4abad2ac7", "INJECTOR_COMMIT_ID" : "e96746336a5dd6618d4a8d5eae7a5542d048f301", "ORIGIN_URI" : "[email protected]:crashappsec/chalk.git", "PLATFORM_WHEN_CHALKED" : "GNU/Linux x86_64", "METADATA_ID" : "2BNPPM-H38H-SWDQ-19XTT8" }