Improved compile speed by running multi-threaded library discovery. (#2625)

* Simplified error reporting in library detection

There is no need to duplicate the preprocessResult/Err variables.
This also simplifies naming making it more straighforward.

* Remove useless targetFilePath variable

* Slight improvement of removeBuildFromSketchFiles

* Rename variables for clarity

* Removed hardcoded build.warn_data_percentage in build.options file

Also fixed the "low memory" warning printer.

* Renamed variables for clarity

* Renamed variables for clarity

* Pre-compute sourceFile fields, and save the in the includes.cache

* Added ObjFileIsUpToDate method to sourceFile

* Implemented parallel task runner

* Simplify use of properties.SplitQuotedString

The new release of the library allow ignoring the returned error.

arduino/go-properties-orderedmap#42

* Use runner.Task in GCC preprocessor

It slightly simplifies code, but also provide the basis for the next
commits.

* Parallelize library discovery phase in compile

* The number of jobs in library detection now follows --jobs flag

* Reordered properties construction for clarity

* Reordered compileFileWithRecipe for clarity

* Added integration test

* fix: libraries are recompiled if the list of include paths changes

Author: cmaglie

commands/service_compile.go

@@ -169,7 +169,7 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 	if buildPathArg := req.GetBuildPath(); buildPathArg != "" {
 		buildPath = paths.New(req.GetBuildPath()).Canonical()
 		if in, _ := buildPath.IsInsideDir(sk.FullPath); in && buildPath.IsDir() {
-			if sk.AdditionalFiles, err = removeBuildFromSketchFiles(sk.AdditionalFiles, buildPath); err != nil {
+			if sk.AdditionalFiles, err = removeBuildPathFromSketchFiles(sk.AdditionalFiles, buildPath); err != nil {
 				return err
 			}
 		}
@@ -220,10 +220,6 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 		return err
 	}
 
-	actualPlatform := buildPlatform
-	otherLibrariesDirs := paths.NewPathList(req.GetLibraries()...)
-	otherLibrariesDirs.Add(s.settings.LibrariesDir())
-
 	var libsManager *librariesmanager.LibrariesManager
 	if profile != nil {
 		libsManager = lm
@@ -253,6 +249,11 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 	if req.GetVerbose() {
 		verbosity = logger.VerbosityVerbose
 	}
+
+	librariesDirs := paths.NewPathList(req.GetLibraries()...) // Array of collection of libraries directories
+	librariesDirs.Add(s.settings.LibrariesDir())
+	libraryDirs := paths.NewPathList(req.GetLibrary()...) // Array of single-library directories
+
 	sketchBuilder, err := builder.NewBuilder(
 		ctx,
 		sk,
@@ -264,16 +265,16 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 		int(req.GetJobs()),
 		req.GetBuildProperties(),
 		s.settings.HardwareDirectories(),
-		otherLibrariesDirs,
+		librariesDirs,
 		s.settings.IDEBuiltinLibrariesDir(),
 		fqbn,
 		req.GetClean(),
 		req.GetSourceOverride(),
 		req.GetCreateCompilationDatabaseOnly(),
-		targetPlatform, actualPlatform,
+		targetPlatform, buildPlatform,
 		req.GetSkipLibrariesDiscovery(),
 		libsManager,
-		paths.NewPathList(req.GetLibrary()...),
+		libraryDirs,
 		outStream, errStream, verbosity, req.GetWarnings(),
 		progressCB,
 		pme.GetEnvVarsForSpawnedProcess(),
@@ -462,15 +463,15 @@ func maybePurgeBuildCache(compilationsBeforePurge uint, cacheTTL time.Duration)
 	buildcache.New(paths.TempDir().Join("arduino", "sketches")).Purge(cacheTTL)
 }
 
-// removeBuildFromSketchFiles removes the files contained in the build directory from
+// removeBuildPathFromSketchFiles removes the files contained in the build directory from
 // the list of the sketch files
-func removeBuildFromSketchFiles(files paths.PathList, build *paths.Path) (paths.PathList, error) {
+func removeBuildPathFromSketchFiles(files paths.PathList, build *paths.Path) (paths.PathList, error) {
 	var res paths.PathList
 	ignored := false
 	for _, file := range files {
 		if isInside, _ := file.IsInsideDir(build); !isInside {
-			res = append(res, file)
-		} else if !ignored {
+			res.Add(file)
+		} else {
 			ignored = true
 		}
 	}

commands/service_monitor.go

@@ -27,7 +27,6 @@ import (
 	"github.com/arduino/arduino-cli/internal/arduino/cores"
 	"github.com/arduino/arduino-cli/internal/arduino/cores/packagemanager"
 	pluggableMonitor "github.com/arduino/arduino-cli/internal/arduino/monitor"
-	"github.com/arduino/arduino-cli/internal/i18n"
 	"github.com/arduino/arduino-cli/pkg/fqbn"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	"github.com/arduino/go-properties-orderedmap"
@@ -265,10 +264,7 @@ func findMonitorAndSettingsForProtocolAndBoard(pme *packagemanager.Explorer, pro
 		} else if recipe, ok := boardPlatform.MonitorsDevRecipes[protocol]; ok {
 			// If we have a recipe we must resolve it
 			cmdLine := boardProperties.ExpandPropsInString(recipe)
-			cmdArgs, err := properties.SplitQuotedString(cmdLine, `"'`, false)
-			if err != nil {
-				return nil, nil, &cmderrors.InvalidArgumentError{Message: i18n.Tr("Invalid recipe in platform.txt"), Cause: err}
-			}
+			cmdArgs, _ := properties.SplitQuotedString(cmdLine, `"'`, false)
 			id := fmt.Sprintf("%s-%s", boardPlatform, protocol)
 			return pluggableMonitor.New(id, cmdArgs...), boardSettings, nil
 		}

commands/service_upload.go

@@ -720,10 +720,7 @@ func runTool(ctx context.Context, recipeID string, props *properties.Map, outStr
 		return errors.New(i18n.Tr("no upload port provided"))
 	}
 	cmdLine := props.ExpandPropsInString(recipe)
-	cmdArgs, err := properties.SplitQuotedString(cmdLine, `"'`, false)
-	if err != nil {
-		return errors.New(i18n.Tr("invalid recipe '%[1]s': %[2]s", recipe, err))
-	}
+	cmdArgs, _ := properties.SplitQuotedString(cmdLine, `"'`, false)
 
 	// Run Tool
 	logrus.WithField("phase", "upload").Tracef("Executing upload tool: %s", cmdLine)

internal/arduino/builder/builder.go

@@ -59,9 +59,6 @@ type Builder struct {
 	// Parallel processes
 	jobs int
 
-	// Custom build properties defined by user (line by line as "key=value" pairs)
-	customBuildProperties []string
-
 	// core related
 	coreBuildCachePath       *paths.Path
 	extraCoreBuildCachePaths paths.PathList
@@ -89,7 +86,7 @@ type Builder struct {
 	lineOffset int
 
 	targetPlatform *cores.PlatformRelease
-	actualPlatform *cores.PlatformRelease
+	buildPlatform  *cores.PlatformRelease
 
 	buildArtifacts *buildArtifacts
 
@@ -125,19 +122,20 @@ func NewBuilder(
 	coreBuildCachePath *paths.Path,
 	extraCoreBuildCachePaths paths.PathList,
 	jobs int,
-	requestBuildProperties []string,
-	hardwareDirs, otherLibrariesDirs paths.PathList,
+	customBuildProperties []string,
+	hardwareDirs paths.PathList,
+	librariesDirs paths.PathList,
 	builtInLibrariesDirs *paths.Path,
 	fqbn *fqbn.FQBN,
 	clean bool,
 	sourceOverrides map[string]string,
 	onlyUpdateCompilationDatabase bool,
-	targetPlatform, actualPlatform *cores.PlatformRelease,
+	targetPlatform, buildPlatform *cores.PlatformRelease,
 	useCachedLibrariesResolution bool,
 	librariesManager *librariesmanager.LibrariesManager,
-	libraryDirs paths.PathList,
+	customLibraryDirs paths.PathList,
 	stdout, stderr io.Writer, verbosity logger.Verbosity, warningsLevel string,
-	progresCB rpc.TaskProgressCB,
+	progressCB rpc.TaskProgressCB,
 	toolEnv []string,
 ) (*Builder, error) {
 	buildProperties := properties.NewMap()
@@ -146,14 +144,12 @@ func NewBuilder(
 	}
 	if sk != nil {
 		buildProperties.SetPath("sketch_path", sk.FullPath)
+		buildProperties.Set("build.project_name", sk.MainFile.Base())
+		buildProperties.SetPath("build.source.path", sk.FullPath)
 	}
 	if buildPath != nil {
 		buildProperties.SetPath("build.path", buildPath)
 	}
-	if sk != nil {
-		buildProperties.Set("build.project_name", sk.MainFile.Base())
-		buildProperties.SetPath("build.source.path", sk.FullPath)
-	}
 	if optimizeForDebug {
 		if debugFlags, ok := buildProperties.GetOk("compiler.optimization_flags.debug"); ok {
 			buildProperties.Set("compiler.optimization_flags", debugFlags)
@@ -165,12 +161,11 @@ func NewBuilder(
 	}
 
 	// Add user provided custom build properties
-	customBuildProperties, err := properties.LoadFromSlice(requestBuildProperties)
-	if err != nil {
+	if p, err := properties.LoadFromSlice(customBuildProperties); err == nil {
+		buildProperties.Merge(p)
+	} else {
 		return nil, fmt.Errorf("invalid build properties: %w", err)
 	}
-	buildProperties.Merge(customBuildProperties)
-	customBuildPropertiesArgs := append(requestBuildProperties, "build.warn_data_percentage=75")
 
 	sketchBuildPath, err := buildPath.Join("sketch").Abs()
 	if err != nil {
@@ -190,16 +185,20 @@ func NewBuilder(
 	}
 
 	log := logger.New(stdout, stderr, verbosity, warningsLevel)
-	libsManager, libsResolver, verboseOut, err := detector.LibrariesLoader(
-		useCachedLibrariesResolution, librariesManager,
-		builtInLibrariesDirs, libraryDirs, otherLibrariesDirs,
-		actualPlatform, targetPlatform,
+	libsManager, libsResolver, libsLoadingWarnings, err := detector.LibrariesLoader(
+		useCachedLibrariesResolution,
+		librariesManager,
+		builtInLibrariesDirs,
+		customLibraryDirs,
+		librariesDirs,
+		buildPlatform,
+		targetPlatform,
 	)
 	if err != nil {
 		return nil, err
 	}
 	if log.VerbosityLevel() == logger.VerbosityVerbose {
-		log.Warn(string(verboseOut))
+		log.Warn(string(libsLoadingWarnings))
 	}
 
 	diagnosticStore := diagnostics.NewStore()
@@ -212,25 +211,26 @@ func NewBuilder(
 		coreBuildPath:                 coreBuildPath,
 		librariesBuildPath:            librariesBuildPath,
 		jobs:                          jobs,
-		customBuildProperties:         customBuildPropertiesArgs,
 		coreBuildCachePath:            coreBuildCachePath,
 		extraCoreBuildCachePaths:      extraCoreBuildCachePaths,
 		logger:                        log,
 		clean:                         clean,
 		sourceOverrides:               sourceOverrides,
 		onlyUpdateCompilationDatabase: onlyUpdateCompilationDatabase,
 		compilationDatabase:           compilation.NewDatabase(buildPath.Join("compile_commands.json")),
-		Progress:                      progress.New(progresCB),
+		Progress:                      progress.New(progressCB),
 		executableSectionsSize:        []ExecutableSectionSize{},
 		buildArtifacts:                &buildArtifacts{},
 		targetPlatform:                targetPlatform,
-		actualPlatform:                actualPlatform,
+		buildPlatform:                 buildPlatform,
 		toolEnv:                       toolEnv,
 		buildOptions: newBuildOptions(
-			hardwareDirs, otherLibrariesDirs,
-			builtInLibrariesDirs, buildPath,
+			hardwareDirs,
+			librariesDirs,
+			builtInLibrariesDirs,
+			buildPath,
 			sk,
-			customBuildPropertiesArgs,
+			customBuildProperties,
 			fqbn,
 			clean,
 			buildProperties.Get("compiler.optimization_flags"),
@@ -322,10 +322,19 @@ func (b *Builder) preprocess() error {
 		b.librariesBuildPath,
 		b.buildProperties,
 		b.targetPlatform.Platform.Architecture,
+		b.jobs,
 	)
 	if err != nil {
 		return err
 	}
+	if b.libsDetector.IncludeFoldersChanged() && b.librariesBuildPath.Exist() {
+		if b.logger.VerbosityLevel() == logger.VerbosityVerbose {
+			b.logger.Info(i18n.Tr("The list of included libraries has been changed... rebuilding all libraries."))
+		}
+		if err := b.librariesBuildPath.RemoveAll(); err != nil {
+			return err
+		}
+	}
 	b.Progress.CompleteStep()
 
 	b.warnAboutArchIncompatibleLibraries(b.libsDetector.ImportedLibraries())
@@ -492,29 +501,26 @@ func (b *Builder) prepareCommandForRecipe(buildProperties *properties.Map, recip
 		commandLine = properties.DeleteUnexpandedPropsFromString(commandLine)
 	}
 
-	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
-	if err != nil {
-		return nil, err
-	}
+	args, _ := properties.SplitQuotedString(commandLine, `"'`, false)
 
 	// if the overall commandline is too long for the platform
 	// try reducing the length by making the filenames relative
 	// and changing working directory to build.path
 	var relativePath string
 	if len(commandLine) > 30000 {
 		relativePath = buildProperties.Get("build.path")
-		for i, arg := range parts {
+		for i, arg := range args {
 			if _, err := os.Stat(arg); os.IsNotExist(err) {
 				continue
 			}
 			rel, err := filepath.Rel(relativePath, arg)
 			if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) {
-				parts[i] = rel
+				args[i] = rel
 			}
 		}
 	}
 
-	command, err := paths.NewProcess(b.toolEnv, parts...)
+	command, err := paths.NewProcess(b.toolEnv, args...)
 	if err != nil {
 		return nil, err
 	}