Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Performance improvement on Sw360 package creator #225

Closed
wants to merge 6 commits into from
Closed

Performance improvement on Sw360 package creator #225

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
065567e
Initial
Jan 10, 2025
5a2c505
Initial
Jan 10, 2025
4454d82
Initial
Jan 10, 2025
9560268
initial
Jan 10, 2025
1adcbf5
With helper paralle foreach
Jan 10, 2025
e50be28
Parallel execution when release already exists
Jan 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/LCT.Common/Constants/Dataconstant.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ public static class Dataconstant
public const string Cdx_JfrogRepoPath = "internal:siemens:clearing:jfrog-repo-path";
public const string Cdx_Siemensfilename = "internal:siemens:clearing:siemens:filename";
public const string Cdx_SiemensDirect = "internal:siemens:clearing:siemens:direct";
public const int MaxDegreeOfParallelism = 8;

public static Dictionary<string, string> PurlCheck()
{
Expand Down
15 changes: 15 additions & 0 deletions src/LCT.Common/ProcessAsyncHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using System.Diagnostics;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace LCT.Common
Expand Down Expand Up @@ -153,6 +154,20 @@ public static StringBuilder STDErrorHandler(Process process, List<Task> processT

}


public static async Task ProcessItemsAsync<T>(IEnumerable<T> items, Func<T, CancellationToken, ValueTask> action, ParallelOptions parallelOptions = null, CancellationToken cancellationToken = default)
{
if (parallelOptions == null)
{
parallelOptions = new ParallelOptions();
}

await Parallel.ForEachAsync(items, parallelOptions, async (item, ct) =>
{
await action(item, ct);
});
}

}

}
123 changes: 82 additions & 41 deletions src/LCT.SW360PackageCreator/ComponentCreator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@
// --------------------------------------------------------------------------------------------------------------------

using CycloneDX.Models;
using JetBrains.Annotations;
using LCT.APICommunications;
using LCT.APICommunications.Model;
using LCT.APICommunications.Model.Foss;
using LCT.Common;
using LCT.Common.Constants;
using LCT.Common.Interface;
using LCT.Common.Model;
using LCT.Services;
using LCT.Services.Interface;
using LCT.Services.Model;
using LCT.SW360PackageCreator.Interfaces;
Expand All @@ -23,8 +25,10 @@
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;


namespace LCT.SW360PackageCreator
{
/// <summary>
Expand Down Expand Up @@ -57,51 +61,65 @@ public async Task<List<ComparisonBomData>> CycloneDxBomParser(CommonAppSettings
private async Task<List<Components>> GetListOfBomData(List<Component> components, CommonAppSettings appSettings)
{
List<Components> lstOfBomDataToBeCompared = new List<Components>();

foreach (Component item in components)
Func<Component, CancellationToken, ValueTask> action = async (item, ct) =>
{
Components componentsData = new Components();

string currName = item.Name;
string currVersion = item.Version;

bool isInternalComponent = GetPackageType(item, ref componentsData);

if (isInternalComponent)
{
Logger.Debug($"{item.Name}-{item.Version} found as internal component. ");
}
else if (componentsData.IsDev == "true" && appSettings.RemoveDevDependency)
{
//do nothing
}
else
try
{
componentsData.DownloadUrl = Dataconstant.DownloadUrlNotFound;
componentsData.Name = GetPackageName(item);
componentsData.Group = item.Group;
componentsData.Version = item.Version;
componentsData.ComponentExternalId = item.Purl.Substring(0, item.Purl.IndexOf('@'));
componentsData.ReleaseExternalId = item.Purl;
Components componentsData = new Components();
string currName = item.Name;
string currVersion = item.Version;

Components component = await GetSourceUrl(componentsData.Name, componentsData.Version, componentsData.ProjectType, item.BomRef);
componentsData.SourceUrl = component.SourceUrl;
bool isInternalComponent = GetPackageType(item, ref componentsData);

if (componentsData.ProjectType.ToUpperInvariant() == "ALPINE")
if (isInternalComponent)
{
componentsData.AlpineSourceData = component.AlpineSourceData;
Logger.Debug($"{item.Name}-{item.Version} found as internal component. ");
}

if (componentsData.ProjectType.ToUpperInvariant() == "DEBIAN")
else if (componentsData.IsDev == "true" && appSettings.RemoveDevDependency)
{
componentsData = component;
//do nothing
}
UpdateToLocalBomFile(componentsData, currName, currVersion);
else
{
componentsData.DownloadUrl = Dataconstant.DownloadUrlNotFound;
componentsData.Name = GetPackageName(item);
componentsData.Group = item.Group;
componentsData.Version = item.Version;
componentsData.ComponentExternalId = item.Purl.Substring(0, item.Purl.IndexOf('@'));
componentsData.ReleaseExternalId = item.Purl;

//Currently only Alpine and Debian source code is downloading intially for all the packages,
//rest of the packages will be downloaded on demand
Components component = new Components();
if (componentsData.ProjectType.ToUpperInvariant() == "ALPINE")
{
component = await GetSourceUrl(componentsData.Name, componentsData.Version, componentsData.ProjectType, item.BomRef);
componentsData.AlpineSourceData = component.AlpineSourceData;
}

if (componentsData.ProjectType.ToUpperInvariant() == "DEBIAN")
{
component = await GetSourceUrl(componentsData.Name, componentsData.Version, componentsData.ProjectType, item.BomRef);
componentsData = component;
}
UpdateToLocalBomFile(componentsData, currName, currVersion);

lstOfBomDataToBeCompared.Add(componentsData);
lstOfBomDataToBeCompared.Add(componentsData);
}
}
}
catch (Exception ex)
{
Logger.Error($"Error processing item {item.Name}: {ex.Message}", ex);
}
};


var parallelOptions = new ParallelOptions
{
MaxDegreeOfParallelism = Dataconstant.MaxDegreeOfParallelism
};

await ProcessAsyncHelper.ProcessItemsAsync(components, action, parallelOptions);
return lstOfBomDataToBeCompared;
}

Expand Down Expand Up @@ -240,7 +258,7 @@ public async Task CreateComponentInSw360(CommonAppSettings appSettings,
bom = await creatorHelper.GetUpdatedComponentsDetails(ListofBomComponents, UpdatedCompareBomData, sw360Service, bom);

var formattedString = CycloneDX.Json.Serializer.Serialize(bom);

fileOperations.WriteContentToOutputBomFile(formattedString, bomGenerationPath,
FileConstant.BomFileName, appSettings.SW360ProjectName);

Expand All @@ -265,7 +283,7 @@ public async Task CreateComponentInSw360(CommonAppSettings appSettings,

Logger.Debug($"CreateComponentInSw360():End");
}

private async Task CreateComponent(ICreatorHelper creatorHelper,
ISw360CreatorService sw360CreatorService, List<ComparisonBomData> componentsToBoms,
string sw360Url, CommonAppSettings appSettings)
Expand All @@ -274,11 +292,36 @@ private async Task CreateComponent(ICreatorHelper creatorHelper,

try
{
foreach (ComparisonBomData item in componentsToBoms)
List<ComparisonBomData> comparisonBomsForMixed = componentsToBoms.Where(avl => (avl.ComponentStatus == Dataconstant.Available && avl.ReleaseStatus == Dataconstant.NotAvailable)
|| (avl.ComponentStatus == Dataconstant.NotAvailable && avl.ReleaseStatus == Dataconstant.NotAvailable)).ToList();
// Getting the list of components & releases are avilable
List<ComparisonBomData> comparisonBomsForAvilables = componentsToBoms.Where(avl=> avl.ComponentStatus == Dataconstant.Available && avl.ReleaseStatus == Dataconstant.Available).ToList();

foreach (ComparisonBomData item in comparisonBomsForMixed)
{
await CreateComponentAndRealease(creatorHelper, sw360CreatorService, item, sw360Url, appSettings);
}

//Processing parallely for existing components
Func<ComparisonBomData, CancellationToken, ValueTask> action = async (item, ct) =>
{
try
{
await ComponentAndReleaseAvailable(item, sw360Url, sw360CreatorService, appSettings);
}
catch (Exception ex)
{
Logger.Error($"CreateComponent() : Error processing item {item.Name}: {ex.Message}", ex);
}
};

var parallelOptions = new ParallelOptions
{
MaxDegreeOfParallelism = Dataconstant.MaxDegreeOfParallelism
};
await ProcessAsyncHelper.ProcessItemsAsync(comparisonBomsForAvilables, action, parallelOptions);


if (appSettings.ProjectType.ToUpperInvariant() == "ALPINE")
{
string localPathforSourceRepo = UrlHelper.GetDownloadPathForAlpineRepo();
Expand Down Expand Up @@ -338,8 +381,6 @@ private async Task CreateComponentAndRealease(ICreatorHelper creatorHelper,
await CreateComponentAndReleaseWhenNotAvailable(item, sw360CreatorService, creatorHelper, appSettings);

await CreateReleaseWhenNotAvailable(item, sw360CreatorService, creatorHelper, appSettings);

await ComponentAndReleaseAvailable(item, sw360Url, sw360CreatorService, appSettings);
}

private async Task CreateComponentAndReleaseWhenNotAvailable(ComparisonBomData item,
Expand Down Expand Up @@ -486,7 +527,7 @@ public static async Task<string> CheckFossologyProcessStatus(string link, ISw360
}
catch (AggregateException ex)
{
Logger.DebugFormat("\tError in TriggerFossologyProcess--{0}",ex);
Logger.DebugFormat("\tError in TriggerFossologyProcess--{0}", ex);
}
return uploadId;
}
Expand Down Expand Up @@ -561,7 +602,7 @@ public void AddReleaseIdToLink(ComparisonBomData item, string releaseIdToLink)
ReleasesFoundInCbom.Add(new ReleaseLinked() { Name = item.Name, Version = item.Version, ReleaseId = releaseIdToLink });
}
else
{
{
Environment.ExitCode = -1;
Logger.Fatal($"Linking release to the project is failed. " +
$"Release version - {item.Version} not found under this component - {item.Name}. ");
Expand Down
Loading
Loading