Skip to content

Commit e5dca16

Browse files
fadnavistanmayBrian Barthel
fadnavistanmay
and
Brian Barthel
authored
Enable Domains for Pipeline Artifact (#4460)
* Initial pass at adding domain support for Pipeline Artifacts * Change missed domainId and some refactor * It works * Saving company's money by using loc string --------- Co-authored-by: Brian Barthel <[email protected]>
1 parent c313e3e commit e5dca16

File tree

5 files changed

+236
-153
lines changed

5 files changed

+236
-153
lines changed

‎src/Agent.Plugins/Artifact/PipelineArtifactConstants.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
namespace Agent.Plugins
55
{
6+
// Use PipelineArtifactContants.cs from ADO, once the latest libs are available.
67
public class PipelineArtifactConstants
78
{
89
public const string AzurePipelinesAgent = "AzurePipelinesAgent";
@@ -18,5 +19,6 @@ public class PipelineArtifactConstants
1819
public const string FileShareArtifact = "filepath";
1920
public const string CustomPropertiesPrefix = "user-";
2021
public const string HashType = "HashType";
22+
public const string DomainId = "DomainId";
2123
}
2224
}

‎src/Agent.Plugins/Artifact/PipelineArtifactProvider.cs‎

Lines changed: 81 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
using Microsoft.VisualStudio.Services.WebApi;
1616
using Microsoft.VisualStudio.Services.Content.Common;
1717
using Microsoft.VisualStudio.Services.BlobStore.Common;
18+
using Microsoft.VisualStudio.Services.BlobStore.Common.Telemetry;
1819

1920
namespace Agent.Plugins
2021
{
@@ -37,12 +38,19 @@ public async Task DownloadSingleArtifactAsync(
3738
CancellationToken cancellationToken,
3839
AgentTaskPluginExecutionContext context)
3940
{
41+
// if properties doesn't have it, use the default domain for backward compatibility
42+
IDomainId domainId = WellKnownDomainIds.DefaultDomainId;
43+
if(buildArtifact.Resource.Properties.TryGetValue(PipelineArtifactConstants.DomainId, out string domainIdString))
44+
{
45+
domainId = DomainIdFactory.Create(domainIdString);
46+
}
47+
4048
var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
4149
this.context.IsSystemDebugTrue(),
4250
(str) => this.context.Output(str),
4351
this.connection,
4452
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
45-
WellKnownDomainIds.DefaultDomainId,
53+
domainId,
4654
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
4755
context,
4856
cancellationToken);
@@ -85,49 +93,81 @@ public async Task DownloadMultipleArtifactsAsync(
8593
CancellationToken cancellationToken,
8694
AgentTaskPluginExecutionContext context)
8795
{
88-
var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
89-
this.context.IsSystemDebugTrue(),
90-
(str) => this.context.Output(str),
91-
this.connection,
92-
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
93-
WellKnownDomainIds.DefaultDomainId,
94-
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
95-
context,
96-
cancellationToken);
96+
// create clients and group artifacts for each domain:
97+
Dictionary<IDomainId, (DedupManifestArtifactClient Client, BlobStoreClientTelemetry Telemetry, Dictionary<string, DedupIdentifier> ArtifactDictionary)> dedupManifestClients =
98+
new();
9799

98-
using (clientTelemetry)
99-
{
100-
var artifactNameAndManifestIds = buildArtifacts.ToDictionary(
101-
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
102-
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
103-
// 2) download to the target path
104-
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
105-
artifactNameAndManifestIds,
106-
downloadParameters.TargetDirectory,
107-
proxyUri: null,
108-
minimatchPatterns: downloadParameters.MinimatchFilters,
109-
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName);
100+
foreach(var buildArtifact in buildArtifacts)
101+
{
102+
// if properties doesn't have it, use the default domain for backward compatibility
103+
IDomainId domainId = WellKnownDomainIds.DefaultDomainId;
104+
if(buildArtifact.Resource.Properties.TryGetValue(PipelineArtifactConstants.DomainId, out string domainIdString))
105+
{
106+
domainId = DomainIdFactory.Create(domainIdString);
107+
}
110108

111-
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
112-
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
113-
await clientTelemetry.MeasureActionAsync(
114-
record: downloadRecord,
115-
actionAsync: async () =>
109+
// Have we already created the clients for this domain?
110+
if(dedupManifestClients.ContainsKey(domainId)) {
111+
// Clients already created for this domain, Just add the artifact to the list:
112+
dedupManifestClients[domainId].ArtifactDictionary.Add(buildArtifact.Name, DedupIdentifier.Create(buildArtifact.Resource.Data));
113+
}
114+
else
115+
{
116+
// create the clients:
117+
var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
118+
this.context.IsSystemDebugTrue(),
119+
(str) => this.context.Output(str),
120+
this.connection,
121+
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
122+
domainId,
123+
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
124+
context,
125+
cancellationToken);
126+
127+
// and create the artifact dictionary with the current artifact
128+
var artifactDictionary = new Dictionary<string, DedupIdentifier>
116129
{
117-
await AsyncHttpRetryHelper.InvokeVoidAsync(
118-
async () =>
119-
{
120-
await dedupManifestClient.DownloadAsync(options, cancellationToken);
121-
},
122-
maxRetries: 3,
123-
tracer: tracer,
124-
canRetryDelegate: e => true,
125-
context: nameof(DownloadMultipleArtifactsAsync),
126-
cancellationToken: cancellationToken,
127-
continueOnCapturedContext: false);
128-
});
129-
// Send results to CustomerIntelligence
130-
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
130+
{ buildArtifact.Name, DedupIdentifier.Create(buildArtifact.Resource.Data) }
131+
};
132+
133+
dedupManifestClients.Add(domainId, (dedupManifestClient, clientTelemetry, artifactDictionary));
134+
}
135+
}
136+
137+
foreach(var clientInfo in dedupManifestClients.Values)
138+
{
139+
using (clientInfo.Telemetry)
140+
{
141+
// 2) download to the target path
142+
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
143+
clientInfo.ArtifactDictionary,
144+
downloadParameters.TargetDirectory,
145+
proxyUri: null,
146+
minimatchPatterns: downloadParameters.MinimatchFilters,
147+
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName);
148+
149+
PipelineArtifactActionRecord downloadRecord = clientInfo.Telemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
150+
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
151+
152+
await clientInfo.Telemetry.MeasureActionAsync(
153+
record: downloadRecord,
154+
actionAsync: async () =>
155+
{
156+
await AsyncHttpRetryHelper.InvokeVoidAsync(
157+
async () =>
158+
{
159+
await clientInfo.Client.DownloadAsync(options, cancellationToken);
160+
},
161+
maxRetries: 3,
162+
tracer: tracer,
163+
canRetryDelegate: e => true,
164+
context: nameof(DownloadMultipleArtifactsAsync),
165+
cancellationToken: cancellationToken,
166+
continueOnCapturedContext: false);
167+
});
168+
// Send results to CustomerIntelligence
169+
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
170+
}
131171
}
132172
}
133173
}

‎src/Agent.Plugins/Artifact/PipelineArtifactServer.cs‎

Lines changed: 20 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,26 @@ internal async Task UploadAsync(
4141
IDictionary<string, string> properties,
4242
CancellationToken cancellationToken)
4343
{
44+
// Get the client settings, if any.
45+
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose: false, (str) => context.Output(str));
4446
VssConnection connection = context.VssConnection;
45-
var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
46-
.CreateDedupManifestClientAsync(
47+
var clientSettings = await DedupManifestArtifactClientFactory.GetClientSettingsAsync(
48+
connection,
49+
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
50+
tracer,
51+
cancellationToken);
52+
53+
// Get the default domain to use:
54+
IDomainId domainId = DedupManifestArtifactClientFactory.GetDefaultDomainId(clientSettings, tracer);
55+
56+
var (dedupManifestClient, clientTelemetry) = DedupManifestArtifactClientFactory.Instance
57+
.CreateDedupManifestClient(
4758
context.IsSystemDebugTrue(),
4859
(str) => context.Output(str),
4960
connection,
5061
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
51-
WellKnownDomainIds.DefaultDomainId,
52-
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
62+
domainId,
63+
clientSettings,
5364
context,
5465
cancellationToken);
5566

@@ -84,7 +95,8 @@ internal async Task UploadAsync(
8495
{ PipelineArtifactConstants.RootId, result.RootId.ValueString },
8596
{ PipelineArtifactConstants.ProofNodes, StringUtil.ConvertToJson(result.ProofNodes.ToArray()) },
8697
{ PipelineArtifactConstants.ArtifactSize, result.ContentSize.ToString() },
87-
{ PipelineArtifactConstants.HashType, dedupManifestClient.HashType.Serialize() }
98+
{ PipelineArtifactConstants.HashType, dedupManifestClient.HashType.Serialize() },
99+
{ PipelineArtifactConstants.DomainId, domainId.Serialize() }
88100
};
89101

90102
BuildArtifact buildArtifact = await AsyncHttpRetryHelper.InvokeAsync(
@@ -140,22 +152,11 @@ internal async Task DownloadAsync(
140152
CancellationToken cancellationToken)
141153
{
142154
VssConnection connection = context.VssConnection;
143-
var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
144-
.CreateDedupManifestClientAsync(
145-
context.IsSystemDebugTrue(),
146-
(str) => context.Output(str),
147-
connection,
148-
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
149-
WellKnownDomainIds.DefaultDomainId,
150-
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
151-
context,
152-
cancellationToken);
155+
PipelineArtifactProvider provider = new PipelineArtifactProvider(context, connection, tracer);
153156

154157
BuildServer buildServer = new(connection);
155158

156-
using (clientTelemetry)
157159
// download all pipeline artifacts if artifact name is missing
158-
{
159160
if (downloadOptions == DownloadOptions.MultiDownload)
160161
{
161162
List<BuildArtifact> artifacts;
@@ -187,40 +188,7 @@ internal async Task DownloadAsync(
187188
else
188189
{
189190
context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count()));
190-
191-
var artifactNameAndManifestIds = pipelineArtifacts.ToDictionary(
192-
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
193-
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
194-
// 2) download to the target path
195-
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
196-
artifactNameAndManifestIds,
197-
downloadParameters.TargetDirectory,
198-
proxyUri: null,
199-
minimatchPatterns: downloadParameters.MinimatchFilters,
200-
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName,
201-
customMinimatchOptions: downloadParameters.CustomMinimatchOptions);
202-
203-
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
204-
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
205-
await clientTelemetry.MeasureActionAsync(
206-
record: downloadRecord,
207-
actionAsync: async () =>
208-
{
209-
await AsyncHttpRetryHelper.InvokeVoidAsync(
210-
async () =>
211-
{
212-
await dedupManifestClient.DownloadAsync(options, cancellationToken);
213-
},
214-
maxRetries: 3,
215-
tracer: tracer,
216-
canRetryDelegate: e => true,
217-
context: nameof(DownloadAsync),
218-
cancellationToken: cancellationToken,
219-
continueOnCapturedContext: false);
220-
});
221-
222-
// Send results to CustomerIntelligence
223-
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
191+
await provider.DownloadMultipleArtifactsAsync(downloadParameters,artifacts, cancellationToken, context);
224192
}
225193
}
226194
else if (downloadOptions == DownloadOptions.SingleDownload)
@@ -246,42 +214,12 @@ await AsyncHttpRetryHelper.InvokeVoidAsync(
246214
{
247215
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
248216
}
249-
250-
var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
251-
var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
252-
manifestId,
253-
downloadParameters.TargetDirectory,
254-
proxyUri: null,
255-
minimatchPatterns: downloadParameters.MinimatchFilters,
256-
customMinimatchOptions: downloadParameters.CustomMinimatchOptions);
257-
258-
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
259-
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
260-
await clientTelemetry.MeasureActionAsync(
261-
record: downloadRecord,
262-
actionAsync: async () =>
263-
{
264-
await AsyncHttpRetryHelper.InvokeVoidAsync(
265-
async () =>
266-
{
267-
await dedupManifestClient.DownloadAsync(options, cancellationToken);
268-
},
269-
maxRetries: 3,
270-
tracer: tracer,
271-
canRetryDelegate: e => true,
272-
context: nameof(DownloadAsync),
273-
cancellationToken: cancellationToken,
274-
continueOnCapturedContext: false);
275-
});
276-
277-
// Send results to CustomerIntelligence
278-
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
217+
await provider.DownloadSingleArtifactAsync(downloadParameters, buildArtifact, cancellationToken, context);
279218
}
280219
else
281220
{
282221
throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!");
283222
}
284-
}
285223
}
286224

287225
// Download for version 2. This decision was made because version 1 is sealed and we didn't want to break any existing customers.

0 commit comments

Comments
 (0)