Skip to content

Commit 556d774

Browse files
vrutkovsvrutkovs
committed
kubelet: create top-level traces for pod sync and GC
This starts new top level OpenTelemetry spans every time syncPod or image / container GC is invoked
1 parent c6f3007 commit 556d774

9 files changed

+73
-15
lines changed

‎hack/verify-flags/excluded-flags.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,4 @@ file_content_in_loop
2424
break_on_expected_content
2525
Premium_LRS
2626
VCP_STRESS_ITERATIONS
27+
update_type

‎pkg/kubelet/images/image_gc_manager.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"sync"
2626
"time"
2727

28+
"go.opentelemetry.io/otel/trace"
2829
v1 "k8s.io/api/core/v1"
2930
"k8s.io/klog/v2"
3031

@@ -38,6 +39,9 @@ import (
3839
"k8s.io/kubernetes/pkg/kubelet/util/sliceutils"
3940
)
4041

42+
// instrumentationScope is OpenTelemetry instrumentation scope name
43+
const instrumentationScope = "k8s.io/kubernetes/pkg/kubelet/images"
44+
4145
// StatsProvider is an interface for fetching stats used during image garbage
4246
// collection.
4347
type StatsProvider interface {
@@ -104,6 +108,9 @@ type realImageGCManager struct {
104108

105109
// sandbox image exempted from GC
106110
sandboxImage string
111+
112+
// tracer for recording spans
113+
tracer trace.Tracer
107114
}
108115

109116
// imageCache caches latest result of ListImages.
@@ -153,7 +160,7 @@ type imageRecord struct {
153160
}
154161

155162
// NewImageGCManager instantiates a new ImageGCManager object.
156-
func NewImageGCManager(runtime container.Runtime, statsProvider StatsProvider, recorder record.EventRecorder, nodeRef *v1.ObjectReference, policy ImageGCPolicy, sandboxImage string) (ImageGCManager, error) {
163+
func NewImageGCManager(runtime container.Runtime, statsProvider StatsProvider, recorder record.EventRecorder, nodeRef *v1.ObjectReference, policy ImageGCPolicy, sandboxImage string, tracerProvider trace.TracerProvider) (ImageGCManager, error) {
157164
// Validate policy.
158165
if policy.HighThresholdPercent < 0 || policy.HighThresholdPercent > 100 {
159166
return nil, fmt.Errorf("invalid HighThresholdPercent %d, must be in range [0-100]", policy.HighThresholdPercent)
@@ -164,6 +171,7 @@ func NewImageGCManager(runtime container.Runtime, statsProvider StatsProvider, r
164171
if policy.LowThresholdPercent > policy.HighThresholdPercent {
165172
return nil, fmt.Errorf("LowThresholdPercent %d can not be higher than HighThresholdPercent %d", policy.LowThresholdPercent, policy.HighThresholdPercent)
166173
}
174+
tracer := tracerProvider.Tracer(instrumentationScope)
167175
im := &realImageGCManager{
168176
runtime: runtime,
169177
policy: policy,
@@ -173,6 +181,7 @@ func NewImageGCManager(runtime container.Runtime, statsProvider StatsProvider, r
173181
nodeRef: nodeRef,
174182
initialized: false,
175183
sandboxImage: sandboxImage,
184+
tracer: tracer,
176185
}
177186

178187
return im, nil
@@ -279,6 +288,8 @@ func (im *realImageGCManager) detectImages(ctx context.Context, detectTime time.
279288
}
280289

281290
func (im *realImageGCManager) GarbageCollect(ctx context.Context) error {
291+
ctx, otelSpan := im.tracer.Start(ctx, "Images/GarbageCollect")
292+
defer otelSpan.End()
282293
// Get disk usage on disk holding images.
283294
fsStats, err := im.statsProvider.ImageFsStats(ctx)
284295
if err != nil {

‎pkg/kubelet/images/image_gc_manager_test.go

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"github.com/stretchr/testify/assert"
2828
"github.com/stretchr/testify/require"
2929

30+
oteltrace "go.opentelemetry.io/otel/trace"
3031
"k8s.io/client-go/tools/record"
3132
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
3233
"k8s.io/kubernetes/pkg/kubelet/container"
@@ -48,6 +49,7 @@ func newRealImageGCManager(policy ImageGCPolicy, mockStatsProvider stats.Provide
4849
statsProvider: mockStatsProvider,
4950
recorder: &record.FakeRecorder{},
5051
sandboxImage: sandboxImage,
52+
tracer: oteltrace.NewNoopTracerProvider().Tracer(""),
5153
}, fakeRuntime
5254
}
5355

@@ -543,7 +545,7 @@ func TestGarbageCollectBelowLowThreshold(t *testing.T) {
543545
manager, _ := newRealImageGCManager(policy, mockStatsProvider)
544546

545547
// Expect 40% usage.
546-
mockStatsProvider.EXPECT().ImageFsStats(ctx).Return(&statsapi.FsStats{
548+
mockStatsProvider.EXPECT().ImageFsStats(gomock.Any()).Return(&statsapi.FsStats{
547549
AvailableBytes: uint64Ptr(600),
548550
CapacityBytes: uint64Ptr(1000),
549551
}, nil)
@@ -562,7 +564,7 @@ func TestGarbageCollectCadvisorFailure(t *testing.T) {
562564
mockStatsProvider := statstest.NewMockProvider(mockCtrl)
563565
manager, _ := newRealImageGCManager(policy, mockStatsProvider)
564566

565-
mockStatsProvider.EXPECT().ImageFsStats(ctx).Return(&statsapi.FsStats{}, fmt.Errorf("error"))
567+
mockStatsProvider.EXPECT().ImageFsStats(gomock.Any()).Return(&statsapi.FsStats{}, fmt.Errorf("error"))
566568
assert.NotNil(t, manager.GarbageCollect(ctx))
567569
}
568570

@@ -579,7 +581,7 @@ func TestGarbageCollectBelowSuccess(t *testing.T) {
579581
manager, fakeRuntime := newRealImageGCManager(policy, mockStatsProvider)
580582

581583
// Expect 95% usage and most of it gets freed.
582-
mockStatsProvider.EXPECT().ImageFsStats(ctx).Return(&statsapi.FsStats{
584+
mockStatsProvider.EXPECT().ImageFsStats(gomock.Any()).Return(&statsapi.FsStats{
583585
AvailableBytes: uint64Ptr(50),
584586
CapacityBytes: uint64Ptr(1000),
585587
}, nil)
@@ -602,7 +604,7 @@ func TestGarbageCollectNotEnoughFreed(t *testing.T) {
602604
manager, fakeRuntime := newRealImageGCManager(policy, mockStatsProvider)
603605

604606
// Expect 95% usage and little of it gets freed.
605-
mockStatsProvider.EXPECT().ImageFsStats(ctx).Return(&statsapi.FsStats{
607+
mockStatsProvider.EXPECT().ImageFsStats(gomock.Any()).Return(&statsapi.FsStats{
606608
AvailableBytes: uint64Ptr(50),
607609
CapacityBytes: uint64Ptr(1000),
608610
}, nil)
@@ -717,7 +719,7 @@ func TestValidateImageGCPolicy(t *testing.T) {
717719
}
718720

719721
for _, tc := range testCases {
720-
if _, err := NewImageGCManager(nil, nil, nil, nil, tc.imageGCPolicy, ""); err != nil {
722+
if _, err := NewImageGCManager(nil, nil, nil, nil, tc.imageGCPolicy, "", oteltrace.NewNoopTracerProvider()); err != nil {
721723
if err.Error() != tc.expectErr {
722724
t.Errorf("[%s:]Expected err:%v, but got:%v", tc.name, tc.expectErr, err.Error())
723725
}

‎pkg/kubelet/kubelet.go

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import (
3737

3838
cadvisorapi "github.com/google/cadvisor/info/v1"
3939
libcontaineruserns "github.com/opencontainers/runc/libcontainer/userns"
40+
"go.opentelemetry.io/otel/attribute"
4041
"go.opentelemetry.io/otel/trace"
4142

4243
"k8s.io/mount-utils"
@@ -198,6 +199,9 @@ const (
198199

199200
// nodeLeaseRenewIntervalFraction is the fraction of lease duration to renew the lease
200201
nodeLeaseRenewIntervalFraction = 0.25
202+
203+
// instrumentationScope is the name of OpenTelemetry instrumentation scope
204+
instrumentationScope = "k8s.io/kubernetes/pkg/kubelet"
201205
)
202206

203207
var (
@@ -511,6 +515,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
511515
insecureContainerLifecycleHTTPClient.CheckRedirect = httpprobe.RedirectChecker(false)
512516
}
513517

518+
tracer := kubeDeps.TracerProvider.Tracer(instrumentationScope)
519+
514520
klet := &Kubelet{
515521
hostname: hostname,
516522
hostnameOverridden: hostnameOverridden,
@@ -561,6 +567,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
561567
experimentalHostUserNamespaceDefaulting: utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalHostUserNamespaceDefaultingGate),
562568
keepTerminatedPodVolumes: keepTerminatedPodVolumes,
563569
nodeStatusMaxImages: nodeStatusMaxImages,
570+
tracer: tracer,
564571
}
565572

566573
if klet.cloud != nil {
@@ -677,6 +684,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
677684
kubeDeps.ContainerManager.GetNodeAllocatableAbsolute,
678685
*kubeCfg.MemoryThrottlingFactor,
679686
kubeDeps.PodStartupLatencyTracker,
687+
kubeDeps.TracerProvider,
680688
)
681689
if err != nil {
682690
return nil, err
@@ -760,7 +768,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
760768
klet.containerDeletor = newPodContainerDeletor(klet.containerRuntime, integer.IntMax(containerGCPolicy.MaxPerPodContainer, minDeadContainerInPod))
761769

762770
// setup imageManager
763-
imageManager, err := images.NewImageGCManager(klet.containerRuntime, klet.StatsProvider, kubeDeps.Recorder, nodeRef, imageGCPolicy, crOptions.PodSandboxImage)
771+
imageManager, err := images.NewImageGCManager(klet.containerRuntime, klet.StatsProvider, kubeDeps.Recorder, nodeRef, imageGCPolicy, crOptions.PodSandboxImage, kubeDeps.TracerProvider)
764772
if err != nil {
765773
return nil, fmt.Errorf("failed to initialize image manager: %v", err)
766774
}
@@ -1252,6 +1260,9 @@ type Kubelet struct {
12521260

12531261
// Mutex to serialize new pod admission and existing pod resizing
12541262
podResizeMutex sync.Mutex
1263+
1264+
// OpenTelemetry Tracer
1265+
tracer trace.Tracer
12551266
}
12561267

12571268
// ListPodStats is delegated to StatsProvider, which implements stats.Provider interface
@@ -1612,10 +1623,17 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
16121623
func (kl *Kubelet) SyncPod(_ context.Context, updateType kubetypes.SyncPodType, pod, mirrorPod *v1.Pod, podStatus *kubecontainer.PodStatus) (isTerminal bool, err error) {
16131624
// TODO(#113606): connect this with the incoming context parameter, which comes from the pod worker.
16141625
// Currently, using that context causes test failures.
1615-
ctx := context.TODO()
1626+
ctx, otelSpan := kl.tracer.Start(context.TODO(), "syncPod", trace.WithAttributes(
1627+
attribute.String("k8s.pod.uid", string(pod.UID)),
1628+
attribute.String("k8s.pod", klog.KObj(pod).String()),
1629+
attribute.String("k8s.pod.name", pod.Name),
1630+
attribute.String("k8s.pod.update_type", updateType.String()),
1631+
attribute.String("k8s.namespace.name", pod.Namespace),
1632+
))
16161633
klog.V(4).InfoS("SyncPod enter", "pod", klog.KObj(pod), "podUID", pod.UID)
16171634
defer func() {
16181635
klog.V(4).InfoS("SyncPod exit", "pod", klog.KObj(pod), "podUID", pod.UID, "isTerminal", isTerminal)
1636+
otelSpan.End()
16191637
}()
16201638

16211639
// Latency measurements for the main workflow are relative to the
@@ -1882,7 +1900,13 @@ func (kl *Kubelet) SyncPod(_ context.Context, updateType kubetypes.SyncPodType,
18821900
func (kl *Kubelet) SyncTerminatingPod(_ context.Context, pod *v1.Pod, podStatus *kubecontainer.PodStatus, gracePeriod *int64, podStatusFn func(*v1.PodStatus)) error {
18831901
// TODO(#113606): connect this with the incoming context parameter, which comes from the pod worker.
18841902
// Currently, using that context causes test failures.
1885-
ctx := context.Background()
1903+
ctx, otelSpan := kl.tracer.Start(context.Background(), "syncTerminatingPod", trace.WithAttributes(
1904+
attribute.String("k8s.pod.uid", string(pod.UID)),
1905+
attribute.String("k8s.pod", klog.KObj(pod).String()),
1906+
attribute.String("k8s.pod.name", pod.Name),
1907+
attribute.String("k8s.namespace.name", pod.Namespace),
1908+
))
1909+
defer otelSpan.End()
18861910
klog.V(4).InfoS("SyncTerminatingPod enter", "pod", klog.KObj(pod), "podUID", pod.UID)
18871911
defer klog.V(4).InfoS("SyncTerminatingPod exit", "pod", klog.KObj(pod), "podUID", pod.UID)
18881912

@@ -2004,6 +2028,13 @@ func (kl *Kubelet) SyncTerminatingRuntimePod(_ context.Context, runningPod *kube
20042028
// This typically occurs when a pod is force deleted from configuration (local disk or API) and the
20052029
// kubelet restarts in the middle of the action.
20062030
func (kl *Kubelet) SyncTerminatedPod(ctx context.Context, pod *v1.Pod, podStatus *kubecontainer.PodStatus) error {
2031+
_, otelSpan := kl.tracer.Start(context.Background(), "syncTerminatedPod", trace.WithAttributes(
2032+
attribute.String("k8s.pod.uid", string(pod.UID)),
2033+
attribute.String("k8s.pod", klog.KObj(pod).String()),
2034+
attribute.String("k8s.pod.name", pod.Name),
2035+
attribute.String("k8s.namespace.name", pod.Namespace),
2036+
))
2037+
defer otelSpan.End()
20072038
klog.V(4).InfoS("SyncTerminatedPod enter", "pod", klog.KObj(pod), "podUID", pod.UID)
20082039
defer klog.V(4).InfoS("SyncTerminatedPod exit", "pod", klog.KObj(pod), "podUID", pod.UID)
20092040

‎pkg/kubelet/kubelet_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,7 @@ func newTestKubeletWithImageList(
253253
kubelet.cadvisor = &cadvisortest.Fake{}
254254
machineInfo, _ := kubelet.cadvisor.MachineInfo()
255255
kubelet.setCachedMachineInfo(machineInfo)
256+
kubelet.tracer = oteltrace.NewNoopTracerProvider().Tracer("")
256257

257258
fakeMirrorClient := podtest.NewFakeMirrorClient()
258259
secretManager := secret.NewSimpleSecretManager(kubelet.kubeClient)
@@ -305,7 +306,7 @@ func newTestKubeletWithImageList(
305306
HighThresholdPercent: 90,
306307
LowThresholdPercent: 80,
307308
}
308-
imageGCManager, err := images.NewImageGCManager(fakeRuntime, kubelet.StatsProvider, fakeRecorder, fakeNodeRef, fakeImageGCPolicy, "")
309+
imageGCManager, err := images.NewImageGCManager(fakeRuntime, kubelet.StatsProvider, fakeRecorder, fakeNodeRef, fakeImageGCPolicy, "", oteltrace.NewNoopTracerProvider())
309310
assert.NoError(t, err)
310311
kubelet.imageManager = &fakeImageGCManager{
311312
fakeImageService: fakeRuntime,

‎pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"time"
2323

2424
cadvisorapi "github.com/google/cadvisor/info/v1"
25+
"go.opentelemetry.io/otel/trace"
2526
v1 "k8s.io/api/core/v1"
2627
"k8s.io/apimachinery/pkg/api/resource"
2728
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -90,7 +91,7 @@ func (f *fakePodPullingTimeRecorder) RecordImageStartedPulling(podUID types.UID)
9091

9192
func (f *fakePodPullingTimeRecorder) RecordImageFinishedPulling(podUID types.UID) {}
9293

93-
func newFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageService internalapi.ImageManagerService, machineInfo *cadvisorapi.MachineInfo, osInterface kubecontainer.OSInterface, runtimeHelper kubecontainer.RuntimeHelper, keyring credentialprovider.DockerKeyring) (*kubeGenericRuntimeManager, error) {
94+
func newFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageService internalapi.ImageManagerService, machineInfo *cadvisorapi.MachineInfo, osInterface kubecontainer.OSInterface, runtimeHelper kubecontainer.RuntimeHelper, keyring credentialprovider.DockerKeyring, tracer trace.Tracer) (*kubeGenericRuntimeManager, error) {
9495
ctx := context.Background()
9596
recorder := &record.FakeRecorder{}
9697
logManager, err := logs.NewContainerLogManager(runtimeService, osInterface, "1", 2)
@@ -122,7 +123,7 @@ func newFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageS
122123
}
123124

124125
podStateProvider := newFakePodStateProvider()
125-
kubeRuntimeManager.containerGC = newContainerGC(runtimeService, podStateProvider, kubeRuntimeManager)
126+
kubeRuntimeManager.containerGC = newContainerGC(runtimeService, podStateProvider, kubeRuntimeManager, tracer)
126127
kubeRuntimeManager.podStateProvider = podStateProvider
127128
kubeRuntimeManager.runtimeName = typedVersion.RuntimeName
128129
kubeRuntimeManager.imagePuller = images.NewImageManager(

‎pkg/kubelet/kuberuntime/kuberuntime_gc.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"sort"
2525
"time"
2626

27+
"go.opentelemetry.io/otel/trace"
2728
"k8s.io/apimachinery/pkg/types"
2829
utilerrors "k8s.io/apimachinery/pkg/util/errors"
2930
"k8s.io/apimachinery/pkg/util/sets"
@@ -38,14 +39,16 @@ type containerGC struct {
3839
client internalapi.RuntimeService
3940
manager *kubeGenericRuntimeManager
4041
podStateProvider podStateProvider
42+
tracer trace.Tracer
4143
}
4244

4345
// NewContainerGC creates a new containerGC.
44-
func newContainerGC(client internalapi.RuntimeService, podStateProvider podStateProvider, manager *kubeGenericRuntimeManager) *containerGC {
46+
func newContainerGC(client internalapi.RuntimeService, podStateProvider podStateProvider, manager *kubeGenericRuntimeManager, tracer trace.Tracer) *containerGC {
4547
return &containerGC{
4648
client: client,
4749
manager: manager,
4850
podStateProvider: podStateProvider,
51+
tracer: tracer,
4952
}
5053
}
5154

@@ -407,6 +410,8 @@ func (cgc *containerGC) evictPodLogsDirectories(ctx context.Context, allSourcesR
407410
// * gets evictable sandboxes which are not ready and contains no containers.
408411
// * removes evictable sandboxes.
409412
func (cgc *containerGC) GarbageCollect(ctx context.Context, gcPolicy kubecontainer.GCPolicy, allSourcesReady bool, evictNonDeletedPods bool) error {
413+
ctx, otelSpan := cgc.tracer.Start(ctx, "Containers/GarbageCollect")
414+
defer otelSpan.End()
410415
errors := []error{}
411416
// Remove evictable containers
412417
if err := cgc.evictContainers(ctx, gcPolicy, allSourcesReady, evictNonDeletedPods); err != nil {

‎pkg/kubelet/kuberuntime/kuberuntime_manager.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"time"
2727

2828
cadvisorapi "github.com/google/cadvisor/info/v1"
29+
"go.opentelemetry.io/otel/trace"
2930
crierror "k8s.io/cri-api/pkg/errors"
3031
"k8s.io/klog/v2"
3132

@@ -77,6 +78,8 @@ const (
7778
versionCacheTTL = 60 * time.Second
7879
// How frequently to report identical errors
7980
identicalErrorDelay = 1 * time.Minute
81+
// OpenTelemetry instrumentation scope name
82+
instrumentationScope = "k8s.io/kubernetes/pkg/kubelet/kuberuntime"
8083
)
8184

8285
var (
@@ -205,10 +208,12 @@ func NewKubeGenericRuntimeManager(
205208
getNodeAllocatable func() v1.ResourceList,
206209
memoryThrottlingFactor float64,
207210
podPullingTimeRecorder images.ImagePodPullingTimeRecorder,
211+
tracerProvider trace.TracerProvider,
208212
) (KubeGenericRuntime, error) {
209213
ctx := context.Background()
210214
runtimeService = newInstrumentedRuntimeService(runtimeService)
211215
imageService = newInstrumentedImageManagerService(imageService)
216+
tracer := tracerProvider.Tracer(instrumentationScope)
212217
kubeRuntimeManager := &kubeGenericRuntimeManager{
213218
recorder: recorder,
214219
cpuCFSQuota: cpuCFSQuota,
@@ -281,7 +286,7 @@ func NewKubeGenericRuntimeManager(
281286
imagePullBurst,
282287
podPullingTimeRecorder)
283288
kubeRuntimeManager.runner = lifecycle.NewHandlerRunner(insecureContainerLifecycleHTTPClient, kubeRuntimeManager, kubeRuntimeManager, recorder)
284-
kubeRuntimeManager.containerGC = newContainerGC(runtimeService, podStateProvider, kubeRuntimeManager)
289+
kubeRuntimeManager.containerGC = newContainerGC(runtimeService, podStateProvider, kubeRuntimeManager, tracer)
285290
kubeRuntimeManager.podStateProvider = podStateProvider
286291

287292
kubeRuntimeManager.versionCache = cache.NewObjectCache(

‎pkg/kubelet/kuberuntime/kuberuntime_manager_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
cadvisorapi "github.com/google/cadvisor/info/v1"
3232
"github.com/stretchr/testify/assert"
3333
"github.com/stretchr/testify/require"
34+
oteltrace "go.opentelemetry.io/otel/trace"
3435

3536
v1 "k8s.io/api/core/v1"
3637
"k8s.io/apimachinery/pkg/api/resource"
@@ -69,7 +70,7 @@ func customTestRuntimeManager(keyring *credentialprovider.BasicDockerKeyring) (*
6970
MemoryCapacity: uint64(memoryCapacityQuantity.Value()),
7071
}
7172
osInterface := &containertest.FakeOS{}
72-
manager, err := newFakeKubeRuntimeManager(fakeRuntimeService, fakeImageService, machineInfo, osInterface, &containertest.FakeRuntimeHelper{}, keyring)
73+
manager, err := newFakeKubeRuntimeManager(fakeRuntimeService, fakeImageService, machineInfo, osInterface, &containertest.FakeRuntimeHelper{}, keyring, oteltrace.NewNoopTracerProvider().Tracer(""))
7374
return fakeRuntimeService, fakeImageService, manager, err
7475
}
7576

0 commit comments

Comments
 (0)