Skip to content

Commit acaa367

Browse files
yitzhaktalyitzhaktal
committed
DEVOPS-2694 - Update the lightrun-k8s-operator deployment to mount Secrets as files via volumes instead of exposing them as environment variables in containers.
1 parent 2be38da commit acaa367

File tree

4 files changed

+159
-109
lines changed

4 files changed

+159
-109
lines changed

‎internal/controller/lightrunjavaagent_controller.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ func (r *LightrunJavaAgentReconciler) reconcileDeployment(ctx context.Context, l
252252

253253
// Create config map
254254
log.V(2).Info("Reconciling config map with agent configuration")
255-
configMap, err := r.createAgentConfig(lightrunJavaAgent)
255+
configMap, err := r.createAgentConfig(lightrunJavaAgent, secret)
256256
if err != nil {
257257
log.Error(err, "unable to create configMap")
258258
return r.errorStatus(ctx, lightrunJavaAgent, err)
@@ -493,7 +493,7 @@ func (r *LightrunJavaAgentReconciler) reconcileStatefulSet(ctx context.Context,
493493

494494
// Create config map
495495
log.V(2).Info("Reconciling config map with agent configuration")
496-
configMap, err := r.createAgentConfig(lightrunJavaAgent)
496+
configMap, err := r.createAgentConfig(lightrunJavaAgent, secret)
497497
if err != nil {
498498
log.Error(err, "unable to create configMap")
499499
return r.errorStatus(ctx, lightrunJavaAgent, err)

‎internal/controller/patch_funcs.go‎

Lines changed: 73 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ const (
2727
annotationAgentName = "lightrun.com/lightrunjavaagent"
2828
)
2929

30-
func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agentv1beta.LightrunJavaAgent) (corev1.ConfigMap, error) {
30+
func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) (corev1.ConfigMap, error) {
3131
populateTags(lightrunJavaAgent.Spec.AgentTags, lightrunJavaAgent.Spec.AgentName, &metadata)
3232
jsonString, err := json.Marshal(metadata)
3333
if err != nil {
@@ -52,26 +52,28 @@ func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agent
5252
}
5353

5454
func (r *LightrunJavaAgentReconciler) patchDeployment(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret, origDeployment *appsv1.Deployment, deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration, cmDataHash uint64) error {
55-
5655
// init spec.template.spec
5756
deploymentApplyConfig.WithSpec(
5857
appsv1ac.DeploymentSpec().WithTemplate(
5958
corev1ac.PodTemplateSpec().WithSpec(
6059
corev1ac.PodSpec(),
6160
).WithAnnotations(map[string]string{
6261
annotationConfigMapHash: fmt.Sprint(cmDataHash),
63-
},
64-
),
62+
}),
6563
),
6664
).WithAnnotations(map[string]string{
6765
annotationAgentName: lightrunJavaAgent.Name,
6866
})
6967
r.addVolume(deploymentApplyConfig, lightrunJavaAgent)
7068
r.addInitContainer(deploymentApplyConfig, lightrunJavaAgent, secret)
71-
err = r.patchAppContainers(lightrunJavaAgent, origDeployment, deploymentApplyConfig)
69+
err := r.patchAppContainers(lightrunJavaAgent, origDeployment, deploymentApplyConfig)
7270
if err != nil {
7371
return err
7472
}
73+
deploymentApplyConfig.Spec.Template.Spec.WithSecurityContext(
74+
corev1ac.PodSecurityContext().
75+
WithFSGroup(1000),
76+
)
7577
return nil
7678
}
7779

@@ -98,55 +100,55 @@ func (r *LightrunJavaAgentReconciler) addVolume(deploymentApplyConfig *appsv1ac.
98100
)
99101
}
100102

103+
// Client side patch, as we can't update value from 2 sources
101104
func (r *LightrunJavaAgentReconciler) addInitContainer(deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration, lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) {
102-
103105
deploymentApplyConfig.Spec.Template.Spec.WithInitContainers(
104106
corev1ac.Container().
105107
WithName(initContainerName).
106108
WithImage(lightrunJavaAgent.Spec.InitContainer.Image).
107109
WithVolumeMounts(
108-
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
110+
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath(lightrunJavaAgent.Spec.InitContainer.SharedVolumeMountPath),
109111
corev1ac.VolumeMount().WithName(cmVolumeName).WithMountPath("/tmp/cm/"),
110-
).WithEnv(
111-
corev1ac.EnvVar().WithName("LIGHTRUN_KEY").WithValueFrom(
112-
corev1ac.EnvVarSource().WithSecretKeyRef(
113-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("lightrun_key"),
114-
),
115-
),
116-
corev1ac.EnvVar().WithName("PINNED_CERT").WithValueFrom(
117-
corev1ac.EnvVarSource().WithSecretKeyRef(
118-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("pinned_cert_hash"),
119-
),
120-
),
121-
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
122-
).
112+
corev1ac.VolumeMount().WithName("lightrun-secret").WithMountPath("/etc/lightrun/secret").WithReadOnly(true),
113+
).
114+
WithEnv(
115+
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
116+
).
117+
WithSecurityContext(
118+
corev1ac.SecurityContext().
119+
WithReadOnlyRootFilesystem(true).
120+
WithAllowPrivilegeEscalation(false).
121+
WithRunAsNonRoot(true).
122+
WithRunAsUser(1000),
123+
).
123124
WithResources(
124125
corev1ac.ResourceRequirements().
125126
WithLimits(
126127
corev1.ResourceList{
127128
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
128-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)), // 500 * 10^6 = 500M
129+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
129130
},
130-
).WithRequests(
131-
corev1.ResourceList{
132-
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
133-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
134-
},
135-
),
136-
).
137-
WithSecurityContext(
138-
corev1ac.SecurityContext().
139-
WithCapabilities(
140-
corev1ac.Capabilities().WithDrop(corev1.Capability("ALL")),
141131
).
142-
WithAllowPrivilegeEscalation(false).
143-
WithRunAsNonRoot(true).
144-
WithSeccompProfile(
145-
corev1ac.SeccompProfile().
146-
WithType(corev1.SeccompProfileTypeRuntimeDefault),
132+
WithRequests(
133+
corev1.ResourceList{
134+
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
135+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
136+
},
147137
),
148138
),
149139
)
140+
141+
// Add volume for secret with proper permissions
142+
deploymentApplyConfig.Spec.Template.Spec.WithVolumes(
143+
corev1ac.Volume().WithName("lightrun-secret").
144+
WithSecret(corev1ac.SecretVolumeSource().
145+
WithSecretName(secret.Name).
146+
WithItems(
147+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
148+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
149+
).
150+
WithDefaultMode(0440)),
151+
)
150152
}
151153

152154
func (r *LightrunJavaAgentReconciler) patchAppContainers(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, origDeployment *appsv1.Deployment, deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration) error {
@@ -167,26 +169,22 @@ func (r *LightrunJavaAgentReconciler) patchAppContainers(lightrunJavaAgent *agen
167169
}
168170
}
169171
if !found {
170-
err = errors.New("unable to find matching container to patch")
171-
return err
172+
return errors.New("unable to find matching container to patch")
172173
}
173174
return nil
174175
}
175176

176177
// Client side patch, as we can't update value from 2 sources
177178
func (r *LightrunJavaAgentReconciler) patchJavaToolEnv(deplAnnotations map[string]string, container *corev1.Container, targetEnvVar string, agentArg string) error {
178-
// Check if some env was already patched before
179179
patchedEnv := deplAnnotations[annotationPatchedEnvName]
180180
patchedEnvValue := deplAnnotations[annotationPatchedEnvValue]
181181

182182
if patchedEnv != targetEnvVar || patchedEnvValue != agentArg {
183-
// If different env was patched before - unpatch it
184183
r.unpatchJavaToolEnv(deplAnnotations, container)
185184
}
186185

187186
targetEnvVarIndex := findEnvVarIndex(targetEnvVar, container.Env)
188187
if targetEnvVarIndex == -1 {
189-
// No such env - add new
190188
container.Env = append(container.Env, corev1.EnvVar{
191189
Name: targetEnvVar,
192190
Value: agentArg,
@@ -223,28 +221,22 @@ func (r *LightrunJavaAgentReconciler) unpatchJavaToolEnv(deplAnnotations map[str
223221

224222
// patchStatefulSet applies changes to a StatefulSet to inject the Lightrun agent
225223
func (r *LightrunJavaAgentReconciler) patchStatefulSet(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret, origStatefulSet *appsv1.StatefulSet, statefulSetApplyConfig *appsv1ac.StatefulSetApplyConfiguration, cmDataHash uint64) error {
226-
// init spec.template.spec
227224
statefulSetApplyConfig.WithSpec(
228225
appsv1ac.StatefulSetSpec().WithTemplate(
229226
corev1ac.PodTemplateSpec().WithSpec(
230227
corev1ac.PodSpec(),
231228
).WithAnnotations(map[string]string{
232229
annotationConfigMapHash: fmt.Sprint(cmDataHash),
233-
},
234-
),
230+
}),
235231
),
236232
).WithAnnotations(map[string]string{
237233
annotationAgentName: lightrunJavaAgent.Name,
238234
})
239235

240-
// Add volumes to the StatefulSet
241236
r.addVolumeToStatefulSet(statefulSetApplyConfig, lightrunJavaAgent)
242-
243-
// Add init container to the StatefulSet
244237
r.addInitContainerToStatefulSet(statefulSetApplyConfig, lightrunJavaAgent, secret)
245238

246-
// Patch app containers in the StatefulSet
247-
err = r.patchStatefulSetAppContainers(lightrunJavaAgent, origStatefulSet, statefulSetApplyConfig)
239+
err := r.patchStatefulSetAppContainers(lightrunJavaAgent, origStatefulSet, statefulSetApplyConfig)
248240
if err != nil {
249241
return err
250242
}
@@ -271,6 +263,15 @@ func (r *LightrunJavaAgentReconciler) addVolumeToStatefulSet(statefulSetApplyCon
271263
corev1ac.KeyToPath().WithKey("metadata").WithPath("agent.metadata.json"),
272264
),
273265
),
266+
).WithVolumes(
267+
corev1ac.Volume().WithName("lightrun-secret").
268+
WithSecret(corev1ac.SecretVolumeSource().
269+
WithSecretName(secret.Name).
270+
WithItems(
271+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
272+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
273+
).
274+
WithDefaultMode(0440)),
274275
)
275276
}
276277

@@ -280,48 +281,45 @@ func (r *LightrunJavaAgentReconciler) addInitContainerToStatefulSet(statefulSetA
280281
WithName(initContainerName).
281282
WithImage(lightrunJavaAgent.Spec.InitContainer.Image).
282283
WithVolumeMounts(
283-
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
284+
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath(lightrunJavaAgent.Spec.InitContainer.SharedVolumeMountPath),
284285
corev1ac.VolumeMount().WithName(cmVolumeName).WithMountPath("/tmp/cm/"),
286+
corev1ac.VolumeMount().WithName("lightrun-secret").WithMountPath("/etc/lightrun/secret").WithReadOnly(true),
285287
).WithEnv(
286-
corev1ac.EnvVar().WithName("LIGHTRUN_KEY").WithValueFrom(
287-
corev1ac.EnvVarSource().WithSecretKeyRef(
288-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("lightrun_key"),
289-
),
290-
),
291-
corev1ac.EnvVar().WithName("PINNED_CERT").WithValueFrom(
292-
corev1ac.EnvVarSource().WithSecretKeyRef(
293-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("pinned_cert_hash"),
294-
),
295-
),
296288
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
297289
).
290+
WithSecurityContext(
291+
corev1ac.SecurityContext().
292+
WithReadOnlyRootFilesystem(true).
293+
WithAllowPrivilegeEscalation(false).
294+
WithRunAsNonRoot(true).
295+
WithRunAsUser(1000),
296+
).
298297
WithResources(
299298
corev1ac.ResourceRequirements().
300299
WithLimits(
301300
corev1.ResourceList{
302301
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
303-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)), // 64M
302+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
304303
},
305304
).WithRequests(
306305
corev1.ResourceList{
307306
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
308307
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
309308
},
310309
),
311-
).
312-
WithSecurityContext(
313-
corev1ac.SecurityContext().
314-
WithCapabilities(
315-
corev1ac.Capabilities().WithDrop(corev1.Capability("ALL")),
316-
).
317-
WithAllowPrivilegeEscalation(false).
318-
WithRunAsNonRoot(true).
319-
WithSeccompProfile(
320-
corev1ac.SeccompProfile().
321-
WithType(corev1.SeccompProfileTypeRuntimeDefault),
322-
),
323310
),
324311
)
312+
313+
statefulSetApplyConfig.Spec.Template.Spec.WithVolumes(
314+
corev1ac.Volume().WithName("lightrun-secret").
315+
WithSecret(corev1ac.SecretVolumeSource().
316+
WithSecretName(secret.Name).
317+
WithItems(
318+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
319+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
320+
).
321+
WithDefaultMode(0440)),
322+
)
325323
}
326324

327325
func (r *LightrunJavaAgentReconciler) patchStatefulSetAppContainers(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, origStatefulSet *appsv1.StatefulSet, statefulSetApplyConfig *appsv1ac.StatefulSetApplyConfiguration) error {
@@ -342,15 +340,13 @@ func (r *LightrunJavaAgentReconciler) patchStatefulSetAppContainers(lightrunJava
342340
}
343341
}
344342
if !found {
345-
err = errors.New("unable to find matching container to patch")
346-
return err
343+
return errors.New("unable to find matching container to patch")
347344
}
348345
return nil
349346
}
350347

351348
// configMapDataHash calculates a hash of the ConfigMap data to detect changes
352349
func configMapDataHash(cmData map[string]string) uint64 {
353-
// Combine all data values into a single string for hashing
354350
var hashString string
355351
for _, v := range cmData {
356352
hashString += v

‎lightrun-init-agent/Dockerfile‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,26 @@
11
ARG base_image_tag=alpine-3.20.0-r1
22

33
FROM lightruncom/prod-base:${base_image_tag}
4-
ARG FILE
5-
6-
COPY lightrun-init-agent/$FILE /tmp/$FILE
4+
ARG FILE
75

86
RUN unzip -o /tmp/$FILE -d /agent ;\
97
rm -rf /tmp/$FILE && \
108
# Erase default values
119
sed -i.bak "s|com.lightrun.secret=.*|com.lightrun.secret=|" /agent/agent.config && rm /agent/agent.config.bak && \
1210
sed -i.bak "s|pinned_certs=.*|pinned_certs=|" /agent/agent.config && rm /agent/agent.config.bak && \
13-
# In openshift UID will be dynamic per project, hence procide permissions to root group (defualt in k8s)
14-
chgrp -R 0 /agent && \
15-
chmod -R g=u /agent
11+
# Set proper permissions for the agent directory
12+
chown -R 1000:1000 /agent && \
13+
chmod -R 750 /agent && \
14+
# Create secret directory with proper permissions
15+
mkdir -p /etc/lightrun/secret && \
16+
chown -R 1000:1000 /etc/lightrun/secret && \
17+
chmod -R 700 /etc/lightrun/secret
18+
19+
# Copy and set permissions for update_config.sh before switching user
20+
COPY update_config.sh /update_config.sh
21+
RUN chmod 750 /update_config.sh && \
22+
chown 1000:1000 /update_config.sh
1623

1724
USER 1000
18-
COPY lightrun-init-agent/update_config.sh /update_config.sh
1925

2026
CMD [ "/bin/sh", "/update_config.sh" ]

0 commit comments

Comments
 (0)