Skip to content

Commit 637d06e

Browse files
LordHepipudLordHepipud
committed
Fix memory leak on background service daemon
1 parent 20a4c8d commit 637d06e

File tree

4 files changed

+63
-24
lines changed

4 files changed

+63
-24
lines changed

‎doc/31-Changelog.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,12 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic
1313

1414
### Bugfixes
1515

16-
## 1.4.1 (pending)
16+
## 1.4.1 (2021-03-10)
1717

1818
### Bugfixes
1919

2020
* [#222](https://github.com/Icinga/icinga-powershell-framework/pull/222) Fixes an issue with [Secure.String] arguments for PowerShell plugins, caused by `ConvertTo-IcingaSecureString` Cmdlet not being pre-loaded
21+
* [#224](https://github.com/Icinga/icinga-powershell-framework/issues/224) Fixes "memory leak" on background daemon for registered service checks, by clearing the error stack and manually calling the PowerShell garbage collector to force freeing of memory
2122

2223
## 1.4.0 (2021-03-02)
2324

‎lib/core/logging/Write-IcingaConsoleOutput.psm1

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,11 @@ function Write-IcingaConsoleOutput()
4040
return;
4141
}
4242

43+
# Never write console output in case the Framework is running as daemon
44+
if ($null -ne $global:IcingaDaemonData -And $null -ne $global:IcingaDaemonData.FrameworkRunningAsDaemon -And $global:IcingaDaemonData.FrameworkRunningAsDaemon -eq $TRUE) {
45+
return;
46+
}
47+
4348
$OutputMessage = $Message;
4449
[int]$Index = 0;
4550

‎lib/daemons/ServiceCheckDaemon/Start-IcingaServiceCheckDaemon.psm1

Lines changed: 47 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@ function Start-IcingaServiceCheckTask()
6666
Use-Icinga -LibOnly -Daemon;
6767
$PassedTime = 0;
6868
$SortedResult = $null;
69-
$OldData = @{ };
7069
$PerfCache = @{ };
7170
$AverageCalc = @{ };
7271
[int]$MaxTime = 0;
@@ -85,8 +84,8 @@ function Start-IcingaServiceCheckTask()
8584
$AverageCalc.Add(
8685
[string]$index,
8786
@{
88-
'Interval' = [int]$index;
89-
'Time' = [int]$index * 60;
87+
'Interval' = ([int]$index);
88+
'Time' = ([int]$index * 60);
9089
'Sum' = 0;
9190
'Count' = 0;
9291
}
@@ -126,19 +125,26 @@ function Start-IcingaServiceCheckTask()
126125
if ($PassedTime -ge $Interval) {
127126
try {
128127
& $CheckCommand @Arguments | Out-Null;
128+
} catch {
129+
# Just for debugging. Not required in production or usable at all
130+
$ErrMsg = $_.Exception.Message;
131+
Write-IcingaConsoleError $ErrMsg;
132+
}
129133

130-
Get-IcingaCheckSchedulerPerfData | Out-Null;
131-
Get-IcingaCheckSchedulerPluginOutput | Out-Null;
132-
134+
try {
133135
$UnixTime = Get-IcingaUnixTime;
134136

135137
foreach ($result in $global:Icinga.CheckData[$CheckCommand]['results'].Keys) {
136138
[string]$HashIndex = $result;
137139
$SortedResult = $global:Icinga.CheckData[$CheckCommand]['results'][$HashIndex].GetEnumerator() | Sort-Object name -Descending;
138-
Add-IcingaHashtableItem -Hashtable $OldData -Key $HashIndex -Value @{ } | Out-Null;
139140
Add-IcingaHashtableItem -Hashtable $PerfCache -Key $HashIndex -Value @{ } | Out-Null;
140141

141142
foreach ($timeEntry in $SortedResult) {
143+
144+
if ((Test-Numeric $timeEntry.Value) -eq $FALSE) {
145+
continue;
146+
}
147+
142148
foreach ($calc in $AverageCalc.Keys) {
143149
if (($UnixTime - $AverageCalc[$calc].Time) -le [int]$timeEntry.Key) {
144150
$AverageCalc[$calc].Sum += $timeEntry.Value;
@@ -147,47 +153,66 @@ function Start-IcingaServiceCheckTask()
147153
}
148154
if (($UnixTime - $MaxTimeInSeconds) -le [int]$timeEntry.Key) {
149155
Add-IcingaHashtableItem -Hashtable $PerfCache[$HashIndex] -Key ([string]$timeEntry.Key) -Value ([string]$timeEntry.Value) | Out-Null;
150-
} else {
151-
Add-IcingaHashtableItem -Hashtable $OldData[$HashIndex] -Key $timeEntry -Value $null | Out-Null;
152156
}
153157
}
154158

155159
foreach ($calc in $AverageCalc.Keys) {
156-
$AverageValue = ($AverageCalc[$calc].Sum / $AverageCalc[$calc].Count);
157-
[string]$MetricName = Format-IcingaPerfDataLabel (
158-
[string]::Format('{0}_{1}', $HashIndex, $AverageCalc[$calc].Interval)
159-
);
160-
161-
Add-IcingaHashtableItem `
162-
-Hashtable $global:Icinga.CheckData[$CheckCommand]['average'] `
163-
-Key $MetricName -Value $AverageValue -Override | Out-Null;
160+
if ($AverageCalc[$calc].Count -ne 0) {
161+
$AverageValue = ($AverageCalc[$calc].Sum / $AverageCalc[$calc].Count);
162+
[string]$MetricName = Format-IcingaPerfDataLabel (
163+
[string]::Format('{0}_{1}', $HashIndex, $AverageCalc[$calc].Interval)
164+
);
165+
166+
Add-IcingaHashtableItem `
167+
-Hashtable $global:Icinga.CheckData[$CheckCommand]['average'] `
168+
-Key $MetricName -Value $AverageValue -Override | Out-Null;
169+
}
164170

165171
$AverageCalc[$calc].Sum = 0;
166172
$AverageCalc[$calc].Count = 0;
167173
}
168174
}
169175

170176
# Flush data we no longer require in our cache to free memory
171-
foreach ($entry in $OldData.Keys) {
172-
foreach ($key in $OldData[$entry].Keys) {
173-
Remove-IcingaHashtableItem -Hashtable $global:Icinga.CheckData[$CheckCommand]['results'][$entry] -Key $key.Name;
177+
[array]$CheckStores = $global:Icinga.CheckData[$CheckCommand]['results'].Keys;
178+
179+
foreach ($CheckStore in $CheckStores) {
180+
[string]$CheckKey = $CheckStore;
181+
[array]$CheckTimeStamps = $global:Icinga.CheckData[$CheckCommand]['results'][$CheckKey].Keys;
182+
183+
foreach ($TimeSample in $CheckTimeStamps) {
184+
if (($UnixTime - $MaxTimeInSeconds) -gt [int]$TimeSample) {
185+
Remove-IcingaHashtableItem -Hashtable $global:Icinga.CheckData[$CheckCommand]['results'][$CheckKey] -Key ([string]$TimeSample);
186+
}
174187
}
175188
}
176189

177190
Set-IcingaCacheData -Space 'sc_daemon' -CacheStore 'checkresult' -KeyName $CheckCommand -Value $global:Icinga.CheckData[$CheckCommand]['average'];
178191
# Write collected metrics to disk in case we reload the daemon. We will load them back into the module after reload then
179192
Set-IcingaCacheData -Space 'sc_daemon' -CacheStore 'checkresult_store' -KeyName $CheckCommand -Value $PerfCache;
180193
} catch {
181-
# Todo: Add error reporting / handling
194+
# Just for debugging. Not required in production or usable at all
195+
$ErrMsg = $_.Exception.Message;
196+
Write-IcingaConsoleError 'Failed to handle check result processing: {0}' -Objects $ErrMsg;
182197
}
183198

199+
# Cleanup the error stack and remove not required data
200+
$Error.Clear();
201+
202+
# Always ensure our check data is cleared regardless of possible
203+
# exceptions which might occur
204+
Get-IcingaCheckSchedulerPerfData | Out-Null;
205+
Get-IcingaCheckSchedulerPluginOutput | Out-Null;
206+
184207
$PassedTime = 0;
185208
$SortedResult.Clear();
186-
$OldData.Clear();
187209
$PerfCache.Clear();
188210
}
211+
189212
$PassedTime += 1;
190213
Start-Sleep -Seconds 1;
214+
# Force PowerShell to call the garbage collector to free memory
215+
[System.GC]::Collect();
191216
}
192217
};
193218

‎lib/icinga/plugin/New-IcingaCheck.psm1

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,16 @@ function New-IcingaCheck()
6363
);
6464
}
6565

66+
# Fix possible error for identical time stamps due to internal exceptions
67+
# and check execution within the same time slot because of this
68+
[string]$TimeIndex = Get-IcingaUnixTime;
69+
70+
if ($global:Icinga.CheckData[$this.checkcommand]['results'][$this.name].ContainsKey($TimeIndex)) {
71+
return;
72+
}
73+
6674
$global:Icinga.CheckData[$this.checkcommand]['results'][$this.name].Add(
67-
(Get-IcingaUnixTime),
75+
$TimeIndex,
6876
$this.value
6977
);
7078
}

0 commit comments

Comments
 (0)