Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update perf #1575

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Update perf #1575

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0e31986
httpd: have much smaller prefork min limits
steveschnepp Jul 5, 2023
1386e3d
u: fix the R/O sqlite for NFS
steveschnepp Jul 5, 2023
1ff62ba
uw: batching RRD updates
steveschnepp Jul 5, 2023
f4e67fd
uw: emit the rrd update as info
steveschnepp Jul 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 16 additions & 7 deletions lib/Munin/Master/Update.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,22 +76,31 @@ sub get_dbh {
# die loudly than injecting some misguided data
use DBI;
my %db_args;
$db_args{ReadOnly} = 1 if $is_read_only;
$db_args{RaiseError} = 1;
$db_args{ReadOnly} = 1 if $is_read_only;

use Carp;
$db_args{HandleError} = sub { confess(shift) };

my $dbh = DBI->connect("dbi:$db_driver:dbname=$datafilename", $db_user, $db_passwd, \%db_args) or die $DBI::errstr;
my $dbstring = "dbname=$datafilename";
# ReadOnly is handled differently in SQLite
if ($db_driver eq "SQLite" && $is_read_only) {
delete $db_args{ReadOnly};
$dbstring = "uri=file:$datafilename?immutable=1";
}

my $dbh = DBI->connect("dbi:$db_driver:$dbstring", $db_user, $db_passwd, \%db_args) or die $DBI::errstr;

DEBUG 'get_dbh: $dbh->{Driver}->{Name} = ' . $dbh->{Driver}->{Name} . ($is_read_only ? "(ro)" : "(rw)");

# Sets some session vars
my $db_journal_mode = $ENV{MUNIN_DB_JOURNAL_MODE} || $config->{db_journal_mode} || "TRUNCATE";
$dbh->do("PRAGMA journal_mode=$db_journal_mode;") if $db_driver eq "SQLite";
# Sets some session vars for SQLite
if ($db_driver eq "SQLite") {
my $db_journal_mode = $ENV{MUNIN_DB_JOURNAL_MODE} || $config->{db_journal_mode} || "TRUNCATE";
$dbh->do("PRAGMA journal_mode=$db_journal_mode;") unless $db_args{ReadOnly};

my $db_synchronous_mode = $ENV{MUNIN_DB_SYNCHRONOUS_MODE} || $config->{db_synchronous_mode} || "OFF";
$dbh->do("PRAGMA main.synchronous=$db_synchronous_mode;") if $db_driver eq "SQLite";
my $db_synchronous_mode = $ENV{MUNIN_DB_SYNCHRONOUS_MODE} || $config->{db_synchronous_mode} || "OFF";
$dbh->do("PRAGMA main.synchronous=$db_synchronous_mode;") unless $db_args{ReadOnly};
}

# AutoCommit when readonly is a no-op anyway
$dbh->{AutoCommit} = 0;
Expand Down
28 changes: 20 additions & 8 deletions lib/Munin/Master/UpdateWorker.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -730,14 +730,17 @@ sub uw_handle_fetch {

my ($update_rate_in_seconds, $is_update_aligned) = parse_update_rate($update_rate);

# Vectorized updates
my %rrd_updates;

# Process all the data in-order
for my $line (@$data) {
next if ($line =~ m/^#/); # Ignore lines with comments
next unless ($line =~ m{\A ([^\.]+)(?:\.(\S+))? \s+ ([\S:]+) }xms);
my ($field, $arg, $value) = ($1, $2, $3);
$arg = "" unless defined $arg;
if ($arg ne "value") {
WARN "got '$line' but it should not be part of a fetch reply as (arg:$arg), ignoring.";
WARN "got '$line' when fetching $plugin but it should not be part of a fetch reply as (arg:$arg), ignoring.";
next;
}

Expand Down Expand Up @@ -783,13 +786,22 @@ sub uw_handle_fetch {

# This is a little convoluted but is needed as the API permits
# vectorized updates
my $ds_values = {
"value" => [ $value, ],
"when" => [ $when, ],
};
DEBUG "[DEBUG] self->_update_rrd_file($rrd_file, $field, $ds_values";
$self->_update_rrd_file($rrd_file, $field, $ds_values);
my $ds_values = $rrd_updates{$rrd_file}{$field};
if (! $ds_values) {
$ds_values = {};
$rrd_updates{$rrd_file}{$field} = $ds_values;
}
push @{ $ds_values->{value} }, $value;
push @{ $ds_values->{when} }, $when;
}

# Updating the RRDs
for my $rrd_file (keys %rrd_updates) {
for my $field (keys %{$rrd_updates{$rrd_file}}) {
my $ds_values = $rrd_updates{$rrd_file}{$field};
DEBUG "[DEBUG] self->_update_rrd_file($rrd_file, $field, $ds_values";
$self->_update_rrd_file($rrd_file, $field, $ds_values);
}
}

$self->{dbh}->commit() unless $self->{dbh}->{AutoCommit};
Expand Down Expand Up @@ -1077,7 +1089,7 @@ sub _update_rrd_file {
}
} else {
# normal vector-update the RRD
DEBUG "RRDs::update($rrd_file, @update_rrd_data)";
INFO "RRDs::update($rrd_file, @update_rrd_data)";
RRDs::update($rrd_file, @update_rrd_data);
}

Expand Down
8 changes: 7 additions & 1 deletion script/munin-httpd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,13 @@ if ($stdout) {

Munin::Common::Logger::INFO("Starting preforking munin http server at $host:$port");

Munin::Master::Http->new( { host => $host, port => $port } )->run( prefork => 1, max_servers => 10 );
Munin::Master::Http->new( { host => $host, port => $port } )->run(
prefork => ($max_servers > 0),
min_servers => 1,
max_servers => $max_servers,
min_spare_servers => 1,
max_spare_servers => $max_servers / 10,
);


sub print_usage_and_exit {
Expand Down