From 04875beb79a8b98f3079fb866beb5ede835772a5 Mon Sep 17 00:00:00 2001 From: Daniel Berteaud Date: Tue, 26 Mar 2019 09:31:08 +0100 Subject: [PATCH] Implement disabling compression, only save cursor every 30 sec or when exiting --- systemd-journal-gelf | 77 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 24 deletions(-) diff --git a/systemd-journal-gelf b/systemd-journal-gelf index 4afd910..f993b17 100644 --- a/systemd-journal-gelf +++ b/systemd-journal-gelf @@ -10,10 +10,50 @@ use Getopt::Long; use YAML::Tiny; use MIME::Base64; + +#### Global vars #### + my $conf = {}; my $cmd = { config => '/etc/systemd/journal-gelf.yml' }; +my $cursor = undef; +my $last_save = 0; +my $cursor_re = qr{^s=[a-z\d]+;i=[a-z\d]+;b=[a-z\d]+;m=[a-z\d]+;t=[a-z\d]+;x=[a-z\d]+$}; + +#### End global vars + + +#### Routines ##### + +sub help { + print <<"_EOF" + +Usage: $0 --url= [--compress|--no-compress] [--user=production --password=secr3t] [--state=/path/to/file] + + * --url is the http or https URL where you will push your gelf formated logs. This is mandatory + * --compress or --no-compress : will turn on or off gzip compression of logs. Default is on, but can be usefull to disable for debugging + * --username and --password may be used if URL is protected with a basic auth mecanism. Either both or none must be provided + * --state can be used to specify where to record the last correctly sent message, so we can start from here when + systemd-journal-gelf is restarted or if there's a network problem. Default value is /var/lib/systemd-journal-gelf/state + +_EOF +} + +sub save_cursor { + if ($cursor and $cursor =~ m/$cursor_re/){ + open CURSOR, ">", $conf->{state}; + print CURSOR $cursor; + close CURSOR + } +} + +sub handle_stop { + print "Saving current cursor to " . $conf->{state} . "\n"; + save_cursor(); +} + +#### End Routines #### GetOptions ( 'c|config=s' => \$cmd->{config}, @@ -62,6 +102,10 @@ if ( print "Starting the Systemd Journal GELF uploader daemon\n"; +# Catch SIGTERM and save the cursor ! +$SIG{TERM} = 'handle_stop'; +$SIG{INT} = 'handle_stop'; + my $ua = LWP::UserAgent->new( env_proxy => 1, keep_alive => 1 @@ -86,7 +130,7 @@ open CURSOR, "+<", $conf->{state}; if ( -e $conf->{state} ){ my $cursor = ; close CURSOR; - if ( $cursor and $cursor =~ m/^s=[a-z\d]+;i=[a-z\d]+;b=[a-z\d]+;m=[a-z\d]+;t=[a-z\d]+;x=[a-z\d]+$/ ){ + if ( $cursor and $cursor =~ m/$cursor_re/ ){ print "Valid cursor found in " . $conf->{state} . ", will start back from here\n"; $cursor_arg = " --after-cursor='" . $cursor . "'"; } else { @@ -149,12 +193,7 @@ while ( my $entry = ){ } $resp = $ua->post( $conf->{url}, - Content => Compress::Zlib::memGzip( - encode( - 'utf-8', - to_json($gelf) - ) - ) + Content => ($conf->{compress}) ? Compress::Zlib::memGzip(encode('utf-8', to_json($gelf))) : encode('utf-8', to_json($gelf)) ); $retry = ($retry > 0) ? $retry * 2 : 1; } while (not $resp->is_success and $retry < 600); @@ -162,9 +201,13 @@ while ( my $entry = ){ # The message has been accepted, we can save the current cursor and # continue if ($resp->is_success){ - open CURSOR, ">", $conf->{state}; - print CURSOR $msg->{__CURSOR}; - close CURSOR + $cursor = $msg->{__CURSOR}; + # Save the current cursor to disk if + # it hasn't been done for the past 30 sec + if (time - $last_save > 30) { + $last_save = time; + save_cursor(); + } } else { # We can't upload our current message for # too much time, no much left we can do, lets die and hope @@ -172,17 +215,3 @@ while ( my $entry = ){ die "Error sending data to GELF server\n"; } } - -sub help { - print <<"_EOF" - -Usage: $0 --url= [--compress|--no-compress] [--user=production --password=secr3t] [--state=/path/to/file] - - * --url is the http or https URL where you will push your gelf formated logs. This is mandatory - * --compress or --no-compress : will turn on or off gzip compression of logs. Default is on, but can be usefull to disable for debugging - * --username and --password may be used if URL is protected with a basic auth mecanism. Either both or none must be provided - * --state can be used to specify where to record the last correctly sent message, so we can start from here when - systemd-journal-gelf is restarted or if there's a network problem. Default value is /var/lib/systemd-journal-gelf/state - -_EOF -}