From 9035410106e5a258519cfdb2f19dbdbbbe3cdc09 Mon Sep 17 00:00:00 2001 From: Daniel Berteaud Date: Mon, 25 Mar 2019 19:27:19 +0100 Subject: [PATCH] Initial import --- systemd-journal-gelf | 114 +++++++++++++++++++++++++++++++++++++++++++ systemd-journal-gelf.service | 22 +++++++++ systemd-journal-gelf.yml | 15 ++++++ 3 files changed, 151 insertions(+) create mode 100644 systemd-journal-gelf create mode 100644 systemd-journal-gelf.service create mode 100644 systemd-journal-gelf.yml diff --git a/systemd-journal-gelf b/systemd-journal-gelf new file mode 100644 index 0000000..ec4ca44 --- /dev/null +++ b/systemd-journal-gelf @@ -0,0 +1,114 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use JSON; +use LWP::UserAgent; +use Encode qw(encode); +use Data::Dumper; +use Compress::Zlib; +use Getopt::Long; +use Config::Any; + +my $config = '/etc/systemd/journal-gelf.yml'; +my $conf = {}; + +if (-e $config) { + print "Reading config file $config\n"; + $conf = Config::Any->load_files( { files => [ $config ], flatten_to_hash => 1, use_ext => 1 } )->{$config}; +} + +GetOptions ( + 'state=s' => \$conf->{state}, + 'compress!' => \$conf->{compress}, + 'url=s' => \$conf->{url}, + 'username=s' => \$conf->{username}, + 'password=s' => \$conf->{password} +); + +$conf->{state} //= '/var/lib/systemd-journal-gelf/state'; +$conf->{compress} //= 1; + +if ( + not $conf->{url} or + ($conf->{username} and not $conf->{password}) or + (not $conf->{username} and $conf->{password}) + ){ + help(); + die; +} + +print "Starting the Systemd Journal GELF uploader daemon\n"; + +my $ua = LWP::UserAgent->new( + # env_proxy => 1, + keep_alive => 1 +); +$ua->default_header( 'Content-Type' => 'application/json' ); +if ( $conf->{compress} ){ + $ua->default_header( 'Accept-Encoding' => HTTP::Message::decodable ); + $ua->default_header( 'Content-Encoding' => 'gzip' ); +} + +# Check if the state file exists and contains a valid cursor +my $cursor_arg = ''; +open CURSOR, "+<", $conf->{state}; +if (-e $conf->{state}){ + my $cursor = ; + close CURSOR; + if ($cursor and $cursor =~ m/^s=[a-z\d]+;i=[a-z\d]+;b=[a-z\d]+;m=[a-z\d]+;t=[a-z\d]+;x=[a-z\d]+$/){ + print "Valid cursor found in " . $conf->{state} . ", will start back from here\n"; + $cursor_arg = " --after-cursor='" . $cursor . "'"; + } else { + print $conf->{state} . " contains an invalid cursor, so we're wiping it\n"; + unlink $conf->{state}; + } +} + +open JOURNAL, "journalctl -f -o json$cursor_arg |"; +while (my $entry = ){ + my $msg = from_json($entry); + my $gelf = { + version => 1.1, + short_message => $msg->{MESSAGE}, + host => $msg->{_HOSTNAME}, + timestamp => int ($msg->{__REALTIME_TIMESTAMP} / (1000 * 1000)), + level => $msg->{PRIORITY} + }; + foreach (grep !/^MESSAGE|_HOSTNAME|__REALTIME_TIMESTAMP|PRIORITY$/, keys %$msg){ + my $key = lc (($_ =~ m/^_/) ? $_ : '_' . $_); + $gelf->{$key} = $msg->{$_}; + } + my $retry = 0; + my $resp; + do { + if ($retry > 0){ + print "Sending message to " . $conf->{url} . " failed : got code " . + $resp->code . " (" . $resp->message . "). Tring again in $retry seconds\n"; + sleep $retry; + } + $resp = $ua->post($conf->{url}, Content => Compress::Zlib::memGzip(encode('utf-8', to_json($gelf)))); + $retry = ($retry > 0) ? $retry * 2 : 1; + } while ($resp->code != 202 and $retry < 600); + if ($resp->code == 202){ + open CURSOR, ">", $conf->{state}; + print CURSOR $msg->{__CURSOR}; + close CURSOR + } else { + die "Error sending data to GELF server\n"; + } +} + +sub help { + print <<"_EOF" + +Usage: $0 --url= [--compress|--no-compress] [--user=production --password=secr3t] [--state=/path/to/file] + + * --url is the http or https URL where you will push your gelf formated logs. This is mandatory + * --compress or --no-compress : will turn on or off gzip compression of logs. Default is on, but can be usefull to disable for debugging + * --username and --password may be used if URL is protected with a basic auth mecanism. Either both or none must be provided + * --state can be used to specify where to record the last correctly sent message, so we can start from here when + systemd-journal-gelf is restarted or if there's a network problem. Default value is /var/lib/systemd-journal-gelf/state + +_EOF +} diff --git a/systemd-journal-gelf.service b/systemd-journal-gelf.service new file mode 100644 index 0000000..18a8e70 --- /dev/null +++ b/systemd-journal-gelf.service @@ -0,0 +1,22 @@ +[Unit] +Description=Systemd Journal GEL HTTP uploader + +[Service] +Type=simple +User=systemd-journal-gelf +Group=adm +ExecStart=/usr/local/bin/systemd-journal-gelf -c /etc/systemd/journal-gelf.yml +PrivateTmp=yes +PrivateDevices=yes +ProtectSystem=full +ProtectHome=yes +NoNewPrivileges=yes +MemoryLimit=200M +SyslogIdentifier=systemd-journal-gelf +Restart=on-failure +StartLimitInterval=0 +RestartSec=30 + +[Install] +WantedBy=multi-user.target + diff --git a/systemd-journal-gelf.yml b/systemd-journal-gelf.yml new file mode 100644 index 0000000..28db0ee --- /dev/null +++ b/systemd-journal-gelf.yml @@ -0,0 +1,15 @@ +--- +# Mandatory setting : URL on which gelf formated message will be posted +# url: https://graylog.domain.net:12201/gelf + +# Path to a file where we save the last correctly sent message, so we can start +# back where we left +# state: /var/lib/systemd-journal-gelf + +# If enabled, data will be compressed before being sent. Default is enabled +# Might be disabled for debuging purpose +# compress: True + +# If you http endpoint is protected with basic auth, set username and password +# username: foo +# password: p@ssw0rd