diff --git a/roles/crowdsec/defaults/main.yml b/roles/crowdsec/defaults/main.yml index 700d6e1..e9027fc 100644 --- a/roles/crowdsec/defaults/main.yml +++ b/roles/crowdsec/defaults/main.yml @@ -1,11 +1,11 @@ --- # Version to install -cs_version: 1.0.7 +cs_version: 1.0.8 # URL of the archive cs_archive_url: https://github.com/crowdsecurity/crowdsec/releases/download/v{{ cs_version }}/crowdsec-release.tgz # Expected sha1 of the archive -cs_archive_sha1: 7c9dc58c6648c8fd43b297427d6a53fe940cbf13 +cs_archive_sha1: 060782df0b6a8a799c1c0e6efc874b26ca9988e6 # Can be sqlite or mysql cs_db_engine: sqlite @@ -69,3 +69,16 @@ cs_postoverflows: [] # - crowdsecurity/rdns # - crowdsecurity/seo-bots-whitelist +# If not set, crowdsec will look for yaml files in /etc/crowdsec/acquis/ +# The default will only read syslog using journalctl +# If defined, only acquisition set by ansible will be used +# cs_aquis: +# - journalctl_filter: +# - '_SYSTEMD_UNIT=sshd.service' +# labels: +# type: syslog +# +# - filename: +# - /var/log/nginx/access.log +# labels: +# type: nginx diff --git a/roles/crowdsec/tasks/.conf.yml.swp b/roles/crowdsec/tasks/.conf.yml.swp new file mode 100644 index 0000000..073f5b9 Binary files /dev/null and b/roles/crowdsec/tasks/.conf.yml.swp differ diff --git a/roles/crowdsec/tasks/conf.yml b/roles/crowdsec/tasks/conf.yml index 61e77b7..d903838 100644 --- a/roles/crowdsec/tasks/conf.yml +++ b/roles/crowdsec/tasks/conf.yml @@ -8,6 +8,7 @@ - simulation.yaml - profiles.yaml - parsers/s02-enrich/trusted_ip.yaml + - dev.yaml notify: reload crowdsec tags: cs diff --git a/roles/crowdsec/tasks/directories.yml b/roles/crowdsec/tasks/directories.yml index 86fe46c..85bc09f 100644 --- a/roles/crowdsec/tasks/directories.yml +++ b/roles/crowdsec/tasks/directories.yml @@ -14,4 +14,5 @@ - dir: /etc/crowdsec/scenarios - dir: /etc/crowdsec/postoverflows/s00-enrich - dir: /etc/crowdsec/postoverflows/s01-whitelist + - dir: /etc/crowdsec/acquis tags: cs diff --git a/roles/crowdsec/templates/acquis.yaml.j2 b/roles/crowdsec/templates/acquis.yaml.j2 index 513d7bb..152bda7 100644 --- a/roles/crowdsec/templates/acquis.yaml.j2 +++ b/roles/crowdsec/templates/acquis.yaml.j2 @@ -1,11 +1,6 @@ +{% if cs_acquis is defined and cs_acquis | length > 0%} +{% for acquis in cs_acquis %} --- -journalctl_filter: - - "_SYSTEMD_UNIT=sshd.service" -labels: - type: syslog ---- -journalctl_filter: - - "_TRASPORT=kernel" -labels: - type: syslog - +{{ acquis | to_nice_yaml }} +{% endfor %} +{% endif %} diff --git a/roles/crowdsec/templates/acquis/system.yaml.j2 b/roles/crowdsec/templates/acquis/system.yaml.j2 new file mode 100644 index 0000000..b8b8149 --- /dev/null +++ b/roles/crowdsec/templates/acquis/system.yaml.j2 @@ -0,0 +1,5 @@ +--- +journalctl_filter: + - "" +labels: + type: syslog diff --git a/roles/crowdsec/templates/config.yaml.j2 b/roles/crowdsec/templates/config.yaml.j2 index e7c11e2..246b6f6 100644 --- a/roles/crowdsec/templates/config.yaml.j2 +++ b/roles/crowdsec/templates/config.yaml.j2 @@ -13,7 +13,11 @@ config_paths: index_path: /etc/crowdsec/hub/.index.json crowdsec_service: +{% if cs_acquis is defined %} acquisition_path: /etc/crowdsec/acquis.yaml +{% else %} + acquisition_dir: /etc/crowdsec/acquis/ +{% endif %} parser_routines: 1 cscli: diff --git a/roles/crowdsec/templates/dev.yaml.j2 b/roles/crowdsec/templates/dev.yaml.j2 new file mode 100644 index 0000000..abd5b45 --- /dev/null +++ b/roles/crowdsec/templates/dev.yaml.j2 @@ -0,0 +1,36 @@ +common: + daemonize: false + log_media: stdout + log_level: info + working_dir: . + +config_paths: + config_dir: /etc/crowdsec/ + data_dir: /var/lib/crowdsec/data/ + simulation_path: /etc/crowdsec/simulation.yaml + hub_dir: /etc/crowdsec/hub/ + index_path: /etc/crowdsec/hub/.index.json + +crowdsec_service: + acquisition_path: /etc/crowdsec/acquis.yaml + parser_routines: 1 + +cscli: + output: human + hub_branch: master + +db_config: + log_level: info + db_path: /var/lib/crowdsec/data/dev.db + flush: + max_items: 100000 + max_age: 730d + +api: + client: + insecure_skip_verify: false + credentials_path: /etc/crowdsec/local_api_credentials.yaml + +prometheus: + enabled: false + diff --git a/roles/g2cs/README.md b/roles/g2cs/README.md new file mode 100644 index 0000000..a5f3a29 --- /dev/null +++ b/roles/g2cs/README.md @@ -0,0 +1,17 @@ +# G2CS + +This is a small daemon writtent in perl to allow a bridge between Graylog and Crowdsec. +This idea is that if you collect your logs to a graylog instance, you can forward them all in a single stream from Graylog to CrowdSec, instead of collecting them all again on every hosts. + +So, this small g2cs daemon is a very simple perl utility which will listen on a port for a syslog stream. It should run a the server which will host your single crowdsec instance. + +On graylog, you have to install the syslog-output plugin, and configure it to output the streams you want to this daemon. You should choose UDP, the port on which g2cs binds, and the CEF format. + +When g2cs receive this stream of logs, it'll just make simple transformations so that your logs can be consumed by crowdsec : + + * nginx logs go to nginx/ + * httpd logs go to httpd/ + * squid logs go to squid/ + * Everything else goes to syslog.log + +Now, you can configure your acquisitions on crowdsec to just read these locations diff --git a/roles/g2cs/defaults/main.yml b/roles/g2cs/defaults/main.yml new file mode 100644 index 0000000..f2a19d7 --- /dev/null +++ b/roles/g2cs/defaults/main.yml @@ -0,0 +1,11 @@ +--- + +# Port on which g2cs will listen +g2cs_port: 3514 + +# Where log files will be created. Thos files won't grow too large as g2cs truncates them after 10000 lines +# so better to use a tmpfs +g2cs_log_dir: /run/g2cs/logs + +# List of IP/CIDR for which g2cs port will be reachable +g2cs_src_ip: [] diff --git a/roles/g2cs/files/g2cs.pl b/roles/g2cs/files/g2cs.pl new file mode 100644 index 0000000..ddd01dc --- /dev/null +++ b/roles/g2cs/files/g2cs.pl @@ -0,0 +1,165 @@ +#!/usr/bin/perl -w + +use IO::Socket; +use Getopt::Long; +use File::Basename; +use File::Path qw(make_path); +use IO::Handle; + +my $maxlen = 16384; +my $port = 514; +my $maxlines = 10000; +my $logdir = '/run/cs-gelf-server/'; + +GetOptions( + "port=i" => \$port, + "maxlines=i" => \$maxlines, + "logdir=s" => \$logdir +); + +if ($port !~ /^\d+$/ or $port < 1 or $port > 65535){ + die "Invalid port $port\n"; +} +if ($maxlines !~ /^\d+/ or $maxlines < 10){ + die "Invalid max line specified\n"; +} +if (not -d $logdir){ + die "$logdir doesn't exists or is not a directory\n"; +} + +# Remove trailing / of the logdir, it's not nice in the logs when you have double / +$logdir =~ s/\/$//; + +# List of syslog_identifier we're not intersted in +my @ignored_syslog_id = qw( + c-icap + charon + unbound + sudo + zed + zimbramon +); +# List of log files we're not interested in +my @ignored_log_files = qw( + /var/log/audit/audit.log + /var/log/squid/cache.log + /var/log/ufdbGuard/ufdbguardd.log + /opt/zimbra/log/gc.log +); + +print "Start listening on UDP port $port\n"; +$sock = IO::Socket::INET->new( + LocalPort => $port, + Proto => 'udp' + ) or die("Socket: $@"); + +my $buf; +my $cnt = {}; +my $loghandles = {}; + +while (1) { + $sock->recv($buf, $maxlen); + my ($port, $ipaddr) = sockaddr_in($sock->peername); + my $fields = {}; + + # We're not really interested in CEF headers. So let's extract + # the various fields + $buf =~ m/(?:(?:CEF:\d+\|)(?:[^=\\]+\|)+)(.*)/; + my $ext = $1; + + # Taken from https://github.com/DavidJBianco/pycef + while ($ext =~ m/([^=\s]+)=((?:[\\]=|[^=])+)(?:\s|$)/g) { + $fields->{$1} = $2; + # Unescape value string + $fields->{$1} =~ s/\\=/=/g; + } + + # Skip lines we're not interested in early. + # So crowdsec will eat less CPU parsing useless stuff + if ( + defined $fields->{syslog_identifier} and grep { $_ eq $fields->{syslog_identifier} } @ignored_syslog_id or + defined $fields->{log_file_path} and grep { $_ eq $fields->{log_file_path} } @ignored_log_files + ) { + next; + } + + # We need a timestamp, a source and a msg at least + if (not defined $fields->{timestamp} or not defined $fields->{source} or not defined $fields->{msg}){ + next; + } + + my $msg; + # Default log will be syslog + my $logfile = $logdir . '/syslog.log'; + + # But for some services, we need special handling. Eg for web access logs + if (defined $fields->{event_dataset}){ + if ($fields->{event_dataset} =~ m/^nginx\.(access|ingress_controller)/){ + $logfile = $logdir . '/nginx/access.log'; + $msg = $fields->{msg}; + } elsif ($fields->{event_dataset} =~ m/^nginx\.error/){ + $logfile = $logdir . '/nginx/error.log'; + $msg = $fields->{msg}; + } elsif ($fields->{event_dataset} =~ m/^apache\.access/){ + $logfile = $logdir . '/httpd/access.log'; + $msg = $fields->{msg}; + } elsif ($fields->{event_dataset} =~ m/^apache\.error/){ + $logfile = $logdir . '/httpd/access.log'; + $msg = $fields->{msg}; + } + } elsif (defined $fields->{log_file_path}){ + if ($fields->{log_file_path} eq '/var/log/pveproxy/access.log'){ + $logfile = $logdir . '/pveproxy/access.log'; + $msg = $fields->{msg}; + } elsif ($fields->{log_file_path} eq '/var/log/squid/access.log'){ + $logfile = $logdir . '/squid/access.log'; + $msg = $fields->{msg}; + } elsif ($fields->{log_file_path} eq '/opt/zimbra/log/nginx.access.log'){ + $logfile = $logdir . '/nginx/access.log'; + $msg = $fields->{msg}; + } + } elsif (defined $fields->{application_name}){ + if ($fields->{application_name} eq 'nginx'){ + $logfile = $logdir . '/nginx/access.log'; + $msg = $fields->{msg}; + } + } + + # OK, no special handling (else $msg would be defined), so let's + # provide a syslog format + if (not defined $msg){ + $msg .= $fields->{timestamp} . ' ' . $fields->{source} . ' '; + my $id = $fields->{syslog_identifier} || $fields->{program} || $fields->{application_name} || $fields->{process_name} || 'unknown'; + # For older PfSense, which sent invalid syslog messages, we might extract + # the syslog identifier from the begining of the message + if ($id eq 'unknown' and $fields->{msg} =~ m/^(\w+(\[\d+\])?):\s(.*)/){ + $id = $1; + $fields->{msg} = $3; + } + $msg .= $id; + # Try to append the pid of the process + if ($id ne 'kernel' and $id ne 'filterlog' and $id !~ m/\[\d+\]$/){ + $msg .= '['; + $msg .= $fields->{process_pid} || $fields->{process_id} || $fields->{pid} || '0'; + $msg .= ']'; + } + $msg .= ': ' . $fields->{msg}; + } + + # Create the log sub dir if needed + my $dir = dirname($logfile); + if (not -d $dir){ + make_path($dir); + } + + defined $loghandles->{$logfile} or open($loghandles->{$logfile}, ">>", $logfile); + # Truncate the file so it's not growing too large + # Crowdsec will read it in nearly real time anyway + if ($cnt->{$logfile}++ > $maxlines){ + print "Truncating $logfile\n"; + truncate $loghandles->{$logfile}, 0; + $cnt->{$logfile} = 0; + } + print { $loghandles->{$logfile} } $msg . "\n"; + $loghandles->{$logfile}->flush; +}; diff --git a/roles/g2cs/handlers/main.yml b/roles/g2cs/handlers/main.yml new file mode 100644 index 0000000..4715c8a --- /dev/null +++ b/roles/g2cs/handlers/main.yml @@ -0,0 +1,4 @@ +--- + +- name: restart g2cs + service: name=g2cs state=restarted diff --git a/roles/g2cs/tasks/install.yml b/roles/g2cs/tasks/install.yml new file mode 100644 index 0000000..5245e92 --- /dev/null +++ b/roles/g2cs/tasks/install.yml @@ -0,0 +1,37 @@ +--- + +- name: Install dependencies + yum: + name: + - perl-IO + - perl-Getopt-Long + tags: g2cs + +- name: Install main script + copy: src=g2cs.pl dest=/usr/local/bin/g2cs mode=755 + tags: g2cs + +- name: Deploy systemd unit + template: src=g2cs.service.j2 dest=/etc/systemd/system/g2cs.service + notify: restart g2cs + register: g2cs_unit + tags: g2cs + +- name: Reload systemd + systemd: daemon_reload=True + when: g2cs_unit.changed + tags: g2cs + +- name: Deploy tmpfiles.d config + copy: + content: | + d /run/g2cs 0755 g2cs g2cs - - + d /run/g2cs/logs 0700 g2cs g2cs - - + dest: /etc/tmpfiles.d/g2cs.conf + register: g2cs_tmpfiles + tags: g2cs + +- name: Create tmpfiles dir + command: systemd-tmpfiles --create + when: g2cs_tmpfiles.changed + tags: g2cs diff --git a/roles/g2cs/tasks/iptables.yml b/roles/g2cs/tasks/iptables.yml new file mode 100644 index 0000000..32a4676 --- /dev/null +++ b/roles/g2cs/tasks/iptables.yml @@ -0,0 +1,8 @@ +--- + +- name: Handle g2cs port in the firewall + iptables_raw: + name: g2cs_port + state: "{{ (g2cs_src_ip | length > 0) | ternary('present','absent') }}" + rules: "-A INPUT -p udp --dport {{ g2cs_port }} -s {{ g2cs_src_ip | join(',') }} -j ACCEPT" + tags: firewall,g2cs diff --git a/roles/g2cs/tasks/main.yml b/roles/g2cs/tasks/main.yml new file mode 100644 index 0000000..472b41c --- /dev/null +++ b/roles/g2cs/tasks/main.yml @@ -0,0 +1,6 @@ +--- + +- include: install.yml +- include: iptables.yml + when: iptables_manage | default(True) +- include: service.yml diff --git a/roles/g2cs/tasks/service.yml b/roles/g2cs/tasks/service.yml new file mode 100644 index 0000000..2d2324d --- /dev/null +++ b/roles/g2cs/tasks/service.yml @@ -0,0 +1,5 @@ +--- + +- name: Start and enable the service + service: name=g2cs state=started enabled=True + tags: g2cs diff --git a/roles/g2cs/templates/g2cs.service.j2 b/roles/g2cs/templates/g2cs.service.j2 new file mode 100644 index 0000000..f102b8a --- /dev/null +++ b/roles/g2cs/templates/g2cs.service.j2 @@ -0,0 +1,25 @@ +[Unit] +Description=Graylog to Crowdsec syslog daemon +After=syslog.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/g2cs --port={{ g2cs_port }} --logdir={{ g2cs_log_dir }} +User=g2cs +Group=g2cs +Restart=always +PrivateTmp=yes +PrivateDevices=yes +ProtectSystem=full +ProtectHome=yes +NoNewPrivileges=yes +SyslogIdentifier=g2cs + +# Allow binding on privileged ports +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +AmbientCapabilities=CAP_NET_BIND_SERVICE + + +[Install] +WantedBy=multi-user.target +