Update to 2021-03-01 20:00

roles/crowdsec/defaults/main.yml

@@ -1,11 +1,11 @@
 ---
 
 # Version to install
-cs_version: 1.0.7
+cs_version: 1.0.8
 # URL of the archive
 cs_archive_url: https://github.com/crowdsecurity/crowdsec/releases/download/v{{ cs_version }}/crowdsec-release.tgz
 # Expected sha1 of the archive
-cs_archive_sha1: 7c9dc58c6648c8fd43b297427d6a53fe940cbf13
+cs_archive_sha1: 060782df0b6a8a799c1c0e6efc874b26ca9988e6
 
 # Can be sqlite or mysql
 cs_db_engine: sqlite
@@ -69,3 +69,16 @@ cs_postoverflows: []
 #  - crowdsecurity/rdns
 #  - crowdsecurity/seo-bots-whitelist
 
+# If not set, crowdsec will look for yaml files in /etc/crowdsec/acquis/
+# The default will only read syslog using journalctl
+# If defined, only acquisition set by ansible will be used
+# cs_aquis:
+#   - journalctl_filter:
+#       - '_SYSTEMD_UNIT=sshd.service'
+#     labels:
+#       type: syslog
+#
+#   - filename:
+#       - /var/log/nginx/access.log
+#     labels:
+#       type: nginx

roles/crowdsec/tasks/conf.yml

@@ -8,6 +8,7 @@
     - simulation.yaml
     - profiles.yaml
     - parsers/s02-enrich/trusted_ip.yaml
+    - dev.yaml
   notify: reload crowdsec
   tags: cs
 

roles/crowdsec/tasks/directories.yml

@@ -14,4 +14,5 @@
     - dir: /etc/crowdsec/scenarios
     - dir: /etc/crowdsec/postoverflows/s00-enrich
     - dir: /etc/crowdsec/postoverflows/s01-whitelist
+    - dir: /etc/crowdsec/acquis
   tags: cs

roles/crowdsec/templates/acquis.yaml.j2

@@ -1,11 +1,6 @@
+{% if cs_acquis is defined and cs_acquis | length > 0%}
+{% for acquis in cs_acquis %}
 ---
-journalctl_filter:
-  - "_SYSTEMD_UNIT=sshd.service"
-labels:
-  type: syslog
----
-journalctl_filter:
-  - "_TRASPORT=kernel"
-labels:
-  type: syslog
-
+{{ acquis | to_nice_yaml }}
+{% endfor %}
+{% endif %}

roles/crowdsec/templates/acquis/system.yaml.j2

@@ -0,0 +1,5 @@
+---
+journalctl_filter:
+  - ""
+labels:
+  type: syslog

roles/crowdsec/templates/config.yaml.j2

@@ -13,7 +13,11 @@ config_paths:
   index_path: /etc/crowdsec/hub/.index.json
 
 crowdsec_service:
+{% if cs_acquis is defined %}
   acquisition_path: /etc/crowdsec/acquis.yaml
+{% else %}
+  acquisition_dir: /etc/crowdsec/acquis/
+{% endif %}
   parser_routines: 1
 
 cscli:

roles/crowdsec/templates/dev.yaml.j2

@@ -0,0 +1,36 @@
+common:
+  daemonize: false
+  log_media: stdout
+  log_level: info
+  working_dir: .
+
+config_paths:
+  config_dir: /etc/crowdsec/
+  data_dir: /var/lib/crowdsec/data/
+  simulation_path: /etc/crowdsec/simulation.yaml
+  hub_dir: /etc/crowdsec/hub/
+  index_path: /etc/crowdsec/hub/.index.json
+
+crowdsec_service:
+  acquisition_path: /etc/crowdsec/acquis.yaml
+  parser_routines: 1
+
+cscli:
+  output: human
+  hub_branch: master
+
+db_config:
+  log_level: info
+  db_path: /var/lib/crowdsec/data/dev.db
+  flush:
+    max_items: 100000
+    max_age: 730d
+
+api:
+  client:
+    insecure_skip_verify: false
+    credentials_path: /etc/crowdsec/local_api_credentials.yaml
+
+prometheus:
+  enabled: false
+

roles/g2cs/README.md

@@ -0,0 +1,17 @@
+# G2CS
+
+This is a small daemon writtent in perl to allow a bridge between Graylog and Crowdsec.
+This idea is that if you collect your logs to a graylog instance, you can forward them all in a single stream from Graylog to CrowdSec, instead of collecting them all again on every hosts.
+
+So, this small g2cs daemon is a very simple perl utility which will listen on a port for a syslog stream. It should run a the server which will host your single crowdsec instance.
+
+On graylog, you have to install the syslog-output plugin, and configure it to output the streams you want to this daemon. You should choose UDP, the port on which g2cs binds, and the CEF format.
+
+When g2cs receive this stream of logs, it'll just make simple transformations so that your logs can be consumed by crowdsec :
+
+  * nginx logs go to nginx/
+  * httpd logs go to httpd/
+  * squid logs go to squid/
+  * Everything else goes to syslog.log
+
+Now, you can configure your acquisitions on crowdsec to just read these locations

roles/g2cs/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+
+# Port on which g2cs will listen
+g2cs_port: 3514
+
+# Where log files will be created. Thos files won't grow too large as g2cs truncates them after 10000 lines
+# so better to use a tmpfs
+g2cs_log_dir: /run/g2cs/logs
+
+# List of IP/CIDR for which g2cs port will be reachable
+g2cs_src_ip: []

roles/g2cs/files/g2cs.pl

@@ -0,0 +1,165 @@
+#!/usr/bin/perl -w
+  
+use IO::Socket;
+use Getopt::Long;
+use File::Basename;
+use File::Path qw(make_path);
+use IO::Handle;
+
+my $maxlen = 16384;
+my $port = 514;
+my $maxlines = 10000;
+my $logdir = '/run/cs-gelf-server/';
+
+GetOptions(
+  "port=i" => \$port,
+  "maxlines=i" => \$maxlines,
+  "logdir=s" => \$logdir
+);
+
+if ($port !~ /^\d+$/ or $port < 1 or $port > 65535){
+  die "Invalid port $port\n";
+}
+if ($maxlines !~ /^\d+/ or $maxlines < 10){
+  die "Invalid max line specified\n";
+}
+if (not -d $logdir){
+  die "$logdir doesn't exists or is not a directory\n";
+}
+
+# Remove trailing / of the logdir, it's not nice in the logs when you have double /
+$logdir =~ s/\/$//;
+
+# List of syslog_identifier we're not intersted in
+my @ignored_syslog_id = qw(
+  c-icap
+  charon
+  unbound
+  sudo
+  zed
+  zimbramon
+);
+# List of log files we're not interested in
+my @ignored_log_files = qw(
+  /var/log/audit/audit.log
+  /var/log/squid/cache.log
+  /var/log/ufdbGuard/ufdbguardd.log
+  /opt/zimbra/log/gc.log
+);
+
+print "Start listening on UDP port $port\n";
+$sock = IO::Socket::INET->new(
+          LocalPort => $port,
+          Proto => 'udp'
+        ) or die("Socket: $@");
+
+my $buf;
+my $cnt = {};
+my $loghandles = {};
+
+while (1) {
+  $sock->recv($buf, $maxlen);
+  my ($port, $ipaddr) = sockaddr_in($sock->peername);
+  my $fields = {};
+
+  # We're not really interested in CEF headers. So let's extract
+  # the various fields
+  $buf =~ m/(?:(?:CEF:\d+\|)(?:[^=\\]+\|)+)(.*)/;
+  my $ext = $1;
+
+  # Taken from https://github.com/DavidJBianco/pycef
+  while ($ext =~ m/([^=\s]+)=((?:[\\]=|[^=])+)(?:\s|$)/g) {
+    $fields->{$1} = $2;
+    # Unescape value string
+    $fields->{$1} =~ s/\\=/=/g;
+  }
+
+  # Skip lines we're not interested in early.
+  # So crowdsec will eat less CPU parsing useless stuff
+  if (
+       defined $fields->{syslog_identifier} and grep { $_ eq $fields->{syslog_identifier} } @ignored_syslog_id or
+       defined $fields->{log_file_path} and grep { $_ eq $fields->{log_file_path} } @ignored_log_files
+     ) {
+    next;
+  }
+
+  # We need a timestamp, a source and a msg at least
+  if (not defined $fields->{timestamp} or not defined $fields->{source} or not defined $fields->{msg}){
+    next;
+  }
+
+  my $msg;
+  # Default log will be syslog
+  my $logfile = $logdir . '/syslog.log';
+
+  # But for some services, we need special handling. Eg for web access logs
+  if (defined $fields->{event_dataset}){
+    if ($fields->{event_dataset} =~ m/^nginx\.(access|ingress_controller)/){
+      $logfile = $logdir . '/nginx/access.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{event_dataset} =~ m/^nginx\.error/){
+      $logfile = $logdir . '/nginx/error.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{event_dataset} =~ m/^apache\.access/){
+      $logfile = $logdir . '/httpd/access.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{event_dataset} =~ m/^apache\.error/){
+      $logfile = $logdir . '/httpd/access.log';
+      $msg = $fields->{msg};
+    }
+  } elsif (defined $fields->{log_file_path}){
+    if ($fields->{log_file_path} eq '/var/log/pveproxy/access.log'){
+      $logfile = $logdir . '/pveproxy/access.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{log_file_path} eq '/var/log/squid/access.log'){
+      $logfile = $logdir . '/squid/access.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{log_file_path} eq '/opt/zimbra/log/nginx.access.log'){
+      $logfile = $logdir . '/nginx/access.log';
+      $msg = $fields->{msg};
+    }
+  } elsif (defined $fields->{application_name}){
+    if ($fields->{application_name} eq 'nginx'){
+      $logfile = $logdir . '/nginx/access.log';
+      $msg = $fields->{msg};
+    }
+  }
+
+  # OK, no special handling (else $msg would be defined), so let's
+  # provide a syslog format
+  if (not defined $msg){
+    $msg .= $fields->{timestamp} . ' ' . $fields->{source} . ' ';
+    my $id = $fields->{syslog_identifier} || $fields->{program} || $fields->{application_name} || $fields->{process_name} || 'unknown';
+    # For older PfSense, which sent invalid syslog messages, we might extract
+    # the syslog identifier from the begining of the message
+    if ($id eq 'unknown' and $fields->{msg} =~ m/^(\w+(\[\d+\])?):\s(.*)/){
+      $id = $1;
+      $fields->{msg} = $3;
+    }
+    $msg .= $id;
+    # Try to append the pid of the process
+    if ($id ne 'kernel' and $id ne 'filterlog' and $id !~ m/\[\d+\]$/){
+      $msg .= '[';
+      $msg .= $fields->{process_pid} || $fields->{process_id} || $fields->{pid} || '0';
+      $msg .= ']';
+    }
+    $msg .= ': ' . $fields->{msg};
+  }
+
+  # Create the log sub dir if needed
+  my $dir = dirname($logfile);
+  if (not -d $dir){
+    make_path($dir);
+  }
+
+  defined $loghandles->{$logfile} or open($loghandles->{$logfile}, ">>", $logfile);
+  # Truncate the file so it's not growing too large
+  # Crowdsec will read it in nearly real time anyway
+  if ($cnt->{$logfile}++ > $maxlines){
+    print "Truncating $logfile\n";
+    truncate $loghandles->{$logfile}, 0;
+    $cnt->{$logfile} = 0;
+  }
+  print { $loghandles->{$logfile} } $msg . "\n";
+  $loghandles->{$logfile}->flush;
+};

roles/g2cs/handlers/main.yml

@@ -0,0 +1,4 @@
+---
+
+- name: restart g2cs
+  service: name=g2cs state=restarted

roles/g2cs/tasks/install.yml

@@ -0,0 +1,37 @@
+---
+
+- name: Install dependencies
+  yum:
+    name:
+      - perl-IO
+      - perl-Getopt-Long
+  tags: g2cs
+
+- name: Install main script
+  copy: src=g2cs.pl dest=/usr/local/bin/g2cs mode=755
+  tags: g2cs
+
+- name: Deploy systemd unit
+  template: src=g2cs.service.j2 dest=/etc/systemd/system/g2cs.service
+  notify: restart g2cs
+  register: g2cs_unit
+  tags: g2cs
+
+- name: Reload systemd
+  systemd: daemon_reload=True
+  when: g2cs_unit.changed
+  tags: g2cs
+
+- name: Deploy tmpfiles.d config
+  copy:
+    content: |
+      d /run/g2cs 0755 g2cs g2cs - -
+      d /run/g2cs/logs 0700 g2cs g2cs - -
+    dest: /etc/tmpfiles.d/g2cs.conf
+  register: g2cs_tmpfiles
+  tags: g2cs
+
+- name: Create tmpfiles dir
+  command: systemd-tmpfiles --create
+  when: g2cs_tmpfiles.changed
+  tags: g2cs

roles/g2cs/tasks/iptables.yml

@@ -0,0 +1,8 @@
+---
+
+- name:  Handle g2cs port in the firewall
+  iptables_raw:
+    name: g2cs_port
+    state: "{{ (g2cs_src_ip | length > 0) | ternary('present','absent') }}"
+    rules: "-A INPUT -p udp --dport {{ g2cs_port }} -s {{ g2cs_src_ip | join(',') }} -j ACCEPT"
+  tags: firewall,g2cs

roles/g2cs/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+
+- include: install.yml
+- include: iptables.yml
+  when: iptables_manage | default(True)
+- include: service.yml

roles/g2cs/tasks/service.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Start and enable the service
+  service: name=g2cs state=started enabled=True
+  tags: g2cs

roles/g2cs/templates/g2cs.service.j2

@@ -0,0 +1,25 @@
+[Unit]
+Description=Graylog to Crowdsec syslog daemon
+After=syslog.target
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/g2cs --port={{ g2cs_port }} --logdir={{ g2cs_log_dir }}
+User=g2cs
+Group=g2cs
+Restart=always
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectSystem=full
+ProtectHome=yes
+NoNewPrivileges=yes
+SyslogIdentifier=g2cs
+
+# Allow binding on privileged ports
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+
+
+[Install]
+WantedBy=multi-user.target
+