From 0d2cff0d96a334e822d3cdebfdac4ea05acbe4d7 Mon Sep 17 00:00:00 2001 From: Daniel Berteaud Date: Mon, 1 Mar 2021 20:00:07 +0100 Subject: [PATCH] Update to 2021-03-01 20:00 --- roles/crowdsec/defaults/main.yml | 17 ++- roles/crowdsec/tasks/.conf.yml.swp | Bin 0 -> 16384 bytes roles/crowdsec/tasks/conf.yml | 1 + roles/crowdsec/tasks/directories.yml | 1 + roles/crowdsec/templates/acquis.yaml.j2 | 15 +-- roles/crowdsec/templates/acquis/system.yaml.j2 | 5 + roles/crowdsec/templates/config.yaml.j2 | 4 + roles/crowdsec/templates/dev.yaml.j2 | 36 ++++++ roles/g2cs/README.md | 17 +++ roles/g2cs/defaults/main.yml | 11 ++ roles/g2cs/files/g2cs.pl | 165 +++++++++++++++++++++++++ roles/g2cs/handlers/main.yml | 4 + roles/g2cs/tasks/install.yml | 37 ++++++ roles/g2cs/tasks/iptables.yml | 8 ++ roles/g2cs/tasks/main.yml | 6 + roles/g2cs/tasks/service.yml | 5 + roles/g2cs/templates/g2cs.service.j2 | 25 ++++ 17 files changed, 345 insertions(+), 12 deletions(-) create mode 100644 roles/crowdsec/tasks/.conf.yml.swp create mode 100644 roles/crowdsec/templates/acquis/system.yaml.j2 create mode 100644 roles/crowdsec/templates/dev.yaml.j2 create mode 100644 roles/g2cs/README.md create mode 100644 roles/g2cs/defaults/main.yml create mode 100644 roles/g2cs/files/g2cs.pl create mode 100644 roles/g2cs/handlers/main.yml create mode 100644 roles/g2cs/tasks/install.yml create mode 100644 roles/g2cs/tasks/iptables.yml create mode 100644 roles/g2cs/tasks/main.yml create mode 100644 roles/g2cs/tasks/service.yml create mode 100644 roles/g2cs/templates/g2cs.service.j2 diff --git a/roles/crowdsec/defaults/main.yml b/roles/crowdsec/defaults/main.yml index 700d6e1..e9027fc 100644 --- a/roles/crowdsec/defaults/main.yml +++ b/roles/crowdsec/defaults/main.yml @@ -1,11 +1,11 @@ --- # Version to install -cs_version: 1.0.7 +cs_version: 1.0.8 # URL of the archive cs_archive_url: https://github.com/crowdsecurity/crowdsec/releases/download/v{{ cs_version }}/crowdsec-release.tgz # Expected sha1 of the archive -cs_archive_sha1: 7c9dc58c6648c8fd43b297427d6a53fe940cbf13 +cs_archive_sha1: 060782df0b6a8a799c1c0e6efc874b26ca9988e6 # Can be sqlite or mysql cs_db_engine: sqlite @@ -69,3 +69,16 @@ cs_postoverflows: [] # - crowdsecurity/rdns # - crowdsecurity/seo-bots-whitelist +# If not set, crowdsec will look for yaml files in /etc/crowdsec/acquis/ +# The default will only read syslog using journalctl +# If defined, only acquisition set by ansible will be used +# cs_aquis: +# - journalctl_filter: +# - '_SYSTEMD_UNIT=sshd.service' +# labels: +# type: syslog +# +# - filename: +# - /var/log/nginx/access.log +# labels: +# type: nginx diff --git a/roles/crowdsec/tasks/.conf.yml.swp b/roles/crowdsec/tasks/.conf.yml.swp new file mode 100644 index 0000000000000000000000000000000000000000..073f5b9de76c96248e901001afac891769fd9986 GIT binary patch literal 16384 zcmeI2ONbmr7{|*bMiY&Snu91(*$AtLnO--bam2Sm(4CpFzQGo9^Mx@vY= z;wECm2#O-8U{0b21%(`iToggUgNF#}#Ron>h>%MVm3Z)9)sN}vomV6Wk!twS)791W zef6(z>Z_iq#?)i8kJ7ExX@cVhLN09Bc95=qeAOq1*Aj;X?vA+!_RQ{{d$78D-@~d( zzoW%tpSc0AdMt87k5|LE)vw;sa)VaGZCSh`d{?%urt5KGbp(%vr7Bt?wk;M2x8ZRs zMzg}Q<8aXt+_oeW^SBIyW_8K;`q?sNMggNhL4l5JR<>@~zh(OFDOI|8ViUdT#{Go~ z=A%)-C}0#Y3K#{90!9I&fKk9G@PAi8Mytp%baGYNQJJ2vD4kcQ*J}pUON-6PC}0#Y z3K#{90!9I&fKk9GU=%P47zK<1MuGpJ0?bL?{m-ixe*FKRz5oAxJs}st_uwsX5wza1y)$UI9k{2X}y5!G-mNoB^+c9pKw_gml0baQ%h6I2>A@00xyGS!ErDTwu4{R67o4X4c-B7gBQS)U?;d8tO0*rNywkz8*m!D z2o}IRI0W{93GgX?OnDEy1dfAeKpWf#ZUcYdN0;xwm*6b;06Yy2fSq6~xEI_4e#H+s z&jAh|0Xx8UunFj~)v-sZ9r`{CoEo)-usxSXp^)JMkDFe&C}<~Y#mwRK$PwyF?$hOE zQlSC!d5!K*Z}mJt2&Iy>glY~SgHXE7r5cU77cz%t+cj{<3&W^JC(y1f>OD=s#sn%X zws}xXjG`6l2CAYP2+2H;JH3xog>=G>q)*Xi9QyS`A`I}^XVINZ%GhmmB%hg7UQgbI za?ceK1_~Wt+3(a)zkbgqKJ*Gz>bKoyK?`0cop^|Og2Q6WTQJRICC;F(jB-plPb(o+ zeaweqP8H_b4E1dAh$OytJ*)d627YFSPNqRH3Ez5#P1rnOu^WnX$O?+;Q0m?vBaHIO zB`F6xJk*yF;gegt9HEmpmW{!%u*oZz`gutq(`{r$9Ot)moR12XymrZ(!Kz<+m_#gA zd$Nd99dmrzE?doO&u=ejK!th-QYLkDXJ!`nR}+D@;PxAkGyaRe&)?%#t)iqt0b@T zSa4ZyGF$42*;ZafObF#ErbB&QXr`{cR23G(*ijh~z3$tH+PXNj5ie{ZujMV+VT4=> zFC_#S`e4C&Ke2o+nU(jWt4-qQ_+~QyhTBlxEZbPAXj#Dl{VRRtl-8L*r36|Y7d;Fs zrbAULPrPb2jM*ZsG^yqA1uI!L=m)b^&Av28BZ{ePbBYuvW9Ct8dPThk;(WfAK9I$R z?!PP4Vh2Js5&I4e<2-cKSkd0wr3R2iv8C1Nma1*D^{lj#EQLmet$wBuw)(m(dbJQZ znYlheH}ctm0g-z#G$8u6p9Vzv-q3(>xW`+VS?V&(7n8aHge2CmY09BEde&ZL8(NAL zq2n{VjlmR@IgZNc7;pWc9`EI4TB)#(3~_3PvCY*P8i2bgA6Q+IbhZ zJr?sU$n@;l8)E6G=SKTQ5t=NlwbdMyxtPTcI-Zm|4a5^Bn`A-fFNwpRTF&WYjWD=e{urMjaC|;(i))G_G6&}Q{-L_=h!9z~nje07^q29_~EEBHZK}vMP zpjeuiE|w%ID=X0p8r7Ep33X)dgsfQrCfQME#7?X|A*52N5cU4=;vN1qpx*!S+NqBp zaD4{6ncjO<-8Y9(z$jo8FbWt2i~>dhqkvJsC}0#Y3K#{90+&yLF@L`J7yr%l{M)Pk G8~F#?aC&M0 literal 0 HcmV?d00001 diff --git a/roles/crowdsec/tasks/conf.yml b/roles/crowdsec/tasks/conf.yml index 61e77b7..d903838 100644 --- a/roles/crowdsec/tasks/conf.yml +++ b/roles/crowdsec/tasks/conf.yml @@ -8,6 +8,7 @@ - simulation.yaml - profiles.yaml - parsers/s02-enrich/trusted_ip.yaml + - dev.yaml notify: reload crowdsec tags: cs diff --git a/roles/crowdsec/tasks/directories.yml b/roles/crowdsec/tasks/directories.yml index 86fe46c..85bc09f 100644 --- a/roles/crowdsec/tasks/directories.yml +++ b/roles/crowdsec/tasks/directories.yml @@ -14,4 +14,5 @@ - dir: /etc/crowdsec/scenarios - dir: /etc/crowdsec/postoverflows/s00-enrich - dir: /etc/crowdsec/postoverflows/s01-whitelist + - dir: /etc/crowdsec/acquis tags: cs diff --git a/roles/crowdsec/templates/acquis.yaml.j2 b/roles/crowdsec/templates/acquis.yaml.j2 index 513d7bb..152bda7 100644 --- a/roles/crowdsec/templates/acquis.yaml.j2 +++ b/roles/crowdsec/templates/acquis.yaml.j2 @@ -1,11 +1,6 @@ +{% if cs_acquis is defined and cs_acquis | length > 0%} +{% for acquis in cs_acquis %} --- -journalctl_filter: - - "_SYSTEMD_UNIT=sshd.service" -labels: - type: syslog ---- -journalctl_filter: - - "_TRASPORT=kernel" -labels: - type: syslog - +{{ acquis | to_nice_yaml }} +{% endfor %} +{% endif %} diff --git a/roles/crowdsec/templates/acquis/system.yaml.j2 b/roles/crowdsec/templates/acquis/system.yaml.j2 new file mode 100644 index 0000000..b8b8149 --- /dev/null +++ b/roles/crowdsec/templates/acquis/system.yaml.j2 @@ -0,0 +1,5 @@ +--- +journalctl_filter: + - "" +labels: + type: syslog diff --git a/roles/crowdsec/templates/config.yaml.j2 b/roles/crowdsec/templates/config.yaml.j2 index e7c11e2..246b6f6 100644 --- a/roles/crowdsec/templates/config.yaml.j2 +++ b/roles/crowdsec/templates/config.yaml.j2 @@ -13,7 +13,11 @@ config_paths: index_path: /etc/crowdsec/hub/.index.json crowdsec_service: +{% if cs_acquis is defined %} acquisition_path: /etc/crowdsec/acquis.yaml +{% else %} + acquisition_dir: /etc/crowdsec/acquis/ +{% endif %} parser_routines: 1 cscli: diff --git a/roles/crowdsec/templates/dev.yaml.j2 b/roles/crowdsec/templates/dev.yaml.j2 new file mode 100644 index 0000000..abd5b45 --- /dev/null +++ b/roles/crowdsec/templates/dev.yaml.j2 @@ -0,0 +1,36 @@ +common: + daemonize: false + log_media: stdout + log_level: info + working_dir: . + +config_paths: + config_dir: /etc/crowdsec/ + data_dir: /var/lib/crowdsec/data/ + simulation_path: /etc/crowdsec/simulation.yaml + hub_dir: /etc/crowdsec/hub/ + index_path: /etc/crowdsec/hub/.index.json + +crowdsec_service: + acquisition_path: /etc/crowdsec/acquis.yaml + parser_routines: 1 + +cscli: + output: human + hub_branch: master + +db_config: + log_level: info + db_path: /var/lib/crowdsec/data/dev.db + flush: + max_items: 100000 + max_age: 730d + +api: + client: + insecure_skip_verify: false + credentials_path: /etc/crowdsec/local_api_credentials.yaml + +prometheus: + enabled: false + diff --git a/roles/g2cs/README.md b/roles/g2cs/README.md new file mode 100644 index 0000000..a5f3a29 --- /dev/null +++ b/roles/g2cs/README.md @@ -0,0 +1,17 @@ +# G2CS + +This is a small daemon writtent in perl to allow a bridge between Graylog and Crowdsec. +This idea is that if you collect your logs to a graylog instance, you can forward them all in a single stream from Graylog to CrowdSec, instead of collecting them all again on every hosts. + +So, this small g2cs daemon is a very simple perl utility which will listen on a port for a syslog stream. It should run a the server which will host your single crowdsec instance. + +On graylog, you have to install the syslog-output plugin, and configure it to output the streams you want to this daemon. You should choose UDP, the port on which g2cs binds, and the CEF format. + +When g2cs receive this stream of logs, it'll just make simple transformations so that your logs can be consumed by crowdsec : + + * nginx logs go to nginx/ + * httpd logs go to httpd/ + * squid logs go to squid/ + * Everything else goes to syslog.log + +Now, you can configure your acquisitions on crowdsec to just read these locations diff --git a/roles/g2cs/defaults/main.yml b/roles/g2cs/defaults/main.yml new file mode 100644 index 0000000..f2a19d7 --- /dev/null +++ b/roles/g2cs/defaults/main.yml @@ -0,0 +1,11 @@ +--- + +# Port on which g2cs will listen +g2cs_port: 3514 + +# Where log files will be created. Thos files won't grow too large as g2cs truncates them after 10000 lines +# so better to use a tmpfs +g2cs_log_dir: /run/g2cs/logs + +# List of IP/CIDR for which g2cs port will be reachable +g2cs_src_ip: [] diff --git a/roles/g2cs/files/g2cs.pl b/roles/g2cs/files/g2cs.pl new file mode 100644 index 0000000..ddd01dc --- /dev/null +++ b/roles/g2cs/files/g2cs.pl @@ -0,0 +1,165 @@ +#!/usr/bin/perl -w + +use IO::Socket; +use Getopt::Long; +use File::Basename; +use File::Path qw(make_path); +use IO::Handle; + +my $maxlen = 16384; +my $port = 514; +my $maxlines = 10000; +my $logdir = '/run/cs-gelf-server/'; + +GetOptions( + "port=i" => \$port, + "maxlines=i" => \$maxlines, + "logdir=s" => \$logdir +); + +if ($port !~ /^\d+$/ or $port < 1 or $port > 65535){ + die "Invalid port $port\n"; +} +if ($maxlines !~ /^\d+/ or $maxlines < 10){ + die "Invalid max line specified\n"; +} +if (not -d $logdir){ + die "$logdir doesn't exists or is not a directory\n"; +} + +# Remove trailing / of the logdir, it's not nice in the logs when you have double / +$logdir =~ s/\/$//; + +# List of syslog_identifier we're not intersted in +my @ignored_syslog_id = qw( + c-icap + charon + unbound + sudo + zed + zimbramon +); +# List of log files we're not interested in +my @ignored_log_files = qw( + /var/log/audit/audit.log + /var/log/squid/cache.log + /var/log/ufdbGuard/ufdbguardd.log + /opt/zimbra/log/gc.log +); + +print "Start listening on UDP port $port\n"; +$sock = IO::Socket::INET->new( + LocalPort => $port, + Proto => 'udp' + ) or die("Socket: $@"); + +my $buf; +my $cnt = {}; +my $loghandles = {}; + +while (1) { + $sock->recv($buf, $maxlen); + my ($port, $ipaddr) = sockaddr_in($sock->peername); + my $fields = {}; + + # We're not really interested in CEF headers. So let's extract + # the various fields + $buf =~ m/(?:(?:CEF:\d+\|)(?:[^=\\]+\|)+)(.*)/; + my $ext = $1; + + # Taken from https://github.com/DavidJBianco/pycef + while ($ext =~ m/([^=\s]+)=((?:[\\]=|[^=])+)(?:\s|$)/g) { + $fields->{$1} = $2; + # Unescape value string + $fields->{$1} =~ s/\\=/=/g; + } + + # Skip lines we're not interested in early. + # So crowdsec will eat less CPU parsing useless stuff + if ( + defined $fields->{syslog_identifier} and grep { $_ eq $fields->{syslog_identifier} } @ignored_syslog_id or + defined $fields->{log_file_path} and grep { $_ eq $fields->{log_file_path} } @ignored_log_files + ) { + next; + } + + # We need a timestamp, a source and a msg at least + if (not defined $fields->{timestamp} or not defined $fields->{source} or not defined $fields->{msg}){ + next; + } + + my $msg; + # Default log will be syslog + my $logfile = $logdir . '/syslog.log'; + + # But for some services, we need special handling. Eg for web access logs + if (defined $fields->{event_dataset}){ + if ($fields->{event_dataset} =~ m/^nginx\.(access|ingress_controller)/){ + $logfile = $logdir . '/nginx/access.log'; + $msg = $fields->{msg}; + } elsif ($fields->{event_dataset} =~ m/^nginx\.error/){ + $logfile = $logdir . '/nginx/error.log'; + $msg = $fields->{msg}; + } elsif ($fields->{event_dataset} =~ m/^apache\.access/){ + $logfile = $logdir . '/httpd/access.log'; + $msg = $fields->{msg}; + } elsif ($fields->{event_dataset} =~ m/^apache\.error/){ + $logfile = $logdir . '/httpd/access.log'; + $msg = $fields->{msg}; + } + } elsif (defined $fields->{log_file_path}){ + if ($fields->{log_file_path} eq '/var/log/pveproxy/access.log'){ + $logfile = $logdir . '/pveproxy/access.log'; + $msg = $fields->{msg}; + } elsif ($fields->{log_file_path} eq '/var/log/squid/access.log'){ + $logfile = $logdir . '/squid/access.log'; + $msg = $fields->{msg}; + } elsif ($fields->{log_file_path} eq '/opt/zimbra/log/nginx.access.log'){ + $logfile = $logdir . '/nginx/access.log'; + $msg = $fields->{msg}; + } + } elsif (defined $fields->{application_name}){ + if ($fields->{application_name} eq 'nginx'){ + $logfile = $logdir . '/nginx/access.log'; + $msg = $fields->{msg}; + } + } + + # OK, no special handling (else $msg would be defined), so let's + # provide a syslog format + if (not defined $msg){ + $msg .= $fields->{timestamp} . ' ' . $fields->{source} . ' '; + my $id = $fields->{syslog_identifier} || $fields->{program} || $fields->{application_name} || $fields->{process_name} || 'unknown'; + # For older PfSense, which sent invalid syslog messages, we might extract + # the syslog identifier from the begining of the message + if ($id eq 'unknown' and $fields->{msg} =~ m/^(\w+(\[\d+\])?):\s(.*)/){ + $id = $1; + $fields->{msg} = $3; + } + $msg .= $id; + # Try to append the pid of the process + if ($id ne 'kernel' and $id ne 'filterlog' and $id !~ m/\[\d+\]$/){ + $msg .= '['; + $msg .= $fields->{process_pid} || $fields->{process_id} || $fields->{pid} || '0'; + $msg .= ']'; + } + $msg .= ': ' . $fields->{msg}; + } + + # Create the log sub dir if needed + my $dir = dirname($logfile); + if (not -d $dir){ + make_path($dir); + } + + defined $loghandles->{$logfile} or open($loghandles->{$logfile}, ">>", $logfile); + # Truncate the file so it's not growing too large + # Crowdsec will read it in nearly real time anyway + if ($cnt->{$logfile}++ > $maxlines){ + print "Truncating $logfile\n"; + truncate $loghandles->{$logfile}, 0; + $cnt->{$logfile} = 0; + } + print { $loghandles->{$logfile} } $msg . "\n"; + $loghandles->{$logfile}->flush; +}; diff --git a/roles/g2cs/handlers/main.yml b/roles/g2cs/handlers/main.yml new file mode 100644 index 0000000..4715c8a --- /dev/null +++ b/roles/g2cs/handlers/main.yml @@ -0,0 +1,4 @@ +--- + +- name: restart g2cs + service: name=g2cs state=restarted diff --git a/roles/g2cs/tasks/install.yml b/roles/g2cs/tasks/install.yml new file mode 100644 index 0000000..5245e92 --- /dev/null +++ b/roles/g2cs/tasks/install.yml @@ -0,0 +1,37 @@ +--- + +- name: Install dependencies + yum: + name: + - perl-IO + - perl-Getopt-Long + tags: g2cs + +- name: Install main script + copy: src=g2cs.pl dest=/usr/local/bin/g2cs mode=755 + tags: g2cs + +- name: Deploy systemd unit + template: src=g2cs.service.j2 dest=/etc/systemd/system/g2cs.service + notify: restart g2cs + register: g2cs_unit + tags: g2cs + +- name: Reload systemd + systemd: daemon_reload=True + when: g2cs_unit.changed + tags: g2cs + +- name: Deploy tmpfiles.d config + copy: + content: | + d /run/g2cs 0755 g2cs g2cs - - + d /run/g2cs/logs 0700 g2cs g2cs - - + dest: /etc/tmpfiles.d/g2cs.conf + register: g2cs_tmpfiles + tags: g2cs + +- name: Create tmpfiles dir + command: systemd-tmpfiles --create + when: g2cs_tmpfiles.changed + tags: g2cs diff --git a/roles/g2cs/tasks/iptables.yml b/roles/g2cs/tasks/iptables.yml new file mode 100644 index 0000000..32a4676 --- /dev/null +++ b/roles/g2cs/tasks/iptables.yml @@ -0,0 +1,8 @@ +--- + +- name: Handle g2cs port in the firewall + iptables_raw: + name: g2cs_port + state: "{{ (g2cs_src_ip | length > 0) | ternary('present','absent') }}" + rules: "-A INPUT -p udp --dport {{ g2cs_port }} -s {{ g2cs_src_ip | join(',') }} -j ACCEPT" + tags: firewall,g2cs diff --git a/roles/g2cs/tasks/main.yml b/roles/g2cs/tasks/main.yml new file mode 100644 index 0000000..472b41c --- /dev/null +++ b/roles/g2cs/tasks/main.yml @@ -0,0 +1,6 @@ +--- + +- include: install.yml +- include: iptables.yml + when: iptables_manage | default(True) +- include: service.yml diff --git a/roles/g2cs/tasks/service.yml b/roles/g2cs/tasks/service.yml new file mode 100644 index 0000000..2d2324d --- /dev/null +++ b/roles/g2cs/tasks/service.yml @@ -0,0 +1,5 @@ +--- + +- name: Start and enable the service + service: name=g2cs state=started enabled=True + tags: g2cs diff --git a/roles/g2cs/templates/g2cs.service.j2 b/roles/g2cs/templates/g2cs.service.j2 new file mode 100644 index 0000000..f102b8a --- /dev/null +++ b/roles/g2cs/templates/g2cs.service.j2 @@ -0,0 +1,25 @@ +[Unit] +Description=Graylog to Crowdsec syslog daemon +After=syslog.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/g2cs --port={{ g2cs_port }} --logdir={{ g2cs_log_dir }} +User=g2cs +Group=g2cs +Restart=always +PrivateTmp=yes +PrivateDevices=yes +ProtectSystem=full +ProtectHome=yes +NoNewPrivileges=yes +SyslogIdentifier=g2cs + +# Allow binding on privileged ports +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +AmbientCapabilities=CAP_NET_BIND_SERVICE + + +[Install] +WantedBy=multi-user.target +