From ea06929d4f0e2488e1ea1b78f16301a340a2a342 Mon Sep 17 00:00:00 2001 From: marius Date: Mon, 24 Apr 2023 16:45:27 +0200 Subject: [PATCH] added rsyslog lab work for logging project --- rsyslog/Readme.txt | 1 + rsyslog/rsyslog/manage_rsyslog_logrotate.yaml | 37 ++++++++++++ rsyslog/rsyslog/pgsql.conf | 17 ++++++ rsyslog/rsyslog/rsyslog-json | 13 ++++ rsyslog/rsyslog/rsyslog.conf | 60 +++++++++++++++++++ rsyslog/rsyslog/run.sh | 10 ++++ 6 files changed, 138 insertions(+) create mode 100644 rsyslog/Readme.txt create mode 100644 rsyslog/rsyslog/manage_rsyslog_logrotate.yaml create mode 100644 rsyslog/rsyslog/pgsql.conf create mode 100644 rsyslog/rsyslog/rsyslog-json create mode 100644 rsyslog/rsyslog/rsyslog.conf create mode 100755 rsyslog/rsyslog/run.sh diff --git a/rsyslog/Readme.txt b/rsyslog/Readme.txt new file mode 100644 index 0000000..0ba02de --- /dev/null +++ b/rsyslog/Readme.txt @@ -0,0 +1 @@ +Config dump from my lab, passwords are not real. diff --git a/rsyslog/rsyslog/manage_rsyslog_logrotate.yaml b/rsyslog/rsyslog/manage_rsyslog_logrotate.yaml new file mode 100644 index 0000000..bdf9e0c --- /dev/null +++ b/rsyslog/rsyslog/manage_rsyslog_logrotate.yaml @@ -0,0 +1,37 @@ +--- +- name: Manage rsyslog and logrotate configurations locally on Ubuntu 20.04 LTS + hosts: localhost + become: yes + connection: local + gather_facts: yes + tasks: + - name: Install rsyslog and logrotate packages + ansible.builtin.package: + name: + - rsyslog + - logrotate + state: present + + - name: Copy rsyslog configuration file + ansible.builtin.copy: + src: rsyslog.conf + dest: /etc/rsyslog.conf + owner: root + group: root + mode: 0644 + notify: restart rsyslog + + - name: Copy logrotate configuration file for rsyslog JSON logs + ansible.builtin.copy: + src: rsyslog-json + dest: /etc/logrotate.d/rsyslog-json + owner: root + group: root + mode: 0644 + + handlers: + - name: restart rsyslog + ansible.builtin.systemd: + name: rsyslog + state: restarted + diff --git a/rsyslog/rsyslog/pgsql.conf b/rsyslog/rsyslog/pgsql.conf new file mode 100644 index 0000000..026a554 --- /dev/null +++ b/rsyslog/rsyslog/pgsql.conf @@ -0,0 +1,17 @@ +### Configuration file for rsyslog-pgsql +### Changes are preserved + +module (load="ompgsql") + +#came from package +# *.* action(type="ompgsql" server="localhost" db="Syslog" uid="rsyslog" pwd="test") + +# Legacy template for PostgreSQL +# $template pgsqlLogFormat,"INSERT INTO logs (log_data) VALUES ('%msg:jsonLogFormat%')",SQL +$template pgsqlCombinedTemplate,"INSERT INTO logs (log_data) VALUES ('{\"timestamp\":\"%timereported:::date-rfc3339%\",\"message\":\"%msg:::json%\",\"host\":\"%hostname:::json%\",\"severity\":\"%syslogseverity-text:::json%\",\"facility\":\"%syslogfacility-text:::json%\",\"syslogtag\":\"%syslogtag:::json%\"}')",SQL + +# Save incoming logs to PostgreSQL DB with caching +if $fromhost-ip != '127.0.0.1' then { + action(type="ompgsql" server="localhost" user="myuser" pass="mypassword" db="logs" template="pgsqlCombinedTemplate" queue.type="LinkedList" queue.size="10000" queue.workerThreads="2" queue.dequeueBatchSize="100" queue.highWatermark="8000" queue.lowWatermark="2000" queue.discardSeverity="0" queue.discardMark="9750") +} + diff --git a/rsyslog/rsyslog/rsyslog-json b/rsyslog/rsyslog/rsyslog-json new file mode 100644 index 0000000..b16f312 --- /dev/null +++ b/rsyslog/rsyslog/rsyslog-json @@ -0,0 +1,13 @@ +/var/log/remote/*/*/*.log /var/log/remote/*/*/*.json { + daily + missingok + rotate 7 + compress + delaycompress + notifempty + create 0640 root adm + postrotate + invoke-rc.d rsyslog rotate >/dev/null 2>&1 || true + endscript +} + diff --git a/rsyslog/rsyslog/rsyslog.conf b/rsyslog/rsyslog/rsyslog.conf new file mode 100644 index 0000000..f2889eb --- /dev/null +++ b/rsyslog/rsyslog/rsyslog.conf @@ -0,0 +1,60 @@ +# Load input modules (Choose TCP or UDP) +module(load="imtcp") +input(type="imtcp" port="514") + +# OR + +module(load="imudp") +input(type="imudp" port="514") + +# Queue configuration for caching +$ActionQueueType LinkedList +$ActionQueueSize 10000 +$ActionQueueWorkerThreads 2 +$ActionQueueDequeueBatchSize 100 +$ActionQueueHighWatermark 8000 +$ActionQueueLowWatermark 2000 +$ActionQueueDiscardSeverity 0 +$ActionQueueDiscardMark 9750 + +# JSON log format template +template(name="jsonLogFormat" type="list") { + constant(value="{") + constant(value="\"timestamp\":\"") property(name="timereported" dateFormat="rfc3339") + constant(value="\",\"message\":\"") property(name="msg") + constant(value="\",\"host\":\"") property(name="hostname") + constant(value="\",\"severity\":\"") property(name="syslogseverity-text") + constant(value="\",\"facility\":\"") property(name="syslogfacility-text") + constant(value="\",\"syslogtag\":\"") property(name="syslogtag") + constant(value="\"}") +} + + +# Dynamic file name template based on date, host, and application +template(name="DynamicFile" type="list") { + constant(value="/var/log/remote/") + property(name="timereported" dateFormat="year") + constant(value="/") + property(name="timereported" dateFormat="month") + constant(value="/") + property(name="timereported" dateFormat="day") + constant(value="/") + property(name="hostname") + constant(value="/") + property(name="programname") + constant(value=".log") +} + + +# Save incoming logs to dynamic file names with caching +if $fromhost-ip != '127.0.0.1' then { + action(type="omfile" dynaFile="DynamicFile" template="jsonLogFormat" queue.type="LinkedList" queue.size="10000" queue.workerThreads="2" queue.dequeueBatchSize="100" queue.highWatermark="8000" queue.lowWatermark="2000" queue.discardSeverity="0" queue.discardMark="9750") +} + +include(file="/etc/rsyslog.d/pgsql.conf") + + + + + + diff --git a/rsyslog/rsyslog/run.sh b/rsyslog/rsyslog/run.sh new file mode 100755 index 0000000..e3e95f2 --- /dev/null +++ b/rsyslog/rsyslog/run.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +playbook_file=$1 + +ansible-playbook $playbook_file --ask-become-pass