diff --git a/loki.org b/loki.org new file mode 100644 index 0000000..0f30873 --- /dev/null +++ b/loki.org @@ -0,0 +1,179 @@ +#+title: Loki + +** Rsyslog forwarding to Promtail and Loki + +#+BEGIN_QUOTE +Running Loki and Promtail on the same host as Prometheus makes managing the firewall and network routes easier. +#+END_QUOTE + +This is roughly what our network looks like: + +*Main Monitoring Node* +- Runs Prometheus, Promtail, Loki, and rsyslog. +- Traffic must be allowed through the firewall on TCP port 514. If using Tailscale, ensure the ACLs are setup correctly. +- It has an rsyslog ruleset that catches all forwarded logs through TCP port 514 and relays them to Promtail on TCP port 1514. +- Promtail pushes the logs its receives via TCP port 1514 to the Loki client listening on TCP port 3100. + +*Regular Node 1* +- It has an rsyslog ruleset that forwards logs to the Main Monitoring Node on TCP port 514. +- Is allowed to access TCP port 514 on the Main Monitoring Node. + +*Regular Node 2* +- It has an rsyslog ruleset that forwards logs to the Main Monitoring Node on TCP port 514. +- Is allowed to access TCP port 514 on the Main Monitoring Node. + +*** Install Rsyslog, Promtail, and Loki on the Main Monitoring Node + +#+BEGIN_SRC shell +# Debian-based hosts +sudo apt install -y promtail loki rsyslog + +# Fedora-based hosts +sudo dnf install -y promtail loki rsyslog +#+END_SRC + +Edit ~/etc/promtail/config.yml~. + +#+BEGIN_SRC yaml +server: + http_listen_port: 9081 + grpc_listen_port: 0 + +positions: + filename: /var/tmp/promtail-syslog-positions.yml + +clients: + - url: http://localhost:3100/loki/api/v1/push + +scrape_configs: + - job_name: syslog + syslog: + listen_address: 0.0.0.0:1514 + labels: + job: syslog + relabel_configs: + - source_labels: [__syslog_message_hostname] + target_label: hostname + - source_labels: [__syslog_message_severity] + target_label: level + - source_labels: [__syslog_message_app_name] + target_label: application + - source_labels: [__syslog_message_facility] + target_label: facility + - source_labels: [__syslog_connection_hostname] + target_label: connection_hostname +#+END_SRC + +Edit ~/etc/loki/config.yml~. + +#+BEGIN_SRC yaml +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +ruler: + alertmanager_url: http://localhost:9093 +#+END_SRC + +Edit ~/etc/rsyslog.d/00-promtail-relay.conf~. + +#+BEGIN_SRC rsyslog +# https://www.rsyslog.com/doc/v8-stable/concepts/multi_ruleset.html#split-local-and-remote-logging +ruleset(name="remote"){ + # https://www.rsyslog.com/doc/v8-stable/configuration/modules/omfwd.html + # https://grafana.com/docs/loki/latest/clients/promtail/scraping/#rsyslog-output-configuration + action(type="omfwd" Target="localhost" Port="1514" Protocol="tcp" Template="RSYSLOG_SyslogProtocol23Format" TCP_Framing="octet-counted") +} + + +# https://www.rsyslog.com/doc/v8-stable/configuration/modules/imudp.html +module(load="imudp") +input(type="imudp" port="514" ruleset="remote") + +# https://www.rsyslog.com/doc/v8-stable/configuration/modules/imtcp.html +module(load="imtcp") +input(type="imtcp" port="514" ruleset="remote") +#+END_SRC + +Ensure the firewall allows TCP traffic to port 514. + +#+BEGIN_SRC shell +sudo firewall-cmd --permanent --zone=tailnet --add-port=514/tcp +sudo firewall-cmd --reload +#+END_SRC + +Restart and/or enable the services. + +#+BEGIN_SRC shell +sudo systemctl enable --now promtail.service +sudo systemctl enable --now loki.service +sudo systemctl enable --now rsyslog.service +#+END_SRC + +*** Install and configure Rsyslog on Regular Node 1 and Regular Node 2 + +#+BEGIN_SRC shell +# Debian +sudo apt install -y rsyslog + +# Fedora +sudo dnf install -y rsyslog +#+END_SRC + +Enable and start the rsyslog service. + +#+BEGIN_SRC shell +sudo systemctl enable --now rsyslog +#+END_SRC + +Edit ~/etc/rsyslog.conf~. + +#+BEGIN_SRC rsyslog +############### +#### RULES #### +############### + +# Forward to Main Monitoring Node +*.* action(type="omfwd" target="" port="514" protocol="tcp" + action.resumeRetryCount="100" + queue.type="linkedList" queue.size="10000") +#+END_SRC + +Restart the rsyslog service. + +#+BEGIN_SRC shell +sudo systemctl restart rsyslog.service +#+END_SRC + +In the Grafana UI, you should now be able to add Loki as a data source. Then go to Home > Explore > loki and start querying logs from Regular Node 1 and Regular Node 2.