Add loki.org

This commit is contained in:
Jeffrey Serio 2024-10-04 19:40:16 -05:00
parent a30ee48fe8
commit 0a2983c2b2

179
loki.org Normal file
View File

@ -0,0 +1,179 @@
#+title: Loki
** Rsyslog forwarding to Promtail and Loki
#+BEGIN_QUOTE
Running Loki and Promtail on the same host as Prometheus makes managing the firewall and network routes easier.
#+END_QUOTE
This is roughly what our network looks like:
*Main Monitoring Node*
- Runs Prometheus, Promtail, Loki, and rsyslog.
- Traffic must be allowed through the firewall on TCP port 514. If using Tailscale, ensure the ACLs are setup correctly.
- It has an rsyslog ruleset that catches all forwarded logs through TCP port 514 and relays them to Promtail on TCP port 1514.
- Promtail pushes the logs its receives via TCP port 1514 to the Loki client listening on TCP port 3100.
*Regular Node 1*
- It has an rsyslog ruleset that forwards logs to the Main Monitoring Node on TCP port 514.
- Is allowed to access TCP port 514 on the Main Monitoring Node.
*Regular Node 2*
- It has an rsyslog ruleset that forwards logs to the Main Monitoring Node on TCP port 514.
- Is allowed to access TCP port 514 on the Main Monitoring Node.
*** Install Rsyslog, Promtail, and Loki on the Main Monitoring Node
#+BEGIN_SRC shell
# Debian-based hosts
sudo apt install -y promtail loki rsyslog
# Fedora-based hosts
sudo dnf install -y promtail loki rsyslog
#+END_SRC
Edit ~/etc/promtail/config.yml~.
#+BEGIN_SRC yaml
server:
http_listen_port: 9081
grpc_listen_port: 0
positions:
filename: /var/tmp/promtail-syslog-positions.yml
clients:
- url: http://localhost:3100/loki/api/v1/push
scrape_configs:
- job_name: syslog
syslog:
listen_address: 0.0.0.0:1514
labels:
job: syslog
relabel_configs:
- source_labels: [__syslog_message_hostname]
target_label: hostname
- source_labels: [__syslog_message_severity]
target_label: level
- source_labels: [__syslog_message_app_name]
target_label: application
- source_labels: [__syslog_message_facility]
target_label: facility
- source_labels: [__syslog_connection_hostname]
target_label: connection_hostname
#+END_SRC
Edit ~/etc/loki/config.yml~.
#+BEGIN_SRC yaml
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
common:
instance_addr: 127.0.0.1
path_prefix: /tmp/loki
storage:
filesystem:
chunks_directory: /tmp/loki/chunks
rules_directory: /tmp/loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
query_range:
results_cache:
cache:
embedded_cache:
enabled: true
max_size_mb: 100
schema_config:
configs:
- from: 2020-10-24
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
ruler:
alertmanager_url: http://localhost:9093
#+END_SRC
Edit ~/etc/rsyslog.d/00-promtail-relay.conf~.
#+BEGIN_SRC rsyslog
# https://www.rsyslog.com/doc/v8-stable/concepts/multi_ruleset.html#split-local-and-remote-logging
ruleset(name="remote"){
# https://www.rsyslog.com/doc/v8-stable/configuration/modules/omfwd.html
# https://grafana.com/docs/loki/latest/clients/promtail/scraping/#rsyslog-output-configuration
action(type="omfwd" Target="localhost" Port="1514" Protocol="tcp" Template="RSYSLOG_SyslogProtocol23Format" TCP_Framing="octet-counted")
}
# https://www.rsyslog.com/doc/v8-stable/configuration/modules/imudp.html
module(load="imudp")
input(type="imudp" port="514" ruleset="remote")
# https://www.rsyslog.com/doc/v8-stable/configuration/modules/imtcp.html
module(load="imtcp")
input(type="imtcp" port="514" ruleset="remote")
#+END_SRC
Ensure the firewall allows TCP traffic to port 514.
#+BEGIN_SRC shell
sudo firewall-cmd --permanent --zone=tailnet --add-port=514/tcp
sudo firewall-cmd --reload
#+END_SRC
Restart and/or enable the services.
#+BEGIN_SRC shell
sudo systemctl enable --now promtail.service
sudo systemctl enable --now loki.service
sudo systemctl enable --now rsyslog.service
#+END_SRC
*** Install and configure Rsyslog on Regular Node 1 and Regular Node 2
#+BEGIN_SRC shell
# Debian
sudo apt install -y rsyslog
# Fedora
sudo dnf install -y rsyslog
#+END_SRC
Enable and start the rsyslog service.
#+BEGIN_SRC shell
sudo systemctl enable --now rsyslog
#+END_SRC
Edit ~/etc/rsyslog.conf~.
#+BEGIN_SRC rsyslog
###############
#### RULES ####
###############
# Forward to Main Monitoring Node
*.* action(type="omfwd" target="<IP addr of Main Monitoring Node>" port="514" protocol="tcp"
action.resumeRetryCount="100"
queue.type="linkedList" queue.size="10000")
#+END_SRC
Restart the rsyslog service.
#+BEGIN_SRC shell
sudo systemctl restart rsyslog.service
#+END_SRC
In the Grafana UI, you should now be able to add Loki as a data source. Then go to Home > Explore > loki and start querying logs from Regular Node 1 and Regular Node 2.