mirror of
https://codeberg.org/hyperreal/techne
synced 2024-11-25 07:43:42 +01:00
Add loki.org
This commit is contained in:
parent
a30ee48fe8
commit
0a2983c2b2
179
loki.org
Normal file
179
loki.org
Normal file
@ -0,0 +1,179 @@
|
||||
#+title: Loki
|
||||
|
||||
** Rsyslog forwarding to Promtail and Loki
|
||||
|
||||
#+BEGIN_QUOTE
|
||||
Running Loki and Promtail on the same host as Prometheus makes managing the firewall and network routes easier.
|
||||
#+END_QUOTE
|
||||
|
||||
This is roughly what our network looks like:
|
||||
|
||||
*Main Monitoring Node*
|
||||
- Runs Prometheus, Promtail, Loki, and rsyslog.
|
||||
- Traffic must be allowed through the firewall on TCP port 514. If using Tailscale, ensure the ACLs are setup correctly.
|
||||
- It has an rsyslog ruleset that catches all forwarded logs through TCP port 514 and relays them to Promtail on TCP port 1514.
|
||||
- Promtail pushes the logs its receives via TCP port 1514 to the Loki client listening on TCP port 3100.
|
||||
|
||||
*Regular Node 1*
|
||||
- It has an rsyslog ruleset that forwards logs to the Main Monitoring Node on TCP port 514.
|
||||
- Is allowed to access TCP port 514 on the Main Monitoring Node.
|
||||
|
||||
*Regular Node 2*
|
||||
- It has an rsyslog ruleset that forwards logs to the Main Monitoring Node on TCP port 514.
|
||||
- Is allowed to access TCP port 514 on the Main Monitoring Node.
|
||||
|
||||
*** Install Rsyslog, Promtail, and Loki on the Main Monitoring Node
|
||||
|
||||
#+BEGIN_SRC shell
|
||||
# Debian-based hosts
|
||||
sudo apt install -y promtail loki rsyslog
|
||||
|
||||
# Fedora-based hosts
|
||||
sudo dnf install -y promtail loki rsyslog
|
||||
#+END_SRC
|
||||
|
||||
Edit ~/etc/promtail/config.yml~.
|
||||
|
||||
#+BEGIN_SRC yaml
|
||||
server:
|
||||
http_listen_port: 9081
|
||||
grpc_listen_port: 0
|
||||
|
||||
positions:
|
||||
filename: /var/tmp/promtail-syslog-positions.yml
|
||||
|
||||
clients:
|
||||
- url: http://localhost:3100/loki/api/v1/push
|
||||
|
||||
scrape_configs:
|
||||
- job_name: syslog
|
||||
syslog:
|
||||
listen_address: 0.0.0.0:1514
|
||||
labels:
|
||||
job: syslog
|
||||
relabel_configs:
|
||||
- source_labels: [__syslog_message_hostname]
|
||||
target_label: hostname
|
||||
- source_labels: [__syslog_message_severity]
|
||||
target_label: level
|
||||
- source_labels: [__syslog_message_app_name]
|
||||
target_label: application
|
||||
- source_labels: [__syslog_message_facility]
|
||||
target_label: facility
|
||||
- source_labels: [__syslog_connection_hostname]
|
||||
target_label: connection_hostname
|
||||
#+END_SRC
|
||||
|
||||
Edit ~/etc/loki/config.yml~.
|
||||
|
||||
#+BEGIN_SRC yaml
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /tmp/loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /tmp/loki/chunks
|
||||
rules_directory: /tmp/loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
query_range:
|
||||
results_cache:
|
||||
cache:
|
||||
embedded_cache:
|
||||
enabled: true
|
||||
max_size_mb: 100
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2020-10-24
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
ruler:
|
||||
alertmanager_url: http://localhost:9093
|
||||
#+END_SRC
|
||||
|
||||
Edit ~/etc/rsyslog.d/00-promtail-relay.conf~.
|
||||
|
||||
#+BEGIN_SRC rsyslog
|
||||
# https://www.rsyslog.com/doc/v8-stable/concepts/multi_ruleset.html#split-local-and-remote-logging
|
||||
ruleset(name="remote"){
|
||||
# https://www.rsyslog.com/doc/v8-stable/configuration/modules/omfwd.html
|
||||
# https://grafana.com/docs/loki/latest/clients/promtail/scraping/#rsyslog-output-configuration
|
||||
action(type="omfwd" Target="localhost" Port="1514" Protocol="tcp" Template="RSYSLOG_SyslogProtocol23Format" TCP_Framing="octet-counted")
|
||||
}
|
||||
|
||||
|
||||
# https://www.rsyslog.com/doc/v8-stable/configuration/modules/imudp.html
|
||||
module(load="imudp")
|
||||
input(type="imudp" port="514" ruleset="remote")
|
||||
|
||||
# https://www.rsyslog.com/doc/v8-stable/configuration/modules/imtcp.html
|
||||
module(load="imtcp")
|
||||
input(type="imtcp" port="514" ruleset="remote")
|
||||
#+END_SRC
|
||||
|
||||
Ensure the firewall allows TCP traffic to port 514.
|
||||
|
||||
#+BEGIN_SRC shell
|
||||
sudo firewall-cmd --permanent --zone=tailnet --add-port=514/tcp
|
||||
sudo firewall-cmd --reload
|
||||
#+END_SRC
|
||||
|
||||
Restart and/or enable the services.
|
||||
|
||||
#+BEGIN_SRC shell
|
||||
sudo systemctl enable --now promtail.service
|
||||
sudo systemctl enable --now loki.service
|
||||
sudo systemctl enable --now rsyslog.service
|
||||
#+END_SRC
|
||||
|
||||
*** Install and configure Rsyslog on Regular Node 1 and Regular Node 2
|
||||
|
||||
#+BEGIN_SRC shell
|
||||
# Debian
|
||||
sudo apt install -y rsyslog
|
||||
|
||||
# Fedora
|
||||
sudo dnf install -y rsyslog
|
||||
#+END_SRC
|
||||
|
||||
Enable and start the rsyslog service.
|
||||
|
||||
#+BEGIN_SRC shell
|
||||
sudo systemctl enable --now rsyslog
|
||||
#+END_SRC
|
||||
|
||||
Edit ~/etc/rsyslog.conf~.
|
||||
|
||||
#+BEGIN_SRC rsyslog
|
||||
###############
|
||||
#### RULES ####
|
||||
###############
|
||||
|
||||
# Forward to Main Monitoring Node
|
||||
*.* action(type="omfwd" target="<IP addr of Main Monitoring Node>" port="514" protocol="tcp"
|
||||
action.resumeRetryCount="100"
|
||||
queue.type="linkedList" queue.size="10000")
|
||||
#+END_SRC
|
||||
|
||||
Restart the rsyslog service.
|
||||
|
||||
#+BEGIN_SRC shell
|
||||
sudo systemctl restart rsyslog.service
|
||||
#+END_SRC
|
||||
|
||||
In the Grafana UI, you should now be able to add Loki as a data source. Then go to Home > Explore > loki and start querying logs from Regular Node 1 and Regular Node 2.
|
Loading…
Reference in New Issue
Block a user