Add loki.org

loki.org

@@ -0,0 +1,179 @@
+#+title: Loki
+
+** Rsyslog forwarding to Promtail and Loki
+
+#+BEGIN_QUOTE
+Running Loki and Promtail on the same host as Prometheus makes managing the firewall and network routes easier.
+#+END_QUOTE
+
+This is roughly what our network looks like:
+
+*Main Monitoring Node*
+- Runs Prometheus, Promtail, Loki, and rsyslog.
+- Traffic must be allowed through the firewall on TCP port 514. If using Tailscale, ensure the ACLs are setup correctly.
+- It has an rsyslog ruleset that catches all forwarded logs through TCP port 514 and relays them to Promtail on TCP port 1514.
+- Promtail pushes the logs its receives via TCP port 1514 to the Loki client listening on TCP port 3100.
+
+*Regular Node 1*
+- It has an rsyslog ruleset that forwards logs to the Main Monitoring Node on TCP port 514.
+- Is allowed to access TCP port 514 on the Main Monitoring Node.
+
+*Regular Node 2*
+- It has an rsyslog ruleset that forwards logs to the Main Monitoring Node on TCP port 514.
+- Is allowed to access TCP port 514 on the Main Monitoring Node.
+
+*** Install Rsyslog, Promtail, and Loki on the Main Monitoring Node
+
+#+BEGIN_SRC shell
+# Debian-based hosts
+sudo apt install -y promtail loki rsyslog
+
+# Fedora-based hosts
+sudo dnf install -y promtail loki rsyslog
+#+END_SRC
+
+Edit ~/etc/promtail/config.yml~.
+
+#+BEGIN_SRC yaml
+server:
+  http_listen_port: 9081
+  grpc_listen_port: 0
+
+positions:
+  filename: /var/tmp/promtail-syslog-positions.yml
+
+clients:
+  - url: http://localhost:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: syslog
+    syslog:
+      listen_address: 0.0.0.0:1514
+      labels:
+        job: syslog
+    relabel_configs:
+      - source_labels: [__syslog_message_hostname]
+        target_label: hostname
+      - source_labels: [__syslog_message_severity]
+        target_label: level
+      - source_labels: [__syslog_message_app_name]
+        target_label: application
+      - source_labels: [__syslog_message_facility]
+        target_label: facility
+      - source_labels: [__syslog_connection_hostname]
+        target_label: connection_hostname
+#+END_SRC
+
+Edit ~/etc/loki/config.yml~.
+
+#+BEGIN_SRC yaml
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+ruler:
+  alertmanager_url: http://localhost:9093
+#+END_SRC
+
+Edit ~/etc/rsyslog.d/00-promtail-relay.conf~.
+
+#+BEGIN_SRC rsyslog
+# https://www.rsyslog.com/doc/v8-stable/concepts/multi_ruleset.html#split-local-and-remote-logging
+ruleset(name="remote"){
+  # https://www.rsyslog.com/doc/v8-stable/configuration/modules/omfwd.html
+  # https://grafana.com/docs/loki/latest/clients/promtail/scraping/#rsyslog-output-configuration
+  action(type="omfwd" Target="localhost" Port="1514" Protocol="tcp" Template="RSYSLOG_SyslogProtocol23Format" TCP_Framing="octet-counted")
+}
+
+
+# https://www.rsyslog.com/doc/v8-stable/configuration/modules/imudp.html
+module(load="imudp")
+input(type="imudp" port="514" ruleset="remote")
+
+# https://www.rsyslog.com/doc/v8-stable/configuration/modules/imtcp.html
+module(load="imtcp")
+input(type="imtcp" port="514" ruleset="remote")
+#+END_SRC
+
+Ensure the firewall allows TCP traffic to port 514.
+
+#+BEGIN_SRC shell
+sudo firewall-cmd --permanent --zone=tailnet --add-port=514/tcp
+sudo firewall-cmd --reload
+#+END_SRC
+
+Restart and/or enable the services.
+
+#+BEGIN_SRC shell
+sudo systemctl enable --now promtail.service
+sudo systemctl enable --now loki.service
+sudo systemctl enable --now rsyslog.service
+#+END_SRC
+
+*** Install and configure Rsyslog on Regular Node 1 and Regular Node 2
+
+#+BEGIN_SRC shell
+# Debian
+sudo apt install -y rsyslog
+
+# Fedora
+sudo dnf install -y rsyslog
+#+END_SRC
+
+Enable and start the rsyslog service.
+
+#+BEGIN_SRC shell
+sudo systemctl enable --now rsyslog
+#+END_SRC
+
+Edit ~/etc/rsyslog.conf~.
+
+#+BEGIN_SRC rsyslog
+###############
+#### RULES ####
+###############
+
+# Forward to Main Monitoring Node
+*.* action(type="omfwd" target="<IP addr of Main Monitoring Node>" port="514" protocol="tcp"
+    action.resumeRetryCount="100"
+    queue.type="linkedList" queue.size="10000")
+#+END_SRC
+
+Restart the rsyslog service.
+
+#+BEGIN_SRC shell
+sudo systemctl restart rsyslog.service
+#+END_SRC
+
+In the Grafana UI, you should now be able to add Loki as a data source. Then go to Home > Explore > loki and start querying logs from Regular Node 1 and Regular Node 2.