From 58aa0588610d1b3e3cbc2ac081bf3271ae353786 Mon Sep 17 00:00:00 2001
From: Jeffrey Serio <hyperreal@moonshadow.dev>
Date: Thu, 3 Oct 2024 23:58:35 -0500
Subject: [PATCH] Update prometheus.org, mastodon.org

---
 mastodon.org   | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++
 prometheus.org | 72 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 156 insertions(+)

diff --git a/mastodon.org b/mastodon.org
index c5eaa2d..5ea55c2 100644
--- a/mastodon.org
+++ b/mastodon.org
@@ -160,3 +160,87 @@ S3_ALIAS_HOST=assets.hyperreal.coffee
 #+begin_src shell
 sudo systemctl restart caddy.service mastodon-web.service mastodon-streaming.service mastodon-sidekiq.service
 #+end_src
+
+** Prometheus metrics with statsd_exporter
+
+On the host running Mastodon, download the latest binary from [[https://github.com/prometheus/statsd_exporter/releases][releases]] page.
+
+#+BEGIN_SRC shell
+tar xzvf statsd_exporter*.tar.gz
+cd statsd_exporter*/
+sudo cp -v statsd_exporter /usr/local/bin/
+#+END_SRC
+
+Install the statsd mapping file from IPng Networks:
+
+#+BEGIN_SRC shell
+curl -OL https://ipng.ch/assets/mastodon/statsd-mapping.yaml
+sudo cp -v statsd-mapping.yml /etc/prometheus/
+#+END_SRC
+
+Create ~/etc/default/statsd_exporter~.
+
+#+BEGIN_SRC shell
+ARGS="--statsd.mapping-config=/etc/prometheus/statsd-mapping.yaml"
+#+END_SRC
+
+Create ~/etc/systemd/system/statsd_exporter.service~.
+
+#+BEGIN_SRC systemd
+[Unit]
+Description=Statsd exporter
+After=network.target
+
+[Service]
+Restart=always
+User=prometheus
+EnvironmentFile=/etc/default/statsd_exporter
+ExecStart=/usr/local/bin/statsd_exporter $ARGS
+ExecReload=/bin/kill -HUP $MAINPID
+TimeoutStopSec=20s
+SendSIGKILL=no
+
+[Install]
+WantedBy=multi-user.target
+#+END_SRC
+
+Ensure port 9102 is open in Firewalld's internal zone.
+
+#+BEGIN_SRC shell
+sudo firewall-cmd --permanent --zone=internal --add-port=9102/tcp
+sudo firewall-cmd --reload
+#+END_SRC
+
+Edit ~/home/mastodon/live/.env.production~.
+
+#+BEGIN_SRC shell
+STATSD_ADDR=localhost:9125
+#+END_SRC
+
+Start and restart the daemons.
+
+#+BEGIN_SRC shell
+sudo systemctl daemon-reload
+sudo systemctl start statsd_exporter.service
+sudo systemctl restart mastodon-sidekiq.service mastodon-streaming.service mastodon-web.service
+#+END_SRC
+
+If using Tailscale, ensure the host running Prometheus can access port 9102 on the host running Mastodon.
+
+On the host running Prometheus, add the statsd config.
+
+#+BEGIN_SRC yaml
+- job_name: "stats_exporter"
+  static_configs:
+  - targets: ["hyperreal:9102"]
+#+END_SRC
+
+Restart Prometheus.
+
+#+BEGIN_SRC shell
+sudo systemctl restart prometheus.service
+#+END_SRC
+
+To import the Grafana dashboard, use ID 17492.
+
+Source: [[https://jaanus.com/mastodon-monitoring-prometheus-grafana/][How to set up monitoring for your Mastodon instance with Prometheus and Grafana]]
diff --git a/prometheus.org b/prometheus.org
index ea32d49..a3939d6 100644
--- a/prometheus.org
+++ b/prometheus.org
@@ -531,3 +531,75 @@ sudo wget https://files.hyperreal.coffee/prometheus/synapse-v2.rules -O /etc/pro
 Restart Prometheus.
 
 Use [[https://files.hyperreal.coffee/grafana/synapse.json][synapse.json]] for Grafana dashboard.
+
+** Monitor Elasticsearch
+
+On the host running Elasticsearch, download the latest binary from the GitHub [[https://github.com/prometheus-community/elasticsearch_exporter/releases][releases]].
+
+#+BEGIN_SRC shell
+tar xvf elasitcsearch_exporter*.tar.gz
+cd elasticsearch_exporter*/
+sudo cp -v elasticsearch_exporter /usr/local/bin/
+#+END_SRC
+
+Create ~/etc/systemd/system/elasticsearch_exporter.service~.
+
+#+BEGIN_SRC systemd
+[Unit]
+Description=elasticsearch exporter
+After=network.target
+
+[Service]
+Restart=always
+User=prometheus
+ExecStart=/usr/local/bin/elasticsearch_exporter --es.uri=http://localhost:9200
+ExecReload=/bin/kill -HUP $MAINPID
+TimeoutStopSec=20s
+SendSIGKILL=no
+
+[Install]
+WantedBy=multi-user.target
+#+END_SRC
+
+Reload the daemons and enable/start elasticsearch_exporter.
+
+#+BEGIN_SRC shell
+sudo systemctl daemon-reload
+sudo systemctl enable --now elasticsearch_exporter.service
+#+END_SRC
+
+Ensure port 9114 is allowed in Firewalld's internal zone.
+
+#+BEGIN_SRC shell
+sudo firewall-cmd --permanent --zone=internal --add-port=9114/tcp
+sudo firewall-cmd --reload
+#+END_SRC
+
+If using Tailscale, ensure the host running Prometheus can access port 9114 on the host running Elasticsearch.
+
+On the host running Prometheus, download the elasticsearch.rules.
+
+#+BEGIN_SRC shell
+wget https://raw.githubusercontent.com/prometheus-community/elasticsearch_exporter/refs/heads/master/examples/prometheus/elasticsearch.rules.yml
+sudo mv elasticsearch.rules.yml /etc/prometheus/
+#+END_SRC
+
+Edit ~/etc/prometheus/prometheus.yml~ to add the elasticsearch_exporter config.
+
+#+BEGIN_SRC yaml
+rule_files:
+  - "/etc/prometheus/elasticsearch.rules.yml"
+...
+...
+- job_name: "elasticsearch_exporter"
+  static_configs:
+    - targets: ["hyperreal:9114"]
+#+END_SRC
+
+Restart Prometheus.
+
+#+BEGIN_SRC shell
+sudo systemctl restart prometheus.service
+#+END_SRC
+
+For a Grafana dashboard, copy the contents of the file located here: [[https://files.hyperreal.coffee/files/grafana/elasticsearch.json]].