This commit is contained in:
Jeffrey Serio 2024-10-14 17:07:17 -05:00
parent eb06ca44da
commit 8700b620c8
8 changed files with 132 additions and 75 deletions

36
mastodon.py Executable file
View File

@ -0,0 +1,36 @@
#!/usr/bin/env python3
import pandas as pd
from pandas import json_normalize
import json
from html import unescape
with open("/home/jas/downloads/outbox.json", "r") as jf:
json_data = json.load(jf)
flattened_df = json_normalize(json_data, record_path=["orderedItems"])
published = []
for item in flattened_df["object.published"]:
published.append(item)
content = []
for item in flattened_df["object.content"]:
content.append(item)
x = zip(published, content)
print("#+TITLE: Mastodon posts, 2024-02-16T15:48:46Z - 2024-10-11T20:15:03Z")
print("#+SETUPFILE: ../org-templates/page.org")
print()
for item in x:
if type(item[0]) is str:
print(f"*** {item[0]}")
if type(item[1]) is str:
print("#+BEGIN_QUOTE")
print("#+BEGIN_EXPORT html")
print(unescape(item[1]))
print("#+END_EXPORT")
print("#+END_QUOTE")
print()

View File

@ -1,26 +0,0 @@
#!/home/jas/virtualenvs/resend/bin/python
#
# Usage:
# 1. Create a virtualenv for resend under ~/virtualenvs/resend
# 2. Activate the virtualenv: `~/virtualenvs/resend/bin/activate`
# 3. Install resend API: `pip install resend`
# 4. Deactivate the virtualenv: `deactivate`
# 5. Make the script executable: `chmod +x /home/jas/bin/resend_error.py`
# 6. Run the script from another script or wherever: `/home/jas/bin/resend_error.py`
import resend
import sys
resend.api_key = "APIKEY GO HERE"
msg = sys.argv[1]
params: resend.Emails.SendParams = {
"from": "NAS <onboarding@resend.dev>",
"to": ["hyperreal@fedoraproject.org"],
"subject": "[NAS] Sync error: " + msg,
"text": "There was an error syncing data: " + msg + ". Please investigate.",
}
email = resend.Emails.send(params)
print(email)

View File

@ -174,14 +174,21 @@ if __name__ == "__main__":
for torrent in knapsack:
print(torrent["link"])
else:
response = requests.get("https://libgen.rs")
response = requests.get("https://libgen.is/")
if not response.ok:
exit(
"It appears https://libgen.rs is currently down. Please try again later."
"It appears https://libgen.is is currently down. Please try again later."
)
for torrent in knapsack:
qb.download_from_link(torrent["link"], category="scihub")
print(f"Added {torrent['name']}")
for torrent in knapsack:
if "gen.lib.rus.ec" in torrent["link"]:
new_torrent = torrent["link"].replace("gen.lib.rus.ec", "libgen.is")
qb.download_from_link(new_torrent, category="scihub")
if "libgen.rs" in torrent["link"]:
new_torrent = torrent["link"].replace("libgen.rs", "libgen.is")
qb.download_from_link(new_torrent, category="scihub")
# print(f"Added {torrent['name']}")
print("----------------")
print(f"Count: {len(knapsack)} torrents")

View File

@ -31,6 +31,7 @@ mkShell {
python312Packages.bpython
python312Packages.docopt
python312Packages.isort
python312Packages.pandas
python312Packages.pytest
python312Packages.requests
python312Packages.resend

View File

@ -1,31 +0,0 @@
#!/usr/bin/env bash
set -eo pipefail
# If the -q (quiet) flag is supplied, send all output to /dev/null.
if [[ "$1" == "-q" ]]; then
exec >/dev/null 2>&1
fi
if ! rsync -avz --delete root@hyperreal.coffee:/srv/borgbackup/hyperreal/ /mnt/borgbackup/hyperreal; then
/home/jas/bin/resend_error.py "hyperreal-borg"
logger -t sync_from_remotes "Error during sync of hyperreal-borg"
fi
logger -t sync_from_remotes "hyperreal-borg successfully synced"
if ! rsync -avz --delete root@hyperreal.coffee:/home/mastodon/live/public/system/ /mnt/mastodata; then
/home/jas/bin/resend_error.py "mastodata"
logger -t sync_from_remotes "Error during sync of mastodata"
fi
logger -t sync_from_remotes "mastodata successfully synced"
if ! rsync -avz --delete root@desktop:/srv/borgbackup/moonshadow/ /mnt/borgbackup/moonshadow; then
/home/jas/bin/resend_error.py "moonshadow-borg"
logger -t sync_from_remotes "Error during sync of moonshadow-borg"
fi
logger -t sync_from_remotes "moonshadow-borg successfully synced"
exit 0

60
sync_from_remotes.py Executable file
View File

@ -0,0 +1,60 @@
#!/usr/bin/env nix-shell
#! nix-shell -i python3 --packages python3 python312Packages.resend
import socket
import subprocess
from pathlib import Path
import resend
def send_email(program: str, log: str):
resend.api_key = Path("/usr/local/etc/resend_api_key.txt").read_text().strip("\n")
match log:
case "ok":
subj = f"[{socket.getfqdn()}] {program} OK ✅"
msg = f"{program} on {socket.getfqdn()} ran successfully!"
case "err":
subj = f"[{socket.getfqdn()}] {program} Error ❌"
msg = f"There was an error running {program} on {socket.getfqdn()}. Please investigate."
case _:
subj = ""
msg = ""
params: resend.Emails.SendParams = {
"from": "Admin <admin@hyperreal.coffee>",
"to": ["hyperreal@moonshadow.dev"],
"subject": subj,
"text": msg,
}
email = resend.Emails.send(params)
print(email)
def sync_from_remotes(src: str, dest: str):
rsync_cmd = ["rsync", "-avz", "--delete", src, dest]
try:
subprocess.run(rsync_cmd, check=True, capture_output=True, text=True)
print(f"Successful sync from {src} to {dest}")
except subprocess.CalledProcessError as e:
print(f"Error during sync from {src} to {dest}: {e}")
send_email("sync_from_remotes", "err")
if __name__ == "__main__":
remotes = [
(
"root@hyperreal.coffee:/srv/borgbackup/hyperreal/",
"/mnt/borgbackup/hyperreal",
),
("root@hyperreal.coffee:/home/mastodon/live/public/system/", "/mnt/mastodata"),
("root@desktop:/srv/borgbackup/moonshadow/", "/mnt/borgbackup/moonshadow"),
]
for remote in remotes:
sync_from_remotes(remote[0], remote[1])
send_email("sync_from_remotes", "ok")

View File

@ -7,19 +7,23 @@ fi
if ! kbackup --verbose --autobg /home/jas/documents/default.kbp; then
logger -t sync_to_remotes -s "Error running kbackup"
/usr/local/bin/resend_status.py sync_to_remotes err
exit 1
fi
if ! rclone sync --transfers 8 -P /home/jas/sync_to_remotes/ protondrive:/desktop_backups/ ; then
if ! rclone sync --transfers 8 -P /home/jas/sync_to_remotes/ protondrive:/desktop-backups/ ; then
logger -t sync_to_remotes -s "Error running rclone for protondrive remote"
/usr/local/bin/resend_status.py sync_to_remotes err
exit 1
fi
if ! rclone sync --transfers 8 -P /home/jas/sync_to_remotes/ wasabi:/moonshadow-desktop/ ; then
if ! rclone sync --transfers 8 -P /home/jas/sync_to_remotes/ wasabi:/desktop-backups-rep/ ; then
logger -t sync_to_remotes -s "Error running rclone for wasabi remote"
/usr/local/bin/resend_status.py sync_to_remotes err
exit 1
fi
logger -t sync_to_remotes "sync_to_remotes ran successfully"
/usr/local/bin/resend_status.py sync_to_remotes ok
exit 0

View File

@ -8,16 +8,20 @@ updates opentracker, and reannounces all torrents to their trackers.
Expectations:
- A JSON qBittorrent authentication file at ~/.config/qbittorrent_auth.json
- SSH pubkey access to torrent tracker server root@bttracker.nirn.quest
- SSH pubkey access to torrent tracker server
- rsync installed on the host system running this script
Usage:
update_tracker.py [--add-nirn-tracker]
update_tracker.py [--add-tracker DOMAIN]
update_tracker.py -h
Options:
--add-nirn-tracker ensure bttracker.nirn.quest:6969/announce is added to each torrent's tracker list
--add-tracker DOMAIN ensure the provided tracker domain is added to each torrent's tracker list
-h, --help show this help message and exit
Examples:
update_tracker.py --add-tracker hyperreal.coffee
update_tracker.py
"""
import json
@ -33,6 +37,8 @@ from rich.text import Text
if __name__ == "__main__":
args = docopt(__doc__) # type: ignore
tracker_domain = args["--add-tracker"]
console = Console()
with console.status("[bold green]Executing the tasks...") as status:
# JSON file containing authentication info for each qBittorrent instance
@ -75,7 +81,7 @@ if __name__ == "__main__":
"scp",
"-q",
ntf.name,
"root@bttracker.nirn.quest:/etc/opentracker/whitelist.txt",
f"root@{tracker_domain}:/etc/opentracker/whitelist.txt",
]
)
@ -85,18 +91,18 @@ if __name__ == "__main__":
subprocess.run(
[
"ssh",
"root@bttracker.nirn.quest",
f"root@{tracker_domain}",
"systemctl",
"restart",
"opentracker.service",
]
)
# Ensure bttracker.nirn.quest:6969/announce is added to each torrent's
# Ensure {tracker_domain}:6969/announce is added to each torrent's
# tracker list.
if args["--add-nirn-tracker"]:
if tracker_domain:
console.log(
"Ensuring bttracker.nirn.quest:6969/announce is added to each torrent's tracker list."
f"Ensuring {tracker_domain}:6969/announce is added to each torrent's tracker list."
)
for item in auth_data["instances"]:
qb = Client(item["hostname"])
@ -104,7 +110,7 @@ if __name__ == "__main__":
for torrent in qb.torrents():
qb.add_trackers(
torrent.get("hash"), # type: ignore
"http://bttracker.nirn.quest:6969/announce\nudp://bttracker.nirn.quest:6969/announce",
f"http://{tracker_domain}:6969/announce\nudp://{tracker_domain}:6969/announce",
)
# Reannounce all torrents in each qBittorrent instance to their trackers
@ -124,9 +130,9 @@ if __name__ == "__main__":
console.print(tasks)
console.print(":white_check_mark: update the tracker's whitelist")
if args["--add-nirn-tracker"]:
if tracker_domain:
console.print(
":white_check_mark: ensure bttracker.nirn.quest:6969/announce is in each torrent's tracker list"
f":white_check_mark: ensure {tracker_domain}:6969/announce is in each torrent's tracker list"
)
console.print(":white_check_mark: reannounce all torrents to their trackers")