From 8700b620c833c1413a4109614ca7f8cbeccc9297 Mon Sep 17 00:00:00 2001 From: Jeffrey Serio Date: Mon, 14 Oct 2024 17:07:17 -0500 Subject: [PATCH] Update --- mastodon.py | 36 ++++++++++++++++++++++++++ resend_error.py | 26 ------------------- scihub_knapsack.py | 15 ++++++++--- shell.nix | 1 + sync_from_remotes | 31 ----------------------- sync_from_remotes.py | 60 ++++++++++++++++++++++++++++++++++++++++++++ sync_to_remotes | 8 ++++-- update_tracker.py | 30 +++++++++++++--------- 8 files changed, 132 insertions(+), 75 deletions(-) create mode 100755 mastodon.py delete mode 100755 resend_error.py delete mode 100755 sync_from_remotes create mode 100755 sync_from_remotes.py diff --git a/mastodon.py b/mastodon.py new file mode 100755 index 0000000..e3f2fb3 --- /dev/null +++ b/mastodon.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import pandas as pd +from pandas import json_normalize +import json +from html import unescape + +with open("/home/jas/downloads/outbox.json", "r") as jf: + json_data = json.load(jf) + +flattened_df = json_normalize(json_data, record_path=["orderedItems"]) + +published = [] +for item in flattened_df["object.published"]: + published.append(item) + +content = [] +for item in flattened_df["object.content"]: + content.append(item) + +x = zip(published, content) + +print("#+TITLE: Mastodon posts, 2024-02-16T15:48:46Z - 2024-10-11T20:15:03Z") +print("#+SETUPFILE: ../org-templates/page.org") +print() +for item in x: + if type(item[0]) is str: + print(f"*** {item[0]}") + + if type(item[1]) is str: + print("#+BEGIN_QUOTE") + print("#+BEGIN_EXPORT html") + print(unescape(item[1])) + print("#+END_EXPORT") + print("#+END_QUOTE") + print() diff --git a/resend_error.py b/resend_error.py deleted file mode 100755 index dcd59fd..0000000 --- a/resend_error.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/home/jas/virtualenvs/resend/bin/python -# -# Usage: -# 1. Create a virtualenv for resend under ~/virtualenvs/resend -# 2. Activate the virtualenv: `~/virtualenvs/resend/bin/activate` -# 3. Install resend API: `pip install resend` -# 4. Deactivate the virtualenv: `deactivate` -# 5. Make the script executable: `chmod +x /home/jas/bin/resend_error.py` -# 6. Run the script from another script or wherever: `/home/jas/bin/resend_error.py` - -import resend -import sys - -resend.api_key = "APIKEY GO HERE" - -msg = sys.argv[1] - -params: resend.Emails.SendParams = { - "from": "NAS ", - "to": ["hyperreal@fedoraproject.org"], - "subject": "[NAS] Sync error: " + msg, - "text": "There was an error syncing data: " + msg + ". Please investigate.", -} - -email = resend.Emails.send(params) -print(email) diff --git a/scihub_knapsack.py b/scihub_knapsack.py index 7aefa23..2f7c41c 100755 --- a/scihub_knapsack.py +++ b/scihub_knapsack.py @@ -174,14 +174,21 @@ if __name__ == "__main__": for torrent in knapsack: print(torrent["link"]) else: - response = requests.get("https://libgen.rs") + response = requests.get("https://libgen.is/") if not response.ok: exit( - "It appears https://libgen.rs is currently down. Please try again later." + "It appears https://libgen.is is currently down. Please try again later." ) for torrent in knapsack: - qb.download_from_link(torrent["link"], category="scihub") - print(f"Added {torrent['name']}") + for torrent in knapsack: + if "gen.lib.rus.ec" in torrent["link"]: + new_torrent = torrent["link"].replace("gen.lib.rus.ec", "libgen.is") + qb.download_from_link(new_torrent, category="scihub") + + if "libgen.rs" in torrent["link"]: + new_torrent = torrent["link"].replace("libgen.rs", "libgen.is") + qb.download_from_link(new_torrent, category="scihub") + # print(f"Added {torrent['name']}") print("----------------") print(f"Count: {len(knapsack)} torrents") diff --git a/shell.nix b/shell.nix index 1020148..11032f0 100644 --- a/shell.nix +++ b/shell.nix @@ -31,6 +31,7 @@ mkShell { python312Packages.bpython python312Packages.docopt python312Packages.isort + python312Packages.pandas python312Packages.pytest python312Packages.requests python312Packages.resend diff --git a/sync_from_remotes b/sync_from_remotes deleted file mode 100755 index 8d30bbc..0000000 --- a/sync_from_remotes +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -# If the -q (quiet) flag is supplied, send all output to /dev/null. -if [[ "$1" == "-q" ]]; then - exec >/dev/null 2>&1 -fi - -if ! rsync -avz --delete root@hyperreal.coffee:/srv/borgbackup/hyperreal/ /mnt/borgbackup/hyperreal; then - /home/jas/bin/resend_error.py "hyperreal-borg" - logger -t sync_from_remotes "Error during sync of hyperreal-borg" -fi - -logger -t sync_from_remotes "hyperreal-borg successfully synced" - -if ! rsync -avz --delete root@hyperreal.coffee:/home/mastodon/live/public/system/ /mnt/mastodata; then - /home/jas/bin/resend_error.py "mastodata" - logger -t sync_from_remotes "Error during sync of mastodata" -fi - -logger -t sync_from_remotes "mastodata successfully synced" - -if ! rsync -avz --delete root@desktop:/srv/borgbackup/moonshadow/ /mnt/borgbackup/moonshadow; then - /home/jas/bin/resend_error.py "moonshadow-borg" - logger -t sync_from_remotes "Error during sync of moonshadow-borg" -fi - -logger -t sync_from_remotes "moonshadow-borg successfully synced" - -exit 0 diff --git a/sync_from_remotes.py b/sync_from_remotes.py new file mode 100755 index 0000000..c1ca5d9 --- /dev/null +++ b/sync_from_remotes.py @@ -0,0 +1,60 @@ +#!/usr/bin/env nix-shell +#! nix-shell -i python3 --packages python3 python312Packages.resend + +import socket +import subprocess +from pathlib import Path + +import resend + + +def send_email(program: str, log: str): + resend.api_key = Path("/usr/local/etc/resend_api_key.txt").read_text().strip("\n") + + match log: + case "ok": + subj = f"[{socket.getfqdn()}] {program} OK ✅" + msg = f"{program} on {socket.getfqdn()} ran successfully!" + case "err": + subj = f"[{socket.getfqdn()}] {program} Error ❌" + msg = f"There was an error running {program} on {socket.getfqdn()}. Please investigate." + case _: + subj = "" + msg = "" + + params: resend.Emails.SendParams = { + "from": "Admin ", + "to": ["hyperreal@moonshadow.dev"], + "subject": subj, + "text": msg, + } + + email = resend.Emails.send(params) + print(email) + + +def sync_from_remotes(src: str, dest: str): + rsync_cmd = ["rsync", "-avz", "--delete", src, dest] + + try: + subprocess.run(rsync_cmd, check=True, capture_output=True, text=True) + print(f"Successful sync from {src} to {dest}") + except subprocess.CalledProcessError as e: + print(f"Error during sync from {src} to {dest}: {e}") + send_email("sync_from_remotes", "err") + + +if __name__ == "__main__": + remotes = [ + ( + "root@hyperreal.coffee:/srv/borgbackup/hyperreal/", + "/mnt/borgbackup/hyperreal", + ), + ("root@hyperreal.coffee:/home/mastodon/live/public/system/", "/mnt/mastodata"), + ("root@desktop:/srv/borgbackup/moonshadow/", "/mnt/borgbackup/moonshadow"), + ] + + for remote in remotes: + sync_from_remotes(remote[0], remote[1]) + + send_email("sync_from_remotes", "ok") diff --git a/sync_to_remotes b/sync_to_remotes index ea28436..54f428c 100755 --- a/sync_to_remotes +++ b/sync_to_remotes @@ -7,19 +7,23 @@ fi if ! kbackup --verbose --autobg /home/jas/documents/default.kbp; then logger -t sync_to_remotes -s "Error running kbackup" + /usr/local/bin/resend_status.py sync_to_remotes err exit 1 fi -if ! rclone sync --transfers 8 -P /home/jas/sync_to_remotes/ protondrive:/desktop_backups/ ; then +if ! rclone sync --transfers 8 -P /home/jas/sync_to_remotes/ protondrive:/desktop-backups/ ; then logger -t sync_to_remotes -s "Error running rclone for protondrive remote" + /usr/local/bin/resend_status.py sync_to_remotes err exit 1 fi -if ! rclone sync --transfers 8 -P /home/jas/sync_to_remotes/ wasabi:/moonshadow-desktop/ ; then +if ! rclone sync --transfers 8 -P /home/jas/sync_to_remotes/ wasabi:/desktop-backups-rep/ ; then logger -t sync_to_remotes -s "Error running rclone for wasabi remote" + /usr/local/bin/resend_status.py sync_to_remotes err exit 1 fi logger -t sync_to_remotes "sync_to_remotes ran successfully" +/usr/local/bin/resend_status.py sync_to_remotes ok exit 0 diff --git a/update_tracker.py b/update_tracker.py index 1cdaab1..ae767ba 100755 --- a/update_tracker.py +++ b/update_tracker.py @@ -8,16 +8,20 @@ updates opentracker, and reannounces all torrents to their trackers. Expectations: - A JSON qBittorrent authentication file at ~/.config/qbittorrent_auth.json -- SSH pubkey access to torrent tracker server root@bttracker.nirn.quest +- SSH pubkey access to torrent tracker server - rsync installed on the host system running this script Usage: - update_tracker.py [--add-nirn-tracker] + update_tracker.py [--add-tracker DOMAIN] update_tracker.py -h Options: - --add-nirn-tracker ensure bttracker.nirn.quest:6969/announce is added to each torrent's tracker list - -h, --help show this help message and exit + --add-tracker DOMAIN ensure the provided tracker domain is added to each torrent's tracker list + -h, --help show this help message and exit + +Examples: + update_tracker.py --add-tracker hyperreal.coffee + update_tracker.py """ import json @@ -33,6 +37,8 @@ from rich.text import Text if __name__ == "__main__": args = docopt(__doc__) # type: ignore + tracker_domain = args["--add-tracker"] + console = Console() with console.status("[bold green]Executing the tasks...") as status: # JSON file containing authentication info for each qBittorrent instance @@ -75,7 +81,7 @@ if __name__ == "__main__": "scp", "-q", ntf.name, - "root@bttracker.nirn.quest:/etc/opentracker/whitelist.txt", + f"root@{tracker_domain}:/etc/opentracker/whitelist.txt", ] ) @@ -85,18 +91,18 @@ if __name__ == "__main__": subprocess.run( [ "ssh", - "root@bttracker.nirn.quest", + f"root@{tracker_domain}", "systemctl", "restart", "opentracker.service", ] ) - # Ensure bttracker.nirn.quest:6969/announce is added to each torrent's + # Ensure {tracker_domain}:6969/announce is added to each torrent's # tracker list. - if args["--add-nirn-tracker"]: + if tracker_domain: console.log( - "Ensuring bttracker.nirn.quest:6969/announce is added to each torrent's tracker list." + f"Ensuring {tracker_domain}:6969/announce is added to each torrent's tracker list." ) for item in auth_data["instances"]: qb = Client(item["hostname"]) @@ -104,7 +110,7 @@ if __name__ == "__main__": for torrent in qb.torrents(): qb.add_trackers( torrent.get("hash"), # type: ignore - "http://bttracker.nirn.quest:6969/announce\nudp://bttracker.nirn.quest:6969/announce", + f"http://{tracker_domain}:6969/announce\nudp://{tracker_domain}:6969/announce", ) # Reannounce all torrents in each qBittorrent instance to their trackers @@ -124,9 +130,9 @@ if __name__ == "__main__": console.print(tasks) console.print(":white_check_mark: update the tracker's whitelist") - if args["--add-nirn-tracker"]: + if tracker_domain: console.print( - ":white_check_mark: ensure bttracker.nirn.quest:6969/announce is in each torrent's tracker list" + f":white_check_mark: ensure {tracker_domain}:6969/announce is in each torrent's tracker list" ) console.print(":white_check_mark: reannounce all torrents to their trackers")