From 8af36a2a5cdf8ed452b8e91fa42727416dd90fdc Mon Sep 17 00:00:00 2001 From: Jeffrey Serio Date: Fri, 8 Nov 2024 17:18:47 -0600 Subject: [PATCH] Update --- .archived/mastodon.py | 36 +++++++++ .envrc | 1 - .gitignore | 2 + calculate_mirror_size.py | 9 ++- fetch_combined_trackers_list.py | 9 ++- fetch_scihub_infohashes.py | 7 +- list_torrents.py | 58 +++++++------- qbt_stats_html.nu | 4 +- qbt_sum_size.py | 38 +++++---- qbth.py | 133 ++++++++++++++++++++++++-------- scihub_knapsack.py | 28 +++++-- shell.nix | 44 ----------- sync_from_remotes.py | 7 +- sync_to_remotes.py | 8 +- update_tracker.py | 81 ++++++++----------- 15 files changed, 279 insertions(+), 186 deletions(-) create mode 100755 .archived/mastodon.py delete mode 100644 .envrc delete mode 100644 shell.nix diff --git a/.archived/mastodon.py b/.archived/mastodon.py new file mode 100755 index 0000000..e3f2fb3 --- /dev/null +++ b/.archived/mastodon.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import pandas as pd +from pandas import json_normalize +import json +from html import unescape + +with open("/home/jas/downloads/outbox.json", "r") as jf: + json_data = json.load(jf) + +flattened_df = json_normalize(json_data, record_path=["orderedItems"]) + +published = [] +for item in flattened_df["object.published"]: + published.append(item) + +content = [] +for item in flattened_df["object.content"]: + content.append(item) + +x = zip(published, content) + +print("#+TITLE: Mastodon posts, 2024-02-16T15:48:46Z - 2024-10-11T20:15:03Z") +print("#+SETUPFILE: ../org-templates/page.org") +print() +for item in x: + if type(item[0]) is str: + print(f"*** {item[0]}") + + if type(item[1]) is str: + print("#+BEGIN_QUOTE") + print("#+BEGIN_EXPORT html") + print(unescape(item[1])) + print("#+END_EXPORT") + print("#+END_QUOTE") + print() diff --git a/.envrc b/.envrc deleted file mode 100644 index 1d953f4..0000000 --- a/.envrc +++ /dev/null @@ -1 +0,0 @@ -use nix diff --git a/.gitignore b/.gitignore index 92b2793..b6628c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ .direnv +.venv + diff --git a/calculate_mirror_size.py b/calculate_mirror_size.py index 86cecba..471c9c5 100755 --- a/calculate_mirror_size.py +++ b/calculate_mirror_size.py @@ -1,4 +1,11 @@ -#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "requests", +# "bs4", +# "docopt", +# "rich", +# ] +# /// """calculate_mirror_size.py diff --git a/fetch_combined_trackers_list.py b/fetch_combined_trackers_list.py index 0c71b5b..ab07c9a 100755 --- a/fetch_combined_trackers_list.py +++ b/fetch_combined_trackers_list.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "requests", +# "docopt", +# ] +# /// """fetch_combined_trackers_list.py @@ -20,7 +25,7 @@ import requests from docopt import docopt if __name__ == "__main__": - args = docopt(__doc__) # type: ignore + args = docopt(__doc__) # type: ignore live_trackers_list_urls = [ "https://newtrackon.com/api/stable", diff --git a/fetch_scihub_infohashes.py b/fetch_scihub_infohashes.py index bd1d1f9..84fdb7f 100755 --- a/fetch_scihub_infohashes.py +++ b/fetch_scihub_infohashes.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "requests", +# "docopt", +# ] +# /// """fetch_scihub_infohashes.py diff --git a/list_torrents.py b/list_torrents.py index 63fcf2b..c607806 100755 --- a/list_torrents.py +++ b/list_torrents.py @@ -1,4 +1,9 @@ -#!/home/jas/virtualenvs/list_torrents/bin/python3 +# /// script +# dependencies = [ +# "qbittorrent-api", +# "docopt", +# ] +# /// """list_torrents.py @@ -15,9 +20,8 @@ Options: -h, --help show this help message and exit """ +import qbittorrentapi from docopt import docopt -from qbittorrent import Client -from tabulate import tabulate # convert byte units @@ -43,36 +47,26 @@ def human_bytes(input_bytes: int) -> str: return "" -def print_table(): - qb = Client("http://localhost:8080/") - qb.login() - table = [] - headers = ["Name", "Total Size", "Trackers Count", "Ratio", "Uploaded"] - sorted_torrents = sorted(qb.torrents(), key=lambda d: d["ratio"], reverse=True) # type: ignore - for torrent in sorted_torrents: - row = [] - row.append(torrent["name"]) # type: ignore - row.append(human_bytes(int(torrent["total_size"]))) # type: ignore - row.append(torrent["trackers_count"]) # type: ignore - row.append(torrent["ratio"]) # type: ignore - row.append(human_bytes(int(torrent["uploaded"]))) # type: ignore - table.append(row) - - print(tabulate(table, headers, tablefmt="grid")) - - def print_ssv(): - qb = Client("http://localhost:8080/") - qb.login() - sorted_torrents = sorted(qb.torrents(), key=lambda d: d["ratio"], reverse=True) # type: ignore - print("Name Size Trackers Ratio Uploaded") - for torrent in sorted_torrents: - name = torrent["name"] # type: ignore - size = human_bytes(int(torrent["total_size"])) # type: ignore - trackers = torrent["trackers_count"] # type: ignore - ratio = torrent["ratio"] # type: ignore - uploaded = human_bytes(int(torrent["uploaded"])) # type: ignore - print(f"{name} {size} {trackers} {ratio} {uploaded}") + with qbittorrentapi.Client( + host="localhost", port=8080, username="", password="" + ) as qbt_client: + try: + qbt_client.auth_log_in() + except qbittorrentapi.LoginFailed as e: + print(e) + + sorted_torrents = sorted( + qbt_client.torrents_info(), key=lambda d: d.ratio, reverse=True + ) + print("Name Size # of Trackers Ratio Uploaded") + for torrent in sorted_torrents: + name = torrent.name + size = human_bytes(torrent.total_size) + trackers = torrent.trackers_count + ratio = torrent.ratio + uploaded = human_bytes(torrent.uploaded) + print(f"{name} {size} {trackers} {ratio} {uploaded}") if __name__ == "__main__": diff --git a/qbt_stats_html.nu b/qbt_stats_html.nu index 9d663ea..3b8645d 100755 --- a/qbt_stats_html.nu +++ b/qbt_stats_html.nu @@ -2,12 +2,12 @@ let old_head = "" let new_head = ( - ["Torrent Stats

Last updated:", (date now | format date "%F %T%:z"), "

"] + ["Torrent Stats

Last updated:", (date now | format date "%F %T%:z"), "

"] | str join ' ' ) ( - /home/jas/admin-scripts/list_torrents.py + uv run -q /home/jas/admin-scripts/list_torrents.py | from ssv -m 2 | to html | str replace ($old_head) ($new_head) diff --git a/qbt_sum_size.py b/qbt_sum_size.py index 2bb9ca1..f5fd7b7 100755 --- a/qbt_sum_size.py +++ b/qbt_sum_size.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "qbittorrent-api", +# "docopt", +# ] +# /// """qbt_sum_size.py @@ -18,8 +23,8 @@ Options: -h, --help show this help message and exit """ +import qbittorrentapi from docopt import docopt -from qbittorrent import Client # convert byte units @@ -48,21 +53,26 @@ def human_bytes(bites: int) -> str: if __name__ == "__main__": args = docopt(__doc__) # type: ignore - # Initialize client and login - qb = Client(args["HOSTNAME"]) - qb.login(username=args["USERNAME"], password=args["PASSWORD"]) - - # get total_completed_bytes completed_torrent_sizes = [] - for torrent in qb.torrents(): - if torrent["completion_on"] != 0: # type: ignore - completed_torrent_sizes.append(torrent["total_size"]) # type: ignore + total_added_bytes = int() + + with qbittorrentapi.Client( + host=args["HOSTNAME"], username=args["USERNAME"], password=args["PASSWORD"] + ) as qbt_client: + try: + qbt_client.auth_log_in() + except qbittorrentapi.LoginFailed as e: + print(e) + + for torrent in qbt_client.torrents_info(): + if torrent.completion_on != 0: + completed_torrent_sizes.append(torrent.total_size) + + total_added_bytes = sum( + [torrent.total_size for torrent in qbt_client.torrents_info()] + ) total_completed_bytes = sum(completed_torrent_sizes) - # get total_added_bytes - total_added_bytes = sum([torrent["total_size"] for torrent in qb.torrents()]) # type: ignore - - # print the results print(f"\nTotal completed size: {human_bytes(total_completed_bytes)}") print(f"Total added size: {human_bytes(total_added_bytes)}\n") diff --git a/qbth.py b/qbth.py index 59025cb..d1618ce 100755 --- a/qbth.py +++ b/qbth.py @@ -1,4 +1,11 @@ -#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "qbittorrent-api", +# "requests", +# "bs4", +# "docopt", +# ] +# /// """qbth.py - qbittorrent helper @@ -17,15 +24,25 @@ Options: import json import os import subprocess +from shutil import which +import qbittorrentapi import requests from bs4 import BeautifulSoup from docopt import docopt -from qbittorrent import Client args = docopt(__doc__) -qb = Client(args["HOSTNAME"]) -qb.login(username=args["USERNAME"], password=args["PASSWORD"]) +conn_info = dict( + host=args["HOSTNAME"], + username=args["USERNAME"], + password=args["PASSWORD"], +) + +try: + with qbittorrentapi.Client(**conn_info) as qbt_client: + qbt_client.auth_log_in() +except qbittorrentapi.LoginFailed as e: + print(e) def add_torrents(urls: list[str]): @@ -35,9 +52,16 @@ def add_torrents(urls: list[str]): Params: urls: list of strings that are URLs. """ - for url in urls: - qb.download_from_link(url, category="distro") - print(f"Added {os.path.basename(url)}") + with qbittorrentapi.Client(**conn_info) as qbt_client: + for url in urls: + response = requests.get(url) + if response.status_code == 200: + if qbt_client.torrents_add(url, category="distro") != "Ok.": + raise Exception("Failed to add torrent: " + os.path.basename(url)) + else: + print(f"Added {os.path.basename(url)}") + else: + print(f"{response.status_code}: {url}") def add_torrents_from_html(webpage_url: str, torrent_substring: str): @@ -52,11 +76,21 @@ def add_torrents_from_html(webpage_url: str, torrent_substring: str): """ reqs = requests.get(webpage_url, timeout=60) soup = BeautifulSoup(reqs.text, "html.parser") - for link in soup.find_all("a"): - if torrent_substring in link.get("href"): - url = f"{webpage_url}/{link.get('href')}" - qb.download_from_link(url, category="distro") - print(f"Added {link.get('href')}") + with qbittorrentapi.Client(**conn_info) as qbt_client: + for link in soup.find_all("a"): + if torrent_substring in link.get("href"): + url = f"{webpage_url}/{link.get('href')}" + response = requests.get(url) + if response.status_code == 200: + if qbt_client.torrents_add(url, category="distro") != "Ok.": + raise Exception( + "Failed to add torrent: " + os.path.basename(url) + ) + else: + print(f"Added {os.path.basename(url)}") + + else: + print(f"{response.status_code}: {url}") def remove_torrents(distro_substring: str): @@ -69,10 +103,13 @@ def remove_torrents(distro_substring: str): distro_substring: a string that is a substring of the distro torrent's file name. """ - for torrent in qb.torrents(): - if distro_substring in torrent.get("name"): # type: ignore - qb.delete_permanently(torrent.get("hash")) # type: ignore - print(f"Removed {torrent.get('name')}") # type: ignore + with qbittorrentapi.Client(**conn_info) as qbt_client: + for torrent in qbt_client.torrents_info(): + if distro_substring in torrent.name: + qbt_client.torrents_delete( + torrent_hashes=torrent.hash, delete_files=True + ) + print(f"Removed {torrent.name}") def add_almalinux(rel_ver: str): @@ -142,8 +179,7 @@ def add_devuan(rel_ver: str): relver: the Devuan release version. """ url = f"https://files.devuan.org/devuan_{rel_ver}.torrent" - qb.download_from_link(url, category="distro") - print(f"Added {os.path.basename(url)}") + add_torrents([url]) def remove_devuan(rel_ver: str): @@ -175,11 +211,13 @@ def remove_fedora(rel_ver: str): Params: relver: the Fedora release version. """ - torrents = qb.torrents() - for torrent in torrents: - if torrent.get("name").startswith("Fedora") and torrent.get("name").endswith(rel_ver): # type: ignore - qb.delete_permanently(torrent.get("hash")) # type: ignore - print(f"Removed {torrent.get('name')}") # type: ignore + with qbittorrentapi.Client(**conn_info) as qbt_client: + for torrent in qbt_client.torrents_info(): + if torrent.name.startswith("Fedora") and torrent.name.endswith(rel_ver): + qbt_client.torrents_delete( + torrent_hashes=torrent.hash, delete_files=True + ) + print(f"Removed {torrent.name}") def add_freebsd(rel_ver: str): @@ -194,10 +232,13 @@ def add_freebsd(rel_ver: str): reqs = requests.get(url, timeout=60) data = reqs.text.split("\n") - for line in data: - if line.startswith("magnet:"): - qb.download_from_link(line, category="distro") - print(f"Added {line.split('=')[2]}") + with qbittorrentapi.Client(**conn_info) as qbt_client: + for line in data: + if line.startswith("magnet:"): + if qbt_client.torrents_add(line) != "Ok.": + raise Exception("Failed to add torrent: " + line.split("=")[2]) + + print(f"Added {line.split('=')[2]}") def remove_freebsd(rel_ver: str): @@ -266,9 +307,25 @@ def add_nixos(): url = "https://api.github.com/repos/AnimMouse/NixOS-ISO-Torrents/releases/latest" reqs = requests.get(url, timeout=60) json_data = json.loads(reqs.text) - for item in json_data["assets"]: - qb.download_from_link(item["browser_download_url"], category="distro") - print(f"Added {os.path.basename(item['browser_download_url'])}") + + with qbittorrentapi.Client(**conn_info) as qbt_client: + for item in json_data["assets"]: + response = requests.get(item["browser_download_url"]) + if response.status_code == 200: + if ( + qbt_client.torrents_add( + item["browser_download_url"], category="distro" + ) + != "Ok." + ): + raise Exception( + "Failed to add torrent: " + + os.path.basename(item["browser_download_url"]) + ) + else: + print(f"Added {os.path.basename(item['browser_download_url'])}") + else: + print(f"{response.status_code}: {item['browser_download_url']}") def remove_nixos(): @@ -289,8 +346,16 @@ def add_qubes(rel_ver: str): relver: the QubesOS release version. """ url = f"https://mirrors.edge.kernel.org/qubes/iso/Qubes-R{rel_ver}-x86_64.torrent" - qb.download_from_link(url, category="distro") - print(f"Added {os.path.basename(url)}") + + response = requests.get(url) + if response.status_code == 200: + with qbittorrentapi.Client(**conn_info) as qbt_client: + if qbt_client.torrents_add(url, category="distro") != "Ok.": + raise Exception("Failed to add torrent: " + os.path.basename(url)) + else: + print(f"Added {os.path.basename(url)}") + else: + print(f"{response.status_code}: {url}") def remove_qubes(rel_ver: str): @@ -356,6 +421,10 @@ def remove_tails(rel_ver: str): if __name__ == "__main__": + # Check if gum is installed. + if which("gum") is None: + exit("Please install gum first. https://github.com/charmbracelet/gum") + # Run the gum program in a subprocess to allow easy selecting of distro # torrents. distro_selection = subprocess.run( diff --git a/scihub_knapsack.py b/scihub_knapsack.py index 2f7c41c..49d2661 100755 --- a/scihub_knapsack.py +++ b/scihub_knapsack.py @@ -1,4 +1,10 @@ -#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "qbittorrent-api", +# "requests", +# "docopt", +# ] +# /// """scihub_knapsack.py @@ -39,9 +45,9 @@ Options: import json +import qbittorrentapi import requests from docopt import docopt -from qbittorrent import Client def get_torrent_health_data() -> list[dict]: @@ -62,6 +68,8 @@ def convert_size_to_bytes(size: str) -> int: Example: 42G --> 45097156608 bytes """ + total_bytes = int() + if size.endswith("T"): total_bytes = int(size.split("T")[0]) * (1024**4) @@ -157,8 +165,14 @@ if __name__ == "__main__": dry_run = args["--dry-run"] # Initialize client and login - qb = Client(hostname) - qb.login(username=username, password=password) + qbt_client = qbittorrentapi.Client( + host=hostname, username=username, password=password + ) + + try: + qbt_client.auth_log_in() + except qbittorrentapi.LoginFailed as e: + print(e) # Fill the knapsack knapsack = fill_knapsack(max_seeders, knapsack_size, smaller) @@ -183,13 +197,15 @@ if __name__ == "__main__": for torrent in knapsack: if "gen.lib.rus.ec" in torrent["link"]: new_torrent = torrent["link"].replace("gen.lib.rus.ec", "libgen.is") - qb.download_from_link(new_torrent, category="scihub") + qbt_client.torrents_add(new_torrent, category="scihub") if "libgen.rs" in torrent["link"]: new_torrent = torrent["link"].replace("libgen.rs", "libgen.is") - qb.download_from_link(new_torrent, category="scihub") + qbt_client.torrents_add(new_torrent, category="scihub") # print(f"Added {torrent['name']}") + qbt_client.auth_log_out() + print("----------------") print(f"Count: {len(knapsack)} torrents") print(f"Total combined size: {get_knapsack_weight(knapsack)}") diff --git a/shell.nix b/shell.nix deleted file mode 100644 index 12eed60..0000000 --- a/shell.nix +++ /dev/null @@ -1,44 +0,0 @@ -with import { }; - -let - python-qbittorrent = pkgs.python312Packages.buildPythonPackage rec { - name = "python-qbittorrent-${version}"; - version = "0.4.3"; - - src = pkgs.fetchurl { - url = "https://files.pythonhosted.org/packages/86/25/a5ad35ad229c8016a8c98327495e649cb795be2fda63f8cace6c9a739af7/python-qbittorrent-${version}.tar.gz"; - sha256 = "4e22cf89890628b054a60aa4bd1161a68c2b0fad48ef0886fa4d325e69d3828a"; - }; - - meta = { - homepage = "https://github.com/v1k45/python-qBittorrent"; - description = "Python wrapper for qBittorrent Web API (for versions above v3.1.x)"; - license = lib.licenses.mit; - maintainers = with maintainers; [ v1k45 ]; - }; - - nativeBuildInputs = with pkgs.python312Packages; [ - pip - requests - ]; - }; - -in -mkShell { - buildInputs = with pkgs; [ - python312Packages.beautifulsoup4 - python312Packages.black - python312Packages.bpython - python312Packages.docopt - python312Packages.isort - python312Packages.pandas - python312Packages.pytest - python312Packages.requests - python312Packages.resend - python312Packages.rich - python312Packages.tabulate - pyright - python-qbittorrent - shellcheck - ]; -} diff --git a/sync_from_remotes.py b/sync_from_remotes.py index 44725dd..3bf86dd 100755 --- a/sync_from_remotes.py +++ b/sync_from_remotes.py @@ -1,5 +1,8 @@ -#!/usr/bin/env nix-shell -#! nix-shell -i python3 --packages python3 python312Packages.resend +# /// script +# dependencies = [ +# "resend", +# ] +# /// import socket import subprocess diff --git a/sync_to_remotes.py b/sync_to_remotes.py index 53192dd..c4f6a8f 100755 --- a/sync_to_remotes.py +++ b/sync_to_remotes.py @@ -1,5 +1,9 @@ -#!/usr/bin/env nix-shell -#! nix-shell -i python3 --packages python3 python312Packages.resend +# /// script +# dependencies = [ +# "resend", +# ] +# /// + import socket import subprocess diff --git a/update_tracker.py b/update_tracker.py index 958ec69..8a73a6c 100755 --- a/update_tracker.py +++ b/update_tracker.py @@ -1,4 +1,10 @@ -#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "qbittorrent-api", +# "docopt", +# "rich", +# ] +# /// """update_tracker.py @@ -28,8 +34,8 @@ import subprocess import tempfile from pathlib import Path +import qbittorrentapi from docopt import docopt -from qbittorrent import Client from rich.console import Console from rich.text import Text @@ -53,11 +59,18 @@ if __name__ == "__main__": ) torrent_infohashes = [] for item in auth_data["instances"]: - qb = Client(item["hostname"]) - qb.login(username=item["username"], password=item["password"]) + with qbittorrentapi.Client( + host=item["hostname"], + username=item["username"], + password=item["password"], + ) as qbt_client: + try: + qbt_client.auth_log_in() + except qbittorrentapi.LoginFailed as e: + print(e) - for torrent in qb.torrents(): - torrent_infohashes.append(torrent.get("hash")) # type: ignore + for torrent in qbt_client.torrents_info(): + torrent_infohashes.append(torrent.hash) # Format the infohashes to have a \n at the end console.log("Formatting infohashes to have a newline at the end.") @@ -65,60 +78,34 @@ if __name__ == "__main__": # Create a NamedTemporaryFile and write all infohashes to it, one per line console.log("Creating temporary file to write infohashes to.") + with tempfile.NamedTemporaryFile() as ntf: with open(ntf.name, "w") as tf: tf.writelines(format_infohashes) - # Use scp to copy the infohashes file to the torrent tracker's config - # directory on the remote torrent tracker server, overwriting the - # whitelist.txt file + # Use `sudo cp -f` to copy the infohashes file to the torrent tracker's config + # directory, overwriting the whitelist.txt file. console.log( - "SSH-copying the temporary infohashes file to the torrent tracker's whitelist." + "Copying the temporary infohashes file to the torrent tracker's whitelist." ) subprocess.run( - [ - "scp", - "-q", - ntf.name, - f"root@{tracker_domain}:/etc/opentracker/whitelist.txt", - ] + ["sudo", "cp", "-f", ntf.name, "/etc/opentracker/whitelist.txt"] ) - # Use SSH to run `systemctl restart opentracker.service` on the remote - # torrent tracker server - console.log("Restarting opentracker.service on the remote server.") - subprocess.run( - [ - "ssh", - f"root@{tracker_domain}", - "systemctl", - "restart", - "opentracker.service", - ] - ) - - # Ensure {tracker_domain}:6969/announce is added to each torrent's - # tracker list. - if tracker_domain: - console.log( - f"Ensuring {tracker_domain}:6969/announce is added to each torrent's tracker list." - ) - for item in auth_data["instances"]: - qb = Client(item["hostname"]) - qb.login(username=item["username"], password=item["password"]) - for torrent in qb.torrents(): - qb.add_trackers( - torrent.get("hash"), # type: ignore - f"http://{tracker_domain}:6969/announce\nudp://{tracker_domain}:6969/announce", - ) + # Run `sudo systemctl restart opentracker.service` + console.log("Restarting opentracker.service") + subprocess.run(["sudo", "systemctl", "restart", "opentracker.service"]) # Reannounce all torrents in each qBittorrent instance to their trackers console.log("Reannouncing all torrents to their trackers.") for item in auth_data["instances"]: - qb = Client(item["hostname"]) - qb.login(username=item["username"], password=item["password"]) - torrent_infohashes = [torrent.get("hash") for torrent in qb.torrents()] # type: ignore - qb.reannounce(torrent_infohashes) + with qbittorrentapi.Client( + host=item["hostname"], + username=item["username"], + password=item["password"], + ) as qbt_client: + for torrent in qbt_client.torrents_info(): + torrent.reannounce() console.log("Done!")