hyperreal/admin-scripts
1
0
Fork 0
mirror of https://codeberg.org/hyperreal/admin-scripts synced 2024-11-25 09:03:41 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update

Browse Source
This commit is contained in:
Jeffrey Serio 2024-11-08 17:18:47 -06:00
parent 7e09519e62
commit 8af36a2a5c
15 changed files with 279 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
.archived/mastodon.py Executable file
View File

@ -0,0 +1,36 @@
#!/usr/bin/env python3
import pandas as pd
from pandas import json_normalize
import json
from html import unescape
with open("/home/jas/downloads/outbox.json", "r") as jf:
json_data = json.load(jf)
flattened_df = json_normalize(json_data, record_path=["orderedItems"])
published = []
for item in flattened_df["object.published"]:
published.append(item)
content = []
for item in flattened_df["object.content"]:
content.append(item)
x = zip(published, content)
print("#+TITLE: Mastodon posts, 2024-02-16T15:48:46Z - 2024-10-11T20:15:03Z")
print("#+SETUPFILE: ../org-templates/page.org")
print()
for item in x:
if type(item[0]) is str:
print(f"*** {item[0]}")
if type(item[1]) is str:
print("#+BEGIN_QUOTE")
print("#+BEGIN_EXPORT html")
print(unescape(item[1]))
print("#+END_EXPORT")
print("#+END_QUOTE")
print()

1
.envrc
View File

@ -1 +0,0 @@
use nix

2
.gitignore vendored
View File

@ -1 +1,3 @@
.direnv
.venv

9
calculate_mirror_size.py
View File

@ -1,4 +1,11 @@
#!/usr/bin/env python3
# /// script
# dependencies = [
# "requests",
# "bs4",
# "docopt",
# "rich",
# ]
# ///
"""calculate_mirror_size.py

9
fetch_combined_trackers_list.py
View File

@ -1,4 +1,9 @@
#!/usr/bin/env python3
# /// script
# dependencies = [
# "requests",
# "docopt",
# ]
# ///
"""fetch_combined_trackers_list.py
@ -20,7 +25,7 @@ import requests
from docopt import docopt
if __name__ == "__main__":
args = docopt(__doc__) # type: ignore
args = docopt(__doc__) # type: ignore
live_trackers_list_urls = [
"https://newtrackon.com/api/stable",

7
fetch_scihub_infohashes.py
View File

@ -1,4 +1,9 @@
#!/usr/bin/env python3
# /// script
# dependencies = [
# "requests",
# "docopt",
# ]
# ///
"""fetch_scihub_infohashes.py

58
list_torrents.py
View File

@ -1,4 +1,9 @@
#!/home/jas/virtualenvs/list_torrents/bin/python3
# /// script
# dependencies = [
# "qbittorrent-api",
# "docopt",
# ]
# ///
"""list_torrents.py
@ -15,9 +20,8 @@ Options:
-h, --help show this help message and exit
"""
import qbittorrentapi
from docopt import docopt
from qbittorrent import Client
from tabulate import tabulate
# convert byte units
@ -43,36 +47,26 @@ def human_bytes(input_bytes: int) -> str:
return ""
def print_table():
qb = Client("http://localhost:8080/")
qb.login()
table = []
headers = ["Name", "Total Size", "Trackers Count", "Ratio", "Uploaded"]
sorted_torrents = sorted(qb.torrents(), key=lambda d: d["ratio"], reverse=True) # type: ignore
for torrent in sorted_torrents:
row = []
row.append(torrent["name"]) # type: ignore
row.append(human_bytes(int(torrent["total_size"]))) # type: ignore
row.append(torrent["trackers_count"]) # type: ignore
row.append(torrent["ratio"]) # type: ignore
row.append(human_bytes(int(torrent["uploaded"]))) # type: ignore
table.append(row)
print(tabulate(table, headers, tablefmt="grid"))
def print_ssv():
qb = Client("http://localhost:8080/")
qb.login()
sorted_torrents = sorted(qb.torrents(), key=lambda d: d["ratio"], reverse=True) # type: ignore
print("Name Size Trackers Ratio Uploaded")
for torrent in sorted_torrents:
name = torrent["name"] # type: ignore
size = human_bytes(int(torrent["total_size"])) # type: ignore
trackers = torrent["trackers_count"] # type: ignore
ratio = torrent["ratio"] # type: ignore
uploaded = human_bytes(int(torrent["uploaded"])) # type: ignore
print(f"{name} {size} {trackers} {ratio} {uploaded}")
with qbittorrentapi.Client(
host="localhost", port=8080, username="", password=""
) as qbt_client:
try:
qbt_client.auth_log_in()
except qbittorrentapi.LoginFailed as e:
print(e)
sorted_torrents = sorted(
qbt_client.torrents_info(), key=lambda d: d.ratio, reverse=True
)
print("Name Size # of Trackers Ratio Uploaded")
for torrent in sorted_torrents:
name = torrent.name
size = human_bytes(torrent.total_size)
trackers = torrent.trackers_count
ratio = torrent.ratio
uploaded = human_bytes(torrent.uploaded)
print(f"{name} {size} {trackers} {ratio} {uploaded}")
if __name__ == "__main__":

4
qbt_stats_html.nu
View File

@ -2,12 +2,12 @@
let old_head = "<html><style>body { background-color:white;color:black; }</style><body>"
let new_head = (
["<html><head><title>Torrent Stats</title><link type="text/css" rel="stylesheet" href="https://files.hyperreal.coffee/css/main-style.css"/></head><body><h4>Last updated:", (date now | format date "%F %T%:z"), "</h4>"]
["<html><head><title>Torrent Stats</title><link type="text/css" rel="stylesheet" href="https://files.hyperreal.coffee/css/style1.css"/></head><body><h4>Last updated:", (date now | format date "%F %T%:z"), "</h4>"]
| str join ' '
)
(
/home/jas/admin-scripts/list_torrents.py
uv run -q /home/jas/admin-scripts/list_torrents.py
| from ssv -m 2
| to html
| str replace ($old_head) ($new_head)

38
qbt_sum_size.py
View File

@ -1,4 +1,9 @@
#!/usr/bin/env python3
# /// script
# dependencies = [
# "qbittorrent-api",
# "docopt",
# ]
# ///
"""qbt_sum_size.py
@ -18,8 +23,8 @@ Options:
-h, --help show this help message and exit
"""
import qbittorrentapi
from docopt import docopt
from qbittorrent import Client
# convert byte units
@ -48,21 +53,26 @@ def human_bytes(bites: int) -> str:
if __name__ == "__main__":
args = docopt(__doc__) # type: ignore
# Initialize client and login
qb = Client(args["HOSTNAME"])
qb.login(username=args["USERNAME"], password=args["PASSWORD"])
# get total_completed_bytes
completed_torrent_sizes = []
for torrent in qb.torrents():
if torrent["completion_on"] != 0: # type: ignore
completed_torrent_sizes.append(torrent["total_size"]) # type: ignore
total_added_bytes = int()
with qbittorrentapi.Client(
host=args["HOSTNAME"], username=args["USERNAME"], password=args["PASSWORD"]
) as qbt_client:
try:
qbt_client.auth_log_in()
except qbittorrentapi.LoginFailed as e:
print(e)
for torrent in qbt_client.torrents_info():
if torrent.completion_on != 0:
completed_torrent_sizes.append(torrent.total_size)
total_added_bytes = sum(
[torrent.total_size for torrent in qbt_client.torrents_info()]
)
total_completed_bytes = sum(completed_torrent_sizes)
# get total_added_bytes
total_added_bytes = sum([torrent["total_size"] for torrent in qb.torrents()]) # type: ignore
# print the results
print(f"\nTotal completed size: {human_bytes(total_completed_bytes)}")
print(f"Total added size: {human_bytes(total_added_bytes)}\n")

133
qbth.py
View File

@ -1,4 +1,11 @@
#!/usr/bin/env python3
# /// script
# dependencies = [
# "qbittorrent-api",
# "requests",
# "bs4",
# "docopt",
# ]
# ///
"""qbth.py - qbittorrent helper
@ -17,15 +24,25 @@ Options:
import json
import os
import subprocess
from shutil import which
import qbittorrentapi
import requests
from bs4 import BeautifulSoup
from docopt import docopt
from qbittorrent import Client
args = docopt(__doc__)
qb = Client(args["HOSTNAME"])
qb.login(username=args["USERNAME"], password=args["PASSWORD"])
conn_info = dict(
host=args["HOSTNAME"],
username=args["USERNAME"],
password=args["PASSWORD"],
)
try:
with qbittorrentapi.Client(**conn_info) as qbt_client:
qbt_client.auth_log_in()
except qbittorrentapi.LoginFailed as e:
print(e)
def add_torrents(urls: list[str]):
@ -35,9 +52,16 @@ def add_torrents(urls: list[str]):
Params:
urls: list of strings that are URLs.
"""
for url in urls:
qb.download_from_link(url, category="distro")
print(f"Added {os.path.basename(url)}")
with qbittorrentapi.Client(**conn_info) as qbt_client:
for url in urls:
response = requests.get(url)
if response.status_code == 200:
if qbt_client.torrents_add(url, category="distro") != "Ok.":
raise Exception("Failed to add torrent: " + os.path.basename(url))
else:
print(f"Added {os.path.basename(url)}")
else:
print(f"{response.status_code}: {url}")
def add_torrents_from_html(webpage_url: str, torrent_substring: str):
@ -52,11 +76,21 @@ def add_torrents_from_html(webpage_url: str, torrent_substring: str):
"""
reqs = requests.get(webpage_url, timeout=60)
soup = BeautifulSoup(reqs.text, "html.parser")
for link in soup.find_all("a"):
if torrent_substring in link.get("href"):
url = f"{webpage_url}/{link.get('href')}"
qb.download_from_link(url, category="distro")
print(f"Added {link.get('href')}")
with qbittorrentapi.Client(**conn_info) as qbt_client:
for link in soup.find_all("a"):
if torrent_substring in link.get("href"):
url = f"{webpage_url}/{link.get('href')}"
response = requests.get(url)
if response.status_code == 200:
if qbt_client.torrents_add(url, category="distro") != "Ok.":
raise Exception(
"Failed to add torrent: " + os.path.basename(url)
)
else:
print(f"Added {os.path.basename(url)}")
else:
print(f"{response.status_code}: {url}")
def remove_torrents(distro_substring: str):
@ -69,10 +103,13 @@ def remove_torrents(distro_substring: str):
distro_substring: a string that is a substring of the distro
torrent's file name.
"""
for torrent in qb.torrents():
if distro_substring in torrent.get("name"): # type: ignore
qb.delete_permanently(torrent.get("hash")) # type: ignore
print(f"Removed {torrent.get('name')}") # type: ignore
with qbittorrentapi.Client(**conn_info) as qbt_client:
for torrent in qbt_client.torrents_info():
if distro_substring in torrent.name:
qbt_client.torrents_delete(
torrent_hashes=torrent.hash, delete_files=True
)
print(f"Removed {torrent.name}")
def add_almalinux(rel_ver: str):
@ -142,8 +179,7 @@ def add_devuan(rel_ver: str):
relver: the Devuan release version.
"""
url = f"https://files.devuan.org/devuan_{rel_ver}.torrent"
qb.download_from_link(url, category="distro")
print(f"Added {os.path.basename(url)}")
add_torrents([url])
def remove_devuan(rel_ver: str):
@ -175,11 +211,13 @@ def remove_fedora(rel_ver: str):
Params:
relver: the Fedora release version.
"""
torrents = qb.torrents()
for torrent in torrents:
if torrent.get("name").startswith("Fedora") and torrent.get("name").endswith(rel_ver): # type: ignore
qb.delete_permanently(torrent.get("hash")) # type: ignore
print(f"Removed {torrent.get('name')}") # type: ignore
with qbittorrentapi.Client(**conn_info) as qbt_client:
for torrent in qbt_client.torrents_info():
if torrent.name.startswith("Fedora") and torrent.name.endswith(rel_ver):
qbt_client.torrents_delete(
torrent_hashes=torrent.hash, delete_files=True
)
print(f"Removed {torrent.name}")
def add_freebsd(rel_ver: str):
@ -194,10 +232,13 @@ def add_freebsd(rel_ver: str):
reqs = requests.get(url, timeout=60)
data = reqs.text.split("\n")
for line in data:
if line.startswith("magnet:"):
qb.download_from_link(line, category="distro")
print(f"Added {line.split('=')[2]}")
with qbittorrentapi.Client(**conn_info) as qbt_client:
for line in data:
if line.startswith("magnet:"):
if qbt_client.torrents_add(line) != "Ok.":
raise Exception("Failed to add torrent: " + line.split("=")[2])
print(f"Added {line.split('=')[2]}")
def remove_freebsd(rel_ver: str):
@ -266,9 +307,25 @@ def add_nixos():
url = "https://api.github.com/repos/AnimMouse/NixOS-ISO-Torrents/releases/latest"
reqs = requests.get(url, timeout=60)
json_data = json.loads(reqs.text)
for item in json_data["assets"]:
qb.download_from_link(item["browser_download_url"], category="distro")
print(f"Added {os.path.basename(item['browser_download_url'])}")
with qbittorrentapi.Client(**conn_info) as qbt_client:
for item in json_data["assets"]:
response = requests.get(item["browser_download_url"])
if response.status_code == 200:
if (
qbt_client.torrents_add(
item["browser_download_url"], category="distro"
)
!= "Ok."
):
raise Exception(
"Failed to add torrent: "
+ os.path.basename(item["browser_download_url"])
)
else:
print(f"Added {os.path.basename(item['browser_download_url'])}")
else:
print(f"{response.status_code}: {item['browser_download_url']}")
def remove_nixos():
@ -289,8 +346,16 @@ def add_qubes(rel_ver: str):
relver: the QubesOS release version.
"""
url = f"https://mirrors.edge.kernel.org/qubes/iso/Qubes-R{rel_ver}-x86_64.torrent"
qb.download_from_link(url, category="distro")
print(f"Added {os.path.basename(url)}")
response = requests.get(url)
if response.status_code == 200:
with qbittorrentapi.Client(**conn_info) as qbt_client:
if qbt_client.torrents_add(url, category="distro") != "Ok.":
raise Exception("Failed to add torrent: " + os.path.basename(url))
else:
print(f"Added {os.path.basename(url)}")
else:
print(f"{response.status_code}: {url}")
def remove_qubes(rel_ver: str):
@ -356,6 +421,10 @@ def remove_tails(rel_ver: str):
if __name__ == "__main__":
# Check if gum is installed.
if which("gum") is None:
exit("Please install gum first. https://github.com/charmbracelet/gum")
# Run the gum program in a subprocess to allow easy selecting of distro
# torrents.
distro_selection = subprocess.run(

28
scihub_knapsack.py
View File

@ -1,4 +1,10 @@
#!/usr/bin/env python3
# /// script
# dependencies = [
# "qbittorrent-api",
# "requests",
# "docopt",
# ]
# ///
"""scihub_knapsack.py
@ -39,9 +45,9 @@ Options:
import json
import qbittorrentapi
import requests
from docopt import docopt
from qbittorrent import Client
def get_torrent_health_data() -> list[dict]:
@ -62,6 +68,8 @@ def convert_size_to_bytes(size: str) -> int:
Example: 42G --> 45097156608 bytes
"""
total_bytes = int()
if size.endswith("T"):
total_bytes = int(size.split("T")[0]) * (1024**4)
@ -157,8 +165,14 @@ if __name__ == "__main__":
dry_run = args["--dry-run"]
# Initialize client and login
qb = Client(hostname)
qb.login(username=username, password=password)
qbt_client = qbittorrentapi.Client(
host=hostname, username=username, password=password
)
try:
qbt_client.auth_log_in()
except qbittorrentapi.LoginFailed as e:
print(e)
# Fill the knapsack
knapsack = fill_knapsack(max_seeders, knapsack_size, smaller)
@ -183,13 +197,15 @@ if __name__ == "__main__":
for torrent in knapsack:
if "gen.lib.rus.ec" in torrent["link"]:
new_torrent = torrent["link"].replace("gen.lib.rus.ec", "libgen.is")
qb.download_from_link(new_torrent, category="scihub")
qbt_client.torrents_add(new_torrent, category="scihub")
if "libgen.rs" in torrent["link"]:
new_torrent = torrent["link"].replace("libgen.rs", "libgen.is")
qb.download_from_link(new_torrent, category="scihub")
qbt_client.torrents_add(new_torrent, category="scihub")
# print(f"Added {torrent['name']}")
qbt_client.auth_log_out()
print("----------------")
print(f"Count: {len(knapsack)} torrents")
print(f"Total combined size: {get_knapsack_weight(knapsack)}")

44
shell.nix
View File

@ -1,44 +0,0 @@
with import <nixpkgs> { };
let
python-qbittorrent = pkgs.python312Packages.buildPythonPackage rec {
name = "python-qbittorrent-${version}";
version = "0.4.3";
src = pkgs.fetchurl {
url = "https://files.pythonhosted.org/packages/86/25/a5ad35ad229c8016a8c98327495e649cb795be2fda63f8cace6c9a739af7/python-qbittorrent-${version}.tar.gz";
sha256 = "4e22cf89890628b054a60aa4bd1161a68c2b0fad48ef0886fa4d325e69d3828a";
};
meta = {
homepage = "https://github.com/v1k45/python-qBittorrent";
description = "Python wrapper for qBittorrent Web API (for versions above v3.1.x)";
license = lib.licenses.mit;
maintainers = with maintainers; [ v1k45 ];
};
nativeBuildInputs = with pkgs.python312Packages; [
pip
requests
];
};
in
mkShell {
buildInputs = with pkgs; [
python312Packages.beautifulsoup4
python312Packages.black
python312Packages.bpython
python312Packages.docopt
python312Packages.isort
python312Packages.pandas
python312Packages.pytest
python312Packages.requests
python312Packages.resend
python312Packages.rich
python312Packages.tabulate
pyright
python-qbittorrent
shellcheck
];
}

7
sync_from_remotes.py
View File

@ -1,5 +1,8 @@
#!/usr/bin/env nix-shell
#! nix-shell -i python3 --packages python3 python312Packages.resend
# /// script
# dependencies = [
# "resend",
# ]
# ///
import socket
import subprocess

8
sync_to_remotes.py
View File

@ -1,5 +1,9 @@
#!/usr/bin/env nix-shell
#! nix-shell -i python3 --packages python3 python312Packages.resend
# /// script
# dependencies = [
# "resend",
# ]
# ///
import socket
import subprocess

81
update_tracker.py
View File

@ -1,4 +1,10 @@
#!/usr/bin/env python3
# /// script
# dependencies = [
# "qbittorrent-api",
# "docopt",
# "rich",
# ]
# ///
"""update_tracker.py
@ -28,8 +34,8 @@ import subprocess
import tempfile
from pathlib import Path
import qbittorrentapi
from docopt import docopt
from qbittorrent import Client
from rich.console import Console
from rich.text import Text
@ -53,11 +59,18 @@ if __name__ == "__main__":
)
torrent_infohashes = []
for item in auth_data["instances"]:
qb = Client(item["hostname"])
qb.login(username=item["username"], password=item["password"])
with qbittorrentapi.Client(
host=item["hostname"],
username=item["username"],
password=item["password"],
) as qbt_client:
try:
qbt_client.auth_log_in()
except qbittorrentapi.LoginFailed as e:
print(e)
for torrent in qb.torrents():
torrent_infohashes.append(torrent.get("hash")) # type: ignore
for torrent in qbt_client.torrents_info():
torrent_infohashes.append(torrent.hash)
# Format the infohashes to have a \n at the end
console.log("Formatting infohashes to have a newline at the end.")
@ -65,60 +78,34 @@ if __name__ == "__main__":
# Create a NamedTemporaryFile and write all infohashes to it, one per line
console.log("Creating temporary file to write infohashes to.")
with tempfile.NamedTemporaryFile() as ntf:
with open(ntf.name, "w") as tf:
tf.writelines(format_infohashes)
# Use scp to copy the infohashes file to the torrent tracker's config
# directory on the remote torrent tracker server, overwriting the
# whitelist.txt file
# Use `sudo cp -f` to copy the infohashes file to the torrent tracker's config
# directory, overwriting the whitelist.txt file.
console.log(
"SSH-copying the temporary infohashes file to the torrent tracker's whitelist."
"Copying the temporary infohashes file to the torrent tracker's whitelist."
)
subprocess.run(
[
"scp",
"-q",
ntf.name,
f"root@{tracker_domain}:/etc/opentracker/whitelist.txt",
]
["sudo", "cp", "-f", ntf.name, "/etc/opentracker/whitelist.txt"]
)
# Use SSH to run `systemctl restart opentracker.service` on the remote
# torrent tracker server
console.log("Restarting opentracker.service on the remote server.")
subprocess.run(
[
"ssh",
f"root@{tracker_domain}",
"systemctl",
"restart",
"opentracker.service",
]
)
# Ensure {tracker_domain}:6969/announce is added to each torrent's
# tracker list.
if tracker_domain:
console.log(
f"Ensuring {tracker_domain}:6969/announce is added to each torrent's tracker list."
)
for item in auth_data["instances"]:
qb = Client(item["hostname"])
qb.login(username=item["username"], password=item["password"])
for torrent in qb.torrents():
qb.add_trackers(
torrent.get("hash"), # type: ignore
f"http://{tracker_domain}:6969/announce\nudp://{tracker_domain}:6969/announce",
)
# Run `sudo systemctl restart opentracker.service`
console.log("Restarting opentracker.service")
subprocess.run(["sudo", "systemctl", "restart", "opentracker.service"])
# Reannounce all torrents in each qBittorrent instance to their trackers
console.log("Reannouncing all torrents to their trackers.")
for item in auth_data["instances"]:
qb = Client(item["hostname"])
qb.login(username=item["username"], password=item["password"])
torrent_infohashes = [torrent.get("hash") for torrent in qb.torrents()] # type: ignore
qb.reannounce(torrent_infohashes)
with qbittorrentapi.Client(
host=item["hostname"],
username=item["username"],
password=item["password"],
) as qbt_client:
for torrent in qbt_client.torrents_info():
torrent.reannounce()
console.log("Done!")