mirror of
https://codeberg.org/hyperreal/archive-fandom-wiki
synced 2024-11-01 08:33:06 +01:00
fix make_archive
This commit is contained in:
parent
56beec24c2
commit
3d82063356
50
afw.py
50
afw.py
@ -3,15 +3,18 @@
|
||||
"""archive-fandom-wiki
|
||||
|
||||
Usage:
|
||||
afw <fandom> [<breezewiki_instance>]
|
||||
afw -f <fandom> [-w <max_workers>] [-b <breezewiki_url>]
|
||||
afw -h
|
||||
|
||||
Options:
|
||||
-h --help Show this help message.
|
||||
-f <fandom> The fandom to archive. (Required)
|
||||
-w <max_workers> The maximum number of workers to use for concurrent threads. (Optional; Default is 4)
|
||||
-b <breezewiki_url> The URL of the BreezeWiki instance to use. (Optional; Default is https://breezewiki.hyperreal.coffee)
|
||||
-h --help Show this help message.
|
||||
|
||||
Examples:
|
||||
afw dishonored https://breezewiki.hyperreal.coffee
|
||||
afw residentevil
|
||||
afw -f dishonored -w 16 -b https://breezewiki.hyperreal.coffee
|
||||
afw -f residentevil
|
||||
"""
|
||||
|
||||
# This file is formatted with `black -l 79' to comply with PEP8 standards.
|
||||
@ -42,9 +45,11 @@ console = Console()
|
||||
class FandomWiki:
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
self.canonical_url = f"https://{name}.fandom.com"
|
||||
self.canonical_name = f"{name}.fandom.com"
|
||||
self.canonical_url = f"https://{self.canonical_name}"
|
||||
self.breezewiki_url = "https://breezewiki.hyperreal.coffee"
|
||||
self.site_dir = Path(f"{name}.fandom.com")
|
||||
self.archive_rootdir = Path.cwd()
|
||||
self.site_dir = self.archive_rootdir.joinpath(f"{self.canonical_name}")
|
||||
self.images_dir = self.site_dir.joinpath("images")
|
||||
|
||||
def set_breezewiki_url(self, breezewiki_url: str):
|
||||
@ -182,7 +187,8 @@ class FandomWiki:
|
||||
console.log(filename)
|
||||
|
||||
def fetch_all_pages(self, hop1_urls: list):
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
max_workers = int(args["-w"]) if args["-w"] else 4
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
executor.map(self.save_page, hop1_urls)
|
||||
|
||||
def archive_site(self):
|
||||
@ -204,8 +210,8 @@ class FandomWiki:
|
||||
with console.status("Fetching hop 1 URLs...", spinner="aesthetic"):
|
||||
hop1_urls = self.get_hop1_urls(hop0_urls)
|
||||
|
||||
self.site_dir.mkdir()
|
||||
self.images_dir.mkdir()
|
||||
# Creates the parent dirs: self.archive_rootdir > self.site_dir > self.images_dir
|
||||
self.images_dir.mkdir(parents=True)
|
||||
|
||||
with console.status("Saving CSS files...", spinner="aesthetic"):
|
||||
self.save_css()
|
||||
@ -221,24 +227,32 @@ class FandomWiki:
|
||||
timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
|
||||
with console.status("Archiving images...", spinner="aesthetic"):
|
||||
shutil.make_archive(f"images-{timestamp}", "xztar", self.images_dir)
|
||||
shutil.make_archive(
|
||||
"images",
|
||||
"xztar",
|
||||
root_dir=self.site_dir,
|
||||
base_dir="images",
|
||||
)
|
||||
shutil.move("images.tar.xz", self.site_dir)
|
||||
shutil.rmtree(self.images_dir)
|
||||
shutil.move(f"images-{timestamp}.tar.xz", self.site_dir)
|
||||
|
||||
with console.status("Archiving web files...", spinner="aesthetic"):
|
||||
shutil.make_archive(f"{self.name}-{timestamp}", "gztar", self.site_dir)
|
||||
|
||||
shutil.rmtree(self.site_dir)
|
||||
shutil.make_archive(
|
||||
f"{self.name}-{timestamp}",
|
||||
"gztar",
|
||||
root_dir=self.archive_rootdir,
|
||||
base_dir=self.canonical_name,
|
||||
)
|
||||
|
||||
console.log(f"\nTotal web files archived: {total_web_files}")
|
||||
console.log(f"Total images archived: {total_image_files}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = docopt(__doc__, options_first=True, help=True, version="1.0.1")
|
||||
args = docopt(__doc__, options_first=True, help=True, version="1.0.1") # type: ignore
|
||||
|
||||
site = FandomWiki(args["<fandom>"])
|
||||
if args["<breezewiki_instance>"]:
|
||||
site.set_breezewiki_url(args["<breezewiki_instance>"])
|
||||
site = FandomWiki(args["-f"])
|
||||
if args["-b"]:
|
||||
site.set_breezewiki_url(args["-b"])
|
||||
|
||||
site.archive_site()
|
||||
|
Loading…
Reference in New Issue
Block a user