refactor: use rich.status and console.log

This commit is contained in:
Jeffrey Serio 2023-09-16 15:41:09 -05:00
parent eec1bc7211
commit d66bf2f768
2 changed files with 21 additions and 15 deletions

View File

@ -66,7 +66,7 @@ class FandomWiki:
starting_url = f"{self.canonical_url}{mw_allpages_nav.find_all('a')[1].get('href')}"
hop0_urls.append(starting_url)
console.print(f"[[bold]HOP 0[/bold]] {starting_url}")
console.log(starting_url)
return hop0_urls
@ -85,7 +85,7 @@ class FandomWiki:
) and "Special:" not in item.get("href"):
new_url = f"{self.breezewiki_url}{item.get('href')}"
hop1_urls.append(new_url)
console.print(f"[[bold]HOP 1[/bold]] {new_url}")
console.log(new_url)
return hop1_urls
@ -108,7 +108,7 @@ class FandomWiki:
with open(css_filename, "wb") as outfile:
outfile.write(response.content)
console.print(f"[[bold green]CSS[/bold green]] {css_filename}")
console.log(css_filename)
def save_img(self, img_url: str):
filename = self.images_dir.joinpath(Path(img_url.split("/revision")[0]).name)
@ -120,9 +120,7 @@ class FandomWiki:
for chunk in response.iter_content(chunk_size=8192):
outfile.write(chunk)
console.print(f"[[bold green]IMG[/bold green]] {filename}")
else:
console.print(f"[[bold yellow]IMG (EXISTS)[/bold yellow]] {filename}")
console.log(filename)
def fetch_all_images(self, page_url: str):
response = requests.get(page_url)
@ -175,13 +173,9 @@ class FandomWiki:
with open(filename, "w") as outfile:
outfile.write(soup.prettify())
console.print(f"[[bold green]HTML[/bold green]] {filename}")
else:
console.print(f"[[bold yellow]HTML (EXISTS)[/bold yellow]] {filename}")
console.log(filename)
def fetch_all_pages(self, hop1_urls: list):
self.save_css()
with concurrent.futures.ThreadPoolExecutor() as executor:
executor.map(self.save_page, hop1_urls)
@ -236,13 +230,25 @@ class FandomWiki:
shutil.rmtree(self.site_dir, ignore_errors=True)
console.print(f"\nTotal web files scraped: {len(web_files)}")
console.print(f"Total images scraped: {len(img_files)}")
console.log(f"\nTotal web files scraped: {len(web_files)}")
console.log(f"Total images scraped: {len(img_files)}")
def archive_site(name: str):
site = FandomWiki(name)
site.fetch_all_pages(site.get_hop1_urls(site.get_hop0_urls()))
with console.status("Fetching hop 0 URLs...", spinner="aesthetic"):
hop0_urls = site.get_hop0_urls()
with console.status("Fetching hop 1 URLs...", spinner="aesthetic"):
hop1_urls = site.get_hop1_urls(hop0_urls)
with console.status("Saving CSS files...", spinner="aesthetic"):
site.save_css()
with console.status("Downloading images and web pages...", spinner="aesthetic"):
site.fetch_all_pages(hop1_urls)
site.archive()

View File

@ -1,6 +1,6 @@
[tool.poetry]
name = "archive-fandom-wiki"
version = "0.1.1"
version = "0.1.2"
description = "Archive fandom wikis"
authors = ["Jeffrey Serio <hyperreal@fedoraproject.org>"]
license = "GPL-3.0"