Add sync_wikimedia_xmldumps

2025-02-20 13:41:43 -06:00 · 2025-02-20 13:41:43 -06:00 · e7989a267d
commit e7989a267d
parent d3693a6370
1 changed files with 11 additions and 0 deletions
--- a/shell/sync_wikimedia_xmldumps
+++ b/shell/sync_wikimedia_xmldumps
@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# Use rclone to sync the last two good Wikimedia XML data dumps.
+
+set -euxo pipefail
+
+wget https://wikimedia.bringyour.com/rsync-filelist-last-2-good.txt -O /home/jas/rsync-filelist-last-2-good.txt
+rclone sync --http-no-head --transfers 8 --include-from /home/jas/rsync-filelist-last-2-good.txt wikimedia.bringyour.com: /naspool/archives/wikimedia-xmldatadumps
+rm -v /home/jas/rsync-filelist-last-2-good.txt
+
+exit 0