From e7989a267d5a594cd3a48919093eefc21a9e5b4e Mon Sep 17 00:00:00 2001 From: Jeffrey Serio Date: Thu, 20 Feb 2025 13:41:43 -0600 Subject: [PATCH] Add sync_wikimedia_xmldumps --- shell/sync_wikimedia_xmldumps | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 shell/sync_wikimedia_xmldumps diff --git a/shell/sync_wikimedia_xmldumps b/shell/sync_wikimedia_xmldumps new file mode 100755 index 0000000..cdd8aef --- /dev/null +++ b/shell/sync_wikimedia_xmldumps @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# Use rclone to sync the last two good Wikimedia XML data dumps. + +set -euxo pipefail + +wget https://wikimedia.bringyour.com/rsync-filelist-last-2-good.txt -O /home/jas/rsync-filelist-last-2-good.txt +rclone sync --http-no-head --transfers 8 --include-from /home/jas/rsync-filelist-last-2-good.txt wikimedia.bringyour.com: /naspool/archives/wikimedia-xmldatadumps +rm -v /home/jas/rsync-filelist-last-2-good.txt + +exit 0