diff --git a/.gitignore b/.gitignore index af6cab7..1b476d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ annastorrents.exe +annastorrents +torrents.db +torrents.db-shm +torrents.db-wal \ No newline at end of file diff --git a/README.md b/README.md index 6270e48..9e0cea1 100644 --- a/README.md +++ b/README.md @@ -31,23 +31,33 @@ You can just go to the releases and download the binary and skip the installatio cd annas-archive-mirror ``` -3. Build the application (see Building section below). +3. Build and Setup the Application: + +``` +sudo chmod +x install.sh +sudo install.sh +``` 4. Run the application: ``` - ./annas-archive-mirror + sudo systemctl start annas-torrents.service ``` +or + ``` + ./annas-torrents + ``` + 5. Access the web interface by opening a browser and navigating to `http://localhost:8080` -## Building +## Build Yourself This application uses CGO and requires GCC to be installed on your system. Follow these steps to build the application: 1. Install Go (version 1.16 or later) from [golang.org](https://golang.org/) 2. Install GCC: - - On Ubuntu/Debian: `sudo apt-get install build-essentials` + - On Ubuntu/Debian: `sudo apt-get install build-essential` - On macOS: Install Xcode Command Line Tools - On Windows: Install MinGW-w64 (I didn't need to do this but other guides say you do) @@ -95,6 +105,7 @@ Example: - `/stats/{btih}`: Detailed statistics for a specific torrent - `/json`: Full torrent list in JSON format - `/generate-torrent-list`: Endpoint for generating custom torrent lists + - `/assets`: Serves everything in the assets folder within --directory (so ./assets by default) 5. **Visualization**: - It uses Chart.js to create visual representations of seeder statistics. diff --git a/assets/favicon.ico b/assets/favicon.ico new file mode 100644 index 0000000..d9d18b8 Binary files /dev/null and b/assets/favicon.ico differ diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..50c4592 --- /dev/null +++ b/install.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +set -e + +# Default values +DEFAULT_PORT=8080 +DEFAULT_DIRECTORY="/var/annas-torrents" + +# Function to install Go +install_go() { + echo "Installing Go..." + GO_VERSION="1.23.0" # Use the specific version you need + ARCH=$(uname -m) + + case $ARCH in + x86_64) ARCH="amd64" ;; + aarch64) ARCH="arm64" ;; + *) echo "Unsupported architecture: $ARCH"; exit 1 ;; + esac + + GO_TARBALL="go${GO_VERSION}.linux-${ARCH}.tar.gz" + wget "https://go.dev/dl/${GO_TARBALL}" + sudo tar -C /usr/local -xzf "${GO_TARBALL}" + rm "${GO_TARBALL}" + + export PATH=$PATH:/usr/local/go/bin + echo "export PATH=\$PATH:/usr/local/go/bin" >> ~/.bashrc +} + +# Function to install GCC +install_gcc() { + echo "Installing GCC..." + if [ -x "$(command -v apt-get)" ]; then + sudo apt-get update + sudo apt-get install -y build-essential + elif [ -x "$(command -v yum)" ]; then + sudo yum groupinstall -y "Development Tools" + elif [ -x "$(command -v zypper)" ]; then + sudo zypper install -t pattern devel_basis + elif [ -x "$(command -v pacman)" ]; then + sudo pacman -Sy --noconfirm base-devel + else + echo "Unsupported package manager. Please install GCC manually." + exit 1 + fi +} + +# Function to set up directory +setup_directory() { + echo "Setting up directory..." + sudo mkdir -p "$INSTALL_DIRECTORY" + sudo chown -R www-data:www-data "$INSTALL_DIRECTORY" + sudo chmod -R 755 "$INSTALL_DIRECTORY" +} + +# Prompt for port +read -p "Enter the port you want to use (default is $DEFAULT_PORT): " PORT +PORT=${PORT:-$DEFAULT_PORT} + +# Prompt for installation directory +read -p "Enter the installation directory (default is $DEFAULT_DIRECTORY): " INSTALL_DIRECTORY +INSTALL_DIRECTORY=${INSTALL_DIRECTORY:-$DEFAULT_DIRECTORY} + +# Prompt for hosting torrents locally +read -p "Do you want to host torrent files locally? (y/n, default is n): " HOST_TORRENTS +HOST_TORRENTS=${HOST_TORRENTS:-n} +TORRENT_FLAG="" +if [[ "$HOST_TORRENTS" =~ ^[Yy]$ ]]; then + TORRENT_FLAG="--torrents true" +fi + +# Check if Go is installed +if ! command -v go &> /dev/null; then + install_go +else + echo "Go is already installed." +fi + +# Check if GCC is installed +if ! command -v gcc &> /dev/null; then + install_gcc +else + echo "GCC is already installed." +fi + +# Install Go dependencies +echo "Installing Go dependencies..." +go get github.com/mattn/go-sqlite3 +go get github.com/etix/goscrape + +# Set up the directory with correct permissions +setup_directory + +# Build the application +echo "Building the application..." +go build -o /usr/bin/annas-torrents + +# If port 80 is selected, ensure the binary can bind to it +if [ "$PORT" -eq 80 ]; then + echo "You selected port 80. Configuring permissions..." + sudo setcap 'cap_net_bind_service=+ep' /usr/bin/annas-torrents +fi + +# Create systemd service file +echo "Creating systemd service..." +SERVICE_FILE="/etc/systemd/system/annas-torrents.service" + +sudo bash -c "cat > $SERVICE_FILE" < 10 THEN data_size ELSE 0 END) / 1099511627776.0 AS high_seeders_tb - FROM torrents - WHERE embargo = 0 - `) - if err != nil { - return fmt.Errorf("error querying seeder stats: %w", err) - } - defer rows.Close() - var lowSeedersTB, mediumSeedersTB, highSeedersTB float64 - if rows.Next() { - err = rows.Scan(&lowSeedersTB, &mediumSeedersTB, &highSeedersTB) + + // Use readDB for the query + err := withReadTx(func(tx *sql.Tx) error { + rows, err := tx.Query(` + SELECT + SUM(CASE WHEN seeders < 4 THEN data_size ELSE 0 END) / 1099511627776.0 AS low_seeders_tb, + SUM(CASE WHEN seeders BETWEEN 4 AND 10 THEN data_size ELSE 0 END) / 1099511627776.0 AS medium_seeders_tb, + SUM(CASE WHEN seeders > 10 THEN data_size ELSE 0 END) / 1099511627776.0 AS high_seeders_tb + FROM torrents + WHERE embargo = 0 + `) if err != nil { - return fmt.Errorf("error scanning row: %w", err) + return fmt.Errorf("error querying seeder stats: %w", err) } + defer rows.Close() + + if rows.Next() { + err = rows.Scan(&lowSeedersTB, &mediumSeedersTB, &highSeedersTB) + if err != nil { + return fmt.Errorf("error scanning row: %w", err) + } + } + + return rows.Err() + }) + + if err != nil { + return fmt.Errorf("error reading seeder stats: %w", err) } - // Check for any errors encountered during iteration - if err := rows.Err(); err != nil { - return fmt.Errorf("error during row iteration: %w", err) + // Use writeDB for the update + tx, err := writeDB.Begin() + if err != nil { + return fmt.Errorf("error starting write transaction: %w", err) } + defer tx.Rollback() // Insert or update the daily stats _, err = tx.Exec(` - INSERT OR REPLACE INTO daily_seeder_stats (date, low_seeders_tb, medium_seeders_tb, high_seeders_tb) - VALUES (DATE('now'), ?, ?, ?)`, + INSERT OR REPLACE INTO daily_seeder_stats (date, low_seeders_tb, medium_seeders_tb, high_seeders_tb) + VALUES (DATE('now'), ?, ?, ?)`, lowSeedersTB, mediumSeedersTB, highSeedersTB) if err != nil { @@ -451,66 +580,74 @@ func handleTorrentStats(w http.ResponseWriter, r *http.Request) error { return fmt.Errorf("method not allowed: %s", r.Method) } - rows, err := db.Query(` - SELECT - url, - top_level_group_name, - group_name, - display_name, - added_to_torrents_list, - is_metadata, - btih, - magnet_link, - torrent_size, - num_files, - data_size, - aa_currently_seeding, - obsolete, - embargo, - seeders, - leechers, - completed, - stats_scraped_at, - partially_broken - FROM torrents - `) - if err != nil { - return fmt.Errorf("error querying database: %w", err) - } - defer rows.Close() - var torrents []Torrent - for rows.Next() { - var t Torrent - if err := rows.Scan( - &t.URL, - &t.TopLevelGroupName, - &t.GroupName, - &t.DisplayName, - &t.AddedToTorrentsList, - &t.IsMetadata, - &t.BTIH, - &t.MagnetLink, - &t.TorrentSize, - &t.NumFiles, - &t.DataSize, - &t.AACurrentlySeeding, - &t.Obsolete, - &t.Embargo, - &t.Seeders, - &t.Leechers, - &t.Completed, - &t.StatsScrapedAt, - &t.PartiallyBroken, - ); err != nil { - return fmt.Errorf("error scanning row: %w", err) + err := withReadTx(func(tx *sql.Tx) error { + rows, err := tx.Query(` + SELECT + url, + top_level_group_name, + group_name, + display_name, + added_to_torrents_list, + is_metadata, + btih, + magnet_link, + torrent_size, + num_files, + data_size, + aa_currently_seeding, + obsolete, + embargo, + seeders, + leechers, + completed, + stats_scraped_at, + partially_broken + FROM torrents + `) + if err != nil { + return fmt.Errorf("error querying database: %w", err) } - torrents = append(torrents, t) - } + defer rows.Close() - if err := rows.Err(); err != nil { - return fmt.Errorf("error after iterating rows: %w", err) + for rows.Next() { + var t Torrent + if err := rows.Scan( + &t.URL, + &t.TopLevelGroupName, + &t.GroupName, + &t.DisplayName, + &t.AddedToTorrentsList, + &t.IsMetadata, + &t.BTIH, + &t.MagnetLink, + &t.TorrentSize, + &t.NumFiles, + &t.DataSize, + &t.AACurrentlySeeding, + &t.Obsolete, + &t.Embargo, + &t.Seeders, + &t.Leechers, + &t.Completed, + &t.StatsScrapedAt, + &t.PartiallyBroken, + ); err != nil { + return fmt.Errorf("error scanning row: %w", err) + } + torrents = append(torrents, t) + } + + if err := rows.Err(); err != nil { + return fmt.Errorf("error after iterating rows: %w", err) + } + + return nil + }) + + if err != nil { + return fmt.Errorf("error fetching torrent stats: %w", err) } w.Header().Set("Content-Type", "application/json") @@ -575,33 +712,46 @@ type SeederStat struct { func fetchDailySeederStats(btih string) ([]SeederStat, error) { query := ` - SELECT - DATE(stats_scraped_at) as date, - AVG(seeders) as avg_seeders - FROM - torrent_updates - WHERE - btih = ? - GROUP BY - DATE(stats_scraped_at) - ORDER BY - date ASC - ` - - rows, err := db.Query(query, btih) - if err != nil { - return nil, err - } - defer rows.Close() + SELECT + DATE(stats_scraped_at) as date, + AVG(seeders) as avg_seeders + FROM + torrent_updates + WHERE + btih = ? + GROUP BY + DATE(stats_scraped_at) + ORDER BY + date ASC + ` var stats []SeederStat - for rows.Next() { - var stat SeederStat - err := rows.Scan(&stat.Date, &stat.Seeders) + + err := withReadTx(func(tx *sql.Tx) error { + rows, err := tx.Query(query, btih) if err != nil { - return nil, err + return fmt.Errorf("error querying database: %w", err) } - stats = append(stats, stat) + defer rows.Close() + + for rows.Next() { + var stat SeederStat + err := rows.Scan(&stat.Date, &stat.Seeders) + if err != nil { + return fmt.Errorf("error scanning row: %w", err) + } + stats = append(stats, stat) + } + + if err := rows.Err(); err != nil { + return fmt.Errorf("error after iterating rows: %w", err) + } + + return nil + }) + + if err != nil { + return nil, fmt.Errorf("error fetching daily seeder stats: %w", err) } return stats, nil @@ -650,44 +800,53 @@ func handleGenerateTorrentList(w http.ResponseWriter, r *http.Request) error { } func fetchTorrents(query string, args ...interface{}) ([]Torrent, error) { - rows, err := db.Query("SELECT * FROM torrents "+query, args...) - if err != nil { - return nil, fmt.Errorf("error querying torrents: %w", err) - } - defer rows.Close() - var torrents []Torrent - for rows.Next() { - var t Torrent - var addedDateStr, statsScrapedAtStr sql.NullString - err := rows.Scan( - &t.BTIH, &t.URL, &t.TopLevelGroupName, &t.GroupName, &t.DisplayName, - &addedDateStr, &t.IsMetadata, &t.MagnetLink, &t.TorrentSize, &t.NumFiles, - &t.DataSize, &t.AACurrentlySeeding, &t.Obsolete, &t.Embargo, &t.Seeders, &t.Leechers, - &t.Completed, &statsScrapedAtStr, &t.PartiallyBroken) + + err := withReadTx(func(tx *sql.Tx) error { + rows, err := tx.Query("SELECT * FROM torrents "+query, args...) if err != nil { - return nil, fmt.Errorf("error scanning torrent: %w", err) + return fmt.Errorf("error querying torrents: %w", err) + } + defer rows.Close() + + for rows.Next() { + var t Torrent + var addedDateStr, statsScrapedAtStr sql.NullString + err := rows.Scan( + &t.BTIH, &t.URL, &t.TopLevelGroupName, &t.GroupName, &t.DisplayName, + &addedDateStr, &t.IsMetadata, &t.MagnetLink, &t.TorrentSize, &t.NumFiles, + &t.DataSize, &t.AACurrentlySeeding, &t.Obsolete, &t.Embargo, &t.Seeders, &t.Leechers, + &t.Completed, &statsScrapedAtStr, &t.PartiallyBroken) + if err != nil { + return fmt.Errorf("error scanning torrent: %w", err) + } + + // Handle NULL values + if addedDateStr.Valid { + t.AddedToTorrentsList = addedDateStr.String + } else { + t.AddedToTorrentsList = "" + } + + if statsScrapedAtStr.Valid { + t.StatsScrapedAt = statsScrapedAtStr.String + } else { + t.StatsScrapedAt = "" + } + + formatTorrent(&t) + torrents = append(torrents, t) } - // Handle NULL values - if addedDateStr.Valid { - t.AddedToTorrentsList = addedDateStr.String - } else { - t.AddedToTorrentsList = "" + if err := rows.Err(); err != nil { + return fmt.Errorf("error iterating rows: %w", err) } - if statsScrapedAtStr.Valid { - t.StatsScrapedAt = statsScrapedAtStr.String - } else { - t.StatsScrapedAt = "" - } + return nil + }) - formatTorrent(&t) - torrents = append(torrents, t) - } - - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error iterating rows: %w", err) + if err != nil { + return nil, fmt.Errorf("error fetching torrents: %w", err) } return torrents, nil @@ -742,6 +901,7 @@ func renderTemplate(w http.ResponseWriter, name string, data interface{}) error } return template.JS(b) }, + "torrentsEnabled": func() bool { return torrentsEnabled }, } tmpl := template.Must(template.New(name).Funcs(funcMap).Parse(getTemplateContent(name))) @@ -754,13 +914,14 @@ func getTemplateContent(name string) string { Anna's Archive Mirror + - +
@@ -775,270 +936,288 @@ func getTemplateContent(name string) string { templates := map[string]string{ "root": header + `
-

Anna's Archive Mirror

-
-

This is a mirror of the Anna's Archive torrent page. We strive to keep everything as up-to-date as possible, however, we are not affiliated with Anna's Archive and this list is maintained privately.

-

This torrent list is the “ultimate unified list” of releases by Anna’s Archive, Library Genesis, Sci-Hub, and others. By seeding these torrents, you help preserve humanity’s knowledge and culture. These torrents represent the vast majority of human knowledge that can be mirrored in bulk.

-

These torrents are not meant for downloading individual books. They are meant for long-term preservation. With these torrents you can set up a full mirror of Anna’s Archive, using their source code and metadata (which can be generated or downloaded as ElasticSearch and MariaDB databases). We scrape the full torrent list from Anna's Archive every 24 hours. We also have full lists of torrents, as JSON, available here.

- - -
-
-
- - -
-
- - -
-
- - -
-
-
-
- -
-
-
- - {{range .}} -

{{.TopLevelGroupName}}

-

{{.GroupName}}

- - - - - - - - - - - - {{range .Torrents}} - - - - - - - - - - - {{end}} -
Torrent NameDate AddedData SizeNum FilesTypeStatusMagnet LinkTorrent Link
{{.DisplayName}}{{.FormattedAddedDate}}{{.FormattedDataSize}}{{.NumFiles}}{{.MetadataLabel}}{{.StatusLabel}}MagnetTorrent
- {{if gt .TotalCount 20}} - View full list - {{end}} - {{end}} -` + footer, +

Anna's Archive Mirror

+ +

This is a mirror of the Anna's Archive torrent page. We strive to keep everything as up-to-date as possible, however, we are not affiliated with Anna's Archive and this list is maintained privately.

+

This torrent list is the “ultimate unified list” of releases by Anna’s Archive, Library Genesis, Sci-Hub, and others. By seeding these torrents, you help preserve humanity’s knowledge and culture. These torrents represent the vast majority of human knowledge that can be mirrored in bulk.

+

These torrents are not meant for downloading individual books. They are meant for long-term preservation. With these torrents you can set up a full mirror of Anna’s Archive, using their source code and metadata (which can be generated or downloaded as ElasticSearch and MariaDB databases). We scrape the full torrent list from Anna's Archive every 24 hours. We also have full lists of torrents, as JSON, available here.

+ + +
+
+
+ + +
+
+ + +
+
+ + +
+
+
+
+ +
+
+
+ {{range .}} +

{{.TopLevelGroupName}}

+

{{.GroupName}}

+ + + + + + + + + + + + {{range .Torrents}} + + + + + + + + + + + {{end}} +
Torrent NameDate AddedData SizeNum FilesTypeStatusMagnet LinkTorrent Link
{{.DisplayName}}{{.FormattedAddedDate}}{{.FormattedDataSize}}{{.NumFiles}}{{.MetadataLabel}}{{.StatusLabel}}Magnet + {{if torrentsEnabled}} + Torrent + {{else}} + Torrent + {{end}} +
+ {{if gt .TotalCount 20}} + View full list + {{end}} + {{end}} + ` + footer, "fullList": header + ` -

{{.GroupName}} - Full List

- - - - - - - - - - - - {{range .Torrents}} - - - - - - - - - - - {{end}} -
Torrent NameDate AddedData SizeNum FilesTypeStatusMagnet LinkTorrent Link
{{.DisplayName}}{{.FormattedAddedDate}}{{.FormattedDataSize}}{{.NumFiles}}{{.MetadataLabel}}{{.StatusLabel}}MagnetTorrent
- Back to main page -` + footer, +

{{.GroupName}} - Full List

+ + + + + + + + + + + + {{range .Torrents}} + + + + + + + + + + + {{end}} +
Torrent NameDate AddedData SizeNum FilesTypeStatusMagnet LinkTorrent Link
{{.DisplayName}}{{.FormattedAddedDate}}{{.FormattedDataSize}}{{.NumFiles}}{{.MetadataLabel}}{{.StatusLabel}}Magnet + {{if $.Torrents}} + Torrent + {{else}} + Torrent + {{end}} +
+ Back to main page + ` + footer, "stats": header + ` -

Torrent Stats: {{.DisplayName}}

- - - - - - - - - - - - -
PropertyValue
Added To Torrents List{{.AddedToTorrentsList}}
Seeders{{.FormattedSeeders}}
Leechers{{.FormattedLeechers}}
Completed{{.Completed}}
Size{{.FormattedDataSize}}
Magnet LinkMagnet
Last Update{{.StatsScrapedAt}}
+

Torrent Stats: {{.DisplayName}}

+ + + + + + + + + + + + + +
PropertyValue
Added To Torrents List{{.AddedToTorrentsList}}
Seeders{{.FormattedSeeders}}
Leechers{{.FormattedLeechers}}
Completed{{.Completed}}
Size{{.FormattedDataSize}}
Magnet LinkMagnet
Torrent Link + {{if $.Torrents}} + Torrent + {{else}} + Torrent + {{end}} +
Last Update{{.StatsScrapedAt}}

Seeder History

- + - - Back to main page -` + footer, +new Chart(ctx, { + type: 'line', + data: { + labels: seederData.map(d => d.date), + datasets: [{ + label: 'Average Daily Seeders', + data: seederData.map(d => d.seeders), + borderColor: 'rgb(75, 192, 192)', + tension: 0.1 + }] + }, + options: { + responsive: true, + scales: { + x: { + type: 'category', + title: { + display: true, + text: 'Date' + } + }, + y: { + beginAtZero: true, + suggestedMin: 0, + suggestedMax: suggestedMax, // Set the suggested maximum value + ticks: { + stepSize: 1, // Ensure whole numbers + callback: function(value) { + return Number.isInteger(value) ? value : null; + } + }, + title: { + display: true, + text: 'Average Seeders' + } + } + }, + plugins: { + title: { + display: true, + text: 'Daily Average Seeder Count' + } + } + } +}); + + +Back to main page + ` + footer, } return templates[name] @@ -1130,47 +1309,59 @@ func generateTorrentList(maxTB float64, listType string) ([]map[string]interface var totalBytes int64 var result []map[string]interface{} - rows, err := db.Query(` - SELECT btih, magnet_link, url, data_size, seeders - FROM torrents - WHERE obsolete = 0 AND embargo = 0 AND seeders > 0 - ORDER BY seeders ASC, data_size DESC - `) - if err != nil { - return nil, err - } - defer rows.Close() - - for rows.Next() { - var btih, magnetLink, url string - var dataSize, seeders int64 - err := rows.Scan(&btih, &magnetLink, &url, &dataSize, &seeders) + err := withReadTx(func(tx *sql.Tx) error { + rows, err := tx.Query(` + SELECT btih, magnet_link, url, data_size, seeders + FROM torrents + WHERE obsolete = 0 AND embargo = 0 AND seeders > 0 + ORDER BY seeders ASC, data_size DESC + `) if err != nil { - return nil, err + return fmt.Errorf("error querying database: %w", err) + } + defer rows.Close() + + for rows.Next() { + var btih, magnetLink, url string + var dataSize, seeders int64 + err := rows.Scan(&btih, &magnetLink, &url, &dataSize, &seeders) + if err != nil { + return fmt.Errorf("error scanning row: %w", err) + } + + if totalBytes+dataSize > maxBytes { + break + } + + torrent := map[string]interface{}{ + "btih": btih, + "data_size": dataSize, + "seeders": seeders, + } + + switch listType { + case "magnet": + torrent["magnet_link"] = magnetLink + case "torrent": + torrent["torrent_url"] = url + case "both": + torrent["magnet_link"] = magnetLink + torrent["torrent_url"] = url + } + + result = append(result, torrent) + totalBytes += dataSize } - if totalBytes+dataSize > maxBytes { - break + if err := rows.Err(); err != nil { + return fmt.Errorf("error after iterating rows: %w", err) } - torrent := map[string]interface{}{ - "btih": btih, - "data_size": dataSize, - "seeders": seeders, - } + return nil + }) - switch listType { - case "magnet": - torrent["magnet_link"] = magnetLink - case "torrent": - torrent["torrent_url"] = url - case "both": - torrent["magnet_link"] = magnetLink - torrent["torrent_url"] = url - } - - result = append(result, torrent) - totalBytes += dataSize + if err != nil { + return nil, fmt.Errorf("error generating torrent list: %w", err) } return result, nil diff --git a/torrents.db b/torrents.db index 250e107..5baa69b 100644 Binary files a/torrents.db and b/torrents.db differ