wget-mirror

#!/bin/bash

# SPDX-FileCopyrightText: 2022 - 2024 sudorook <daemon@nullcodon.com>
#
# SPDX-License-Identifier: GPL-3.0-or-later
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <https://www.gnu.org/licenses/>.

set -eu

ROOT="$(dirname "${0}")"

source "${ROOT}"/globals

! check_command wget && exit 3

#
# Globals
#

AGENTSTRING='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
URL="${1}"
DOMAIN="${URL//http*\:\/\//}"
DOMAIN="${DOMAIN#www\.}"
DOMAIN="${DOMAIN/\/*/}"
NAME="${DOMAIN%.*}"

#
# Main
#

show_header "Crawling ${URL}..."
mkdir -p "${DOMAIN}"
pushd "${DOMAIN}" > /dev/null
wget \
  --mirror \
  --warc-file="${NAME}" \
  --warc-cdx \
  --page-requisites \
  --html-extension \
  --convert-links \
  --execute robots=off \
  --directory-prefix=. \
  --span-hosts \
  --domains="${DOMAIN}",www."${DOMAIN}",cdn."${DOMAIN}" \
  --user-agent="${AGENTSTRING}" \
  --wait=10 \
  --random-wait \
  "${URL}"
popd > /dev/null