Created
February 16, 2024 14:26
-
-
Save MateiBuzdea/2003c9f5221720f622bb4d0664d5bb4a to your computer and use it in GitHub Desktop.
Small script to download files from a list
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| # Function to extract the filename and folder path from a URL | |
| # Usage: extract_filename_and_path <url> | |
| extract_filename_and_path() { | |
| local url="$1" | |
| # Extract the filename from the URL | |
| local filename=$(basename "$url") | |
| # Extract the folder path from the URL (excluding the protocol and domain) | |
| local path=$(echo "$url" | sed -E 's|^https?://[^/]+||' | sed 's|/[^/]*$||') | |
| # Return the filename and folder path | |
| echo "$filename" "$path" | |
| } | |
| # Function to check if a URL ends with a specific extension | |
| # Usage: ends_with_extension <url> <extension> | |
| ends_with_extension() { | |
| local url="$1" | |
| local extension="$2" | |
| # Extract the file extension from the URL | |
| local url_extension=$(echo "$url" | grep -oP '\.[^./]+$') | |
| # Check if the extracted extension matches the specified extension | |
| [ "$url_extension" == ".$extension" ] | |
| } | |
| # Function to download a URL and save its contents locally | |
| download_url() { | |
| local url="$1" | |
| local output_dir="$2" | |
| # Extract the filename and folder path from the URL | |
| local filename path | |
| read -r filename path <<< $(extract_filename_and_path "$url") | |
| # Ensure the output directory exists | |
| mkdir -p "$output_dir$path" | |
| # Download the URL and save the contents locally | |
| curl -sS $(printf "%q" "$url") > "$output_dir$path/$filename" | |
| } | |
| # Check if file argument is provided | |
| if [ $# -lt 2 ] || [ ! -f "$1" ]; then | |
| echo "Usage: $0 <file_with_urls> <target_directory> [extension]" | |
| exit 1 | |
| fi | |
| # Output directory | |
| output_dir=$(realpath "$2") | |
| # Ensure the output directory exists | |
| mkdir -p "$output_dir" | |
| # Default extension | |
| extension="js" | |
| if [ -n "$3" ]; then | |
| extension="$3" | |
| fi | |
| echo "[i] Downloading URLs from $1 with extension .$extension to $output_dir..." | |
| # Read each URL from the file and download its contents | |
| while IFS= read -r url; do | |
| url="${url%$'\r'}" | |
| url="${url%$'\n'}" | |
| # Check if the URL ends with the specified extension | |
| if ends_with_extension "$url" "$extension"; then | |
| echo "Downloading $url"; | |
| download_url "$url" "$output_dir"; | |
| else | |
| echo "Skipping $url (does not end with .$extension)"; | |
| fi | |
| done < "$1" | |
| echo "[i] Download complete. Contents saved in $output_dir." |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment