Created
November 5, 2024 11:17
-
-
Save phaser/ad6693c6f7ae40264341419be434a4f6 to your computer and use it in GitHub Desktop.
Parallel copy using dd
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| # Parse command line arguments | |
| while [[ $# -gt 0 ]]; do | |
| case $1 in | |
| --file) | |
| source_file="$2" | |
| shift 2 | |
| ;; | |
| --dest) | |
| destination="$2" | |
| shift 2 | |
| ;; | |
| --parts) | |
| num_parts="$2" | |
| shift 2 | |
| ;; | |
| *) | |
| echo "Unknown parameter: $1" | |
| exit 1 | |
| ;; | |
| esac | |
| done | |
| # Validate required parameters | |
| if [[ -z "$source_file" || -z "$destination" || -z "$num_parts" ]]; then | |
| echo "Usage: $0 --file <source_file> --dest <destination> --parts <number_of_parts>" | |
| exit 1 | |
| fi | |
| # Check if source file exists | |
| if [[ ! -f "$source_file" ]]; then | |
| echo "Source file does not exist: $source_file" | |
| exit 1 | |
| fi | |
| # Create destination directory if it doesn't exist | |
| mkdir -p "$(dirname "$destination")" | |
| # Get file size and calculate chunk size | |
| file_size=$(stat -f%z "$source_file") | |
| chunk_size=$(( (file_size + num_parts - 1) / num_parts )) | |
| # Copy chunks in parallel directly to destination | |
| for ((i=0; i<num_parts; i++)); do | |
| offset=$((i * chunk_size)) | |
| if [ $i -eq $((num_parts-1)) ]; then | |
| # Last chunk - copy remaining bytes | |
| dd if="$source_file" of="$destination" bs=1024k skip=$((offset/1024/1024)) seek=$((offset/1024/1024)) conv=notrunc & | |
| else | |
| # Copy the next chunk | |
| dd if="$source_file" of="$destination" bs=1024k skip=$((offset/1024/1024)) seek=$((offset/1024/1024)) count=$((chunk_size/1024/1024)) conv=notrunc & | |
| fi | |
| done | |
| # Wait for all background processes to finish | |
| wait |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment