Difference between revisions of "Rclone md5 per file"

From UVOO Tech Wiki
Jump to navigation Jump to search
 
Line 17: Line 17:
  
 
```
 
```
−
#!/bin/bash
+
set -euox pipefail
−
set -euxo pipefail
 
  
 
REMOTE="$1"
 
REMOTE="$1"
−
HASHSUM="MD5" # Or another hash like SHA1
+
MODE="${2:---mode=dir}" # Default to 'file' mode if not provided. Accepts --mode=file or --mode=dir
 +
 
 +
HASHSUM="MD5"
 +
 
 +
if [[ "$MODE" != "--mode=file" && "$MODE" != "--mode=dir" ]]; then
 +
  echo "Error: Invalid mode. Please use --mode=file or --mode=dir" >&2
 +
  exit 1
 +
fi
  
−
# Create a temporary directory to stage the checksum files
 
 
TMP_DIR=$(mktemp -d)
 
TMP_DIR=$(mktemp -d)
−
 
+
# Ensure the tmp dir is removed when the script exits
−
# Ensure the temporary directory is removed when the script exits
 
 
trap 'rm -rf -- "$TMP_DIR"' EXIT
 
trap 'rm -rf -- "$TMP_DIR"' EXIT
  
−
echo "Generating $HASHSUM checksums for all files in $REMOTE..."
+
echo "Starting hash generation with mode: ${MODE#--mode=}"
  
−
# 1. Get all checksums in ONE command and create sidecar files locally.
 
−
#    The 'while read' loop only processes text here; it does not run new commands.
 
 
rclone hashsum "$HASHSUM" "$REMOTE" | while read -r checksum file; do
 
rclone hashsum "$HASHSUM" "$REMOTE" | while read -r checksum file; do
−
  # Re-create the directory structure within the temp folder
 
−
  mkdir -p "$TMP_DIR/$(dirname "$file")"
 
  
−
   # Write the checksum file
+
   if [[ "$MODE" == "--mode=file" ]]; then
−
  echo "$checksum  $file" > "$TMP_DIR/$file.$HASHSUM"
+
    mkdir -p "$TMP_DIR/$(dirname "$file")"
 +
    echo "$checksum  ${file##*/}" > "$TMP_DIR/$file.$HASHSUM"
 +
 
 +
  elif [[ "$MODE" == "--mode=dir" ]]; then
 +
    DIR_PATH=$(dirname "$file")
 +
    if [[ "$DIR_PATH" == "." ]]; then
 +
      DIR_PATH="" # Store at the top level of the temp dir
 +
    fi
 +
    mkdir -p "$TMP_DIR/$DIR_PATH"
 +
    echo "$checksum  ${file##*/}" >> "$TMP_DIR/$DIR_PATH/MD5SUMS"
 +
  fi
 +
 
 
done
 
done
  
 
echo "Uploading all .$HASHSUM files to $REMOTE..."
 
echo "Uploading all .$HASHSUM files to $REMOTE..."
  
−
# 2. Upload all the generated checksum files in ONE command.
+
rclone copy "$TMP_DIR" "$REMOTE" --include="**MD5SUMS"
−
rclone copy "$TMP_DIR" "$REMOTE" --include="**.$HASHSUM"
 
  
−
echo "$HASHSUM checksum files have been added to the remote."
+
echo "Checksum files have been added successfully."
 
```
 
```

Latest revision as of 16:25, 3 July 2025

MD5 for blobs

Adds MD5 hashsum per file on blob that doesn't support the backend command

if backend

rclone backend set-md5 --all "azdst:test/foo"

Example of

test.txt
test.txt.MD5
d8e8fca2dc0f896fd7cb4cb0031ba249  test.txt

If not

set -euox pipefail

REMOTE="$1"
MODE="${2:---mode=dir}" # Default to 'file' mode if not provided. Accepts --mode=file or --mode=dir

HASHSUM="MD5"

if [[ "$MODE" != "--mode=file" && "$MODE" != "--mode=dir" ]]; then
  echo "Error: Invalid mode. Please use --mode=file or --mode=dir" >&2
  exit 1
fi

TMP_DIR=$(mktemp -d)
# Ensure the tmp dir is removed when the script exits
trap 'rm -rf -- "$TMP_DIR"' EXIT

echo "Starting hash generation with mode: ${MODE#--mode=}"

rclone hashsum "$HASHSUM" "$REMOTE" | while read -r checksum file; do

  if [[ "$MODE" == "--mode=file" ]]; then
    mkdir -p "$TMP_DIR/$(dirname "$file")"
    echo "$checksum  ${file##*/}" > "$TMP_DIR/$file.$HASHSUM"

  elif [[ "$MODE" == "--mode=dir" ]]; then
    DIR_PATH=$(dirname "$file")
    if [[ "$DIR_PATH" == "." ]]; then
      DIR_PATH="" # Store at the top level of the temp dir
    fi
    mkdir -p "$TMP_DIR/$DIR_PATH"
    echo "$checksum  ${file##*/}" >> "$TMP_DIR/$DIR_PATH/MD5SUMS"
  fi

done

echo "Uploading all .$HASHSUM files to $REMOTE..."

rclone copy "$TMP_DIR" "$REMOTE" --include="**MD5SUMS"

echo "Checksum files have been added successfully."
Retrieved from "https://tech.uvoo.io/index.php?title=Rclone_md5_per_file&oldid=5567"