Rclone md5 per file

From UVOO Tech Wiki
Jump to navigation Jump to search

MD5 for blobs

Adds MD5 hashsum per file on blob that doesn't support the backend command

if backend

rclone backend set-md5 --all "azdst:test/foo"

Example of

test.txt
test.txt.MD5
d8e8fca2dc0f896fd7cb4cb0031ba249  test.txt

If not

set -euox pipefail

REMOTE="$1"
MODE="${2:---mode=dir}" # Default to 'file' mode if not provided. Accepts --mode=file or --mode=dir

HASHSUM="MD5"

if [[ "$MODE" != "--mode=file" && "$MODE" != "--mode=dir" ]]; then
  echo "Error: Invalid mode. Please use --mode=file or --mode=dir" >&2
  exit 1
fi

TMP_DIR=$(mktemp -d)
# Ensure the tmp dir is removed when the script exits
trap 'rm -rf -- "$TMP_DIR"' EXIT

echo "Starting hash generation with mode: ${MODE#--mode=}"

rclone hashsum "$HASHSUM" "$REMOTE" | while read -r checksum file; do

  if [[ "$MODE" == "--mode=file" ]]; then
    mkdir -p "$TMP_DIR/$(dirname "$file")"
    echo "$checksum  ${file##*/}" > "$TMP_DIR/$file.$HASHSUM"

  elif [[ "$MODE" == "--mode=dir" ]]; then
    DIR_PATH=$(dirname "$file")
    if [[ "$DIR_PATH" == "." ]]; then
      DIR_PATH="" # Store at the top level of the temp dir
    fi
    mkdir -p "$TMP_DIR/$DIR_PATH"
    echo "$checksum  ${file##*/}" >> "$TMP_DIR/$DIR_PATH/MD5SUMS"
  fi

done

echo "Uploading all .$HASHSUM files to $REMOTE..."

rclone copy "$TMP_DIR" "$REMOTE" --include="**MD5SUMS"

echo "Checksum files have been added successfully."