diff --git a/README.MD b/README.MD index cb0869a..a38b1d0 100644 --- a/README.MD +++ b/README.MD @@ -1,45 +1,47 @@ # Bash Music sync -(c) 2018-2019 Jeroen De Meerleer +(c) 2018-2020 Jeroen De Meerleer Script to sync music from one folder to another ## Usage: ``` -music-sync -s|--source -d|--dest -music-sync + Syncronises music from one folder to another. -Syncronises music from one folder to another. + Usage: + music-sync -s|--source -d|--dest + music-sync -Options: - -s, --source The source folder of the music - -d, --dest The destination folder of the music - -t, --temp The temporary cache for converted files (default: /tmp/converted) - -c, --convert Convert files to a given bitrate in kbps before syncing (default: 192) - -a, --resize-art Resize album-art before syncing (default width: 200) - -v, --verbose <0-6> Set log level (default: 2) - -h, --help Display this help text + Options: + -s, --source The source folder of the music + -d, --dest The destination folder of the music + -t, --temp The temporary cache for converted files (default: /tmp/converted) + -c, --convert Convert files to a given bitrate in kbps before syncing (default: 192) + -a, --resize-art Resize album-art before syncing (default width: 200) + -j, --jobs Number of processes to use in multi-threading (default: nproc - 2) + -v, --verbose <0-6> Set log level (default: 2) + -h, --help Display this help text -Log levels: - 0 | Verbose - 1 | Debug - 2 | Info - 3 | Warning - 4 | Error - 5 | Fatal - 6 | Silence + Log levels: + 0 | Verbose + 1 | Debug + 2 | Info + 3 | Warning + 4 | Error + 5 | Fatal + 6 | Silence -Exit Codes: - 1 Dependencies not met - 2 Invalid Argument - 3 Source Unreachable - 4 Destination Unreachable - 5 Command failed + Exit Codes: + 1 Dependencies not met + 2 Invalid Argument + 3 Source Unreachable + 4 Destination Unreachable + 5 Command failed ``` ## Licence -Copyright 2018-2019 Jeroen De Meerleer +Copyright 2018-2020 Jeroen De Meerleer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/music-sync.sh b/music-sync.sh index 0b71f64..8b3cc36 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -1,4 +1,38 @@ #!/bin/bash +# +#/ +#/ Usage: +#/ music-sync -s|--source -d|--dest +#/ music-sync +#/ +#/ Syncronises music from one folder to another. +#/ +#/ Options: +#/ -s, --source The source folder of the music +#/ -d, --dest The destination folder of the music +#/ -t, --temp The temporary cache for converted files (default: /tmp/converted) +#/ -c, --convert Convert files to a given bitrate in kbps before syncing (default: 192) +#/ -a, --resize-art Resize album-art before syncing (default width: 200) +#/ -j, --jobs Number of processes to use in multi-threading (default: nproc - 2) +#/ -v, --verbose <0-6> Set log level (default: 2) +#/ -h, --help Display this help text +#/ +#/ Log levels: +#/ 0 | Verbose +#/ 1 | Debug +#/ 2 | Info +#/ 3 | Warning +#/ 4 | Error +#/ 5 | Fatal +#/ 6 | Silence +#/ +#/ Exit Codes: +#/ 1 Dependencies not met +#/ 2 Invalid Argument +#/ 3 Source Unreachable +#/ 4 Destination Unreachable +#/ 5 Command failed +#/ begin=$(date +"%s") # saner programming env: these switches turn some bugs into errors @@ -12,9 +46,13 @@ help=false temp="/tmp/converted" convertart=false coverartsize=200 +jobcount=$(expr $(nproc) - 2) +multithread=false +script_name=$(basename "${0}") +script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) CheckDeps() { - if [[ $1 == 0 ]]; then + if [[ $1 == 2 ]]; then # Check getopt ! getopt --test > /dev/null if [[ ${PIPESTATUS[0]} -ne 4 ]]; then @@ -24,44 +62,70 @@ CheckDeps() { ExecTime exit 1 fi + VerboseOutput 0 "\`getopt --test\` succeeded" fi if [[ $1 == 1 ]]; then - # Check lame - if [[ $convert == true && ! $(lame --version 2>/dev/null) ]]; then - VerboseOutput 5 "\`lame --version\` failed" - VerboseOutput 5 "Sorry, It seems that lame is not installed on your system" - VerboseOutput 5 "Please install lame from your repositories and make sure it is available in your \$PATH" - VerboseOutput 5 "Otherwise disable conversion" - ExecTime - exit 1 + # Check ffmpeg + if [[ $convert == true ]]; then + if [[ ! $(ffmpeg -h 2>/dev/null) ]]; then + VerboseOutput 5 "\`ffmpeg -h\` failed" + VerboseOutput 5 "Sorry, It seems that ffmpeg is not installed on your system" + VerboseOutput 5 "Please install ffmpeg from your repositories and make sure it is available in your \$PATH" + VerboseOutput 5 "Otherwise disable conversion" + ExecTime + exit 1 + fi + VerboseOutput 0 "\`ffmpeg -h\` succeeded" fi + # Check EyeD3 - if [[ $convertart == true && ! $(eyeD3 --version 2>/dev/null) ]]; then - VerboseOutput 5 "\`eyeD3 --version\` failed" - VerboseOutput 5 "Sorry, It seems that eyeD3 is not installed on your system" - VerboseOutput 5 "Please install eyeD3 from your repositories and make sure it is available in your \$PATH" - VerboseOutput 5 "Otherwise disable albumart conversion" - ExecTime - exit 1 + if [[ $convertart == true ]]; then + if [[ ! $(eyeD3 --version 2>/dev/null) ]]; then + VerboseOutput 5 "\`eyeD3 --version\` failed" + VerboseOutput 5 "Sorry, It seems that eyeD3 is not installed on your system" + VerboseOutput 5 "Please install eyeD3 from your repositories and make sure it is available in your \$PATH" + VerboseOutput 5 "Otherwise disable albumart conversion" + ExecTime + exit 1 + fi + VerboseOutput 0 "\`eyeD3 --version\` succeeded" fi + # Check ImageMagick - if [[ $convertart == true && ! $(convert --version 2>/dev/null) ]]; then - VerboseOutput 5 "\`convert --version\` failed" - VerboseOutput 5 "Sorry, It seems that ImageMagick is not installed on your system" - VerboseOutput 5 "Please install ImageMagick from your repositories and make sure it is available in your \$PATH" - VerboseOutput 5 "Otherwise disable albumart conversion" - ExecTime - exit 1 + if [[ $convertart == true ]]; then + if [[ ! $(convert --version 2>/dev/null) ]]; then + VerboseOutput 5 "\`convert --version\` failed" + VerboseOutput 5 "Sorry, It seems that ImageMagick is not installed on your system" + VerboseOutput 5 "Please install ImageMagick from your repositories and make sure it is available in your \$PATH" + VerboseOutput 5 "Otherwise disable albumart conversion" + ExecTime + exit 1 + fi + VerboseOutput 0 "\`convert --version\` succeeded" + fi + + # Check Gnu parallel + if [[ $multithread == true ]]; then + if [[ ! $(parallel -h 2>/dev/null) ]]; then + VerboseOutput 5 "\`parallel -h\` failed" + VerboseOutput 5 "Sorry, It seems that parallel is not installed on your system" + VerboseOutput 5 "Please install gnu-parallel and env_parallel from your repositories and make sure it is available in your \$PATH" + VerboseOutput 5 "Otherwise disable multithreading" + ExecTime + exit 1 + fi + VerboseOutput 0 "\`parallel -h \` succeeded" fi fi + VerboseOutput 1 "Dependency test OK" } GetOptions() { # https://stackoverflow.com/a/29754866 - OPTIONS=s:d:t:c::a::v::h - LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help + OPTIONS=s:d:t:c::a::v::hj:: + LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help,jobs:: # -use ! and PIPESTATUS to get exit code with errexit set # -temporarily store output to be able to check for errors @@ -85,43 +149,60 @@ GetOptions() { verbose=2 if [[ $2 != "" ]]; then verbose=${2} - shift fi - shift + shift 2 ;; -h|--help) help=true shift ;; -s|--source) + VerboseOutput 0 "--source given." + VerboseOutput 1 "Source is ${2}" source="$2" shift 2 ;; -d|--dest) + VerboseOutput 0 "--dest given." + VerboseOutput 1 "Destination is ${2}" dest="$2" shift 2 ;; -t|--temp) + VerboseOutput 0 "--temp given" + VerboseOutput 1 "Tempfolder is ${2}" temp="$2" shift 2 ;; -c|--convert) + VerboseOutput 0 "--convert given" convert=true bitrate=192 if [[ $2 != "" ]]; then bitrate=${2} - shift fi - shift + VerboseOutput 1 "Converted bitrate is ${bitrate}" + shift 2 ;; -a|--resize-art) + VerboseOutput 0 "--resize-art given" convertart=true coverartsize=200 if [[ $2 != "" ]]; then coverartsize=${2} - shift fi - shift + shift 2 + VerboseOutput 1 "Album art will ${coverartsize}px wide" + ;; + -j|--jobs) + VerboseOutput 0 "--jobs given" + multithread=true + jobcount=$(expr $(nproc) - 2) + if [[ $2 != "" ]]; then + jobcount=${2} + fi + shift 2 + VerboseOutput 1 "Multithreading will use ${jobcount} threads" ;; --) shift @@ -135,51 +216,29 @@ GetOptions() { done if [[ ! -z "${1+x}" ]]; then + if [[ ${source} != "-" ]]; then + VerboseOutput 4 "Source provided twice. Continueing with ${1}" + fi source="$1" + VerboseOutput 1 "Source is ${1}" fi if [[ ! -z "${2+x}" ]]; then + if [[ ${dest} != "-" ]]; then + VerboseOutput 4 "Destination provided twice. Continueing with ${2}" + fi dest="$2" + VerboseOutput 1 "Destination is ${2}" fi if [[ $dest == "-" ]] || [[ $source == "-" ]]; then help=true fi + VerboseOutput 1 "Checks OK. Going on" } Usage() { - echo "" - echo "Usage:" - echo "music-sync -s|--source -d|--dest " - echo "music-sync " - echo "" - echo "Syncronises music from one folder to another." - echo "" - echo "Options:" - echo " -s, --source The source folder of the music" - echo " -d, --dest The destination folder of the music" - echo " -t, --temp The temporary cache for converted files (default: /tmp/converted)" - echo " -c, --convert Convert files to a given bitrate in kbps before syncing (default: 192)" - echo " -a, --resize-art Resize album-art before syncing (default width: 200)" - echo " -v, --verbose <0-6> Set log level (default: 2)" - echo " -h, --help Display this help text" - echo "" - echo "Log levels:" - echo " 0 | Verbose" - echo " 1 | Debug" - echo " 2 | Info" - echo " 3 | Warning" - echo " 4 | Error" - echo " 5 | Fatal" - echo " 6 | Silence" - echo "" - echo "Exit Codes:" - echo " 1 Dependencies not met" - echo " 2 Invalid Argument" - echo " 3 Source Unreachable" - echo " 4 Destination Unreachable" - echo " 5 Command failed" - echo "" + grep '^#/' "${script_dir}/${script_name}" | sed 's/^#\/\w*//' } VerboseOutput() { @@ -219,99 +278,156 @@ CreateFileList() { sourcepath="${sourcepath/\]/\\\]}" for file in $sourcepath; do origfile="${file#"$1/"}" - relfile=$(echo ${origfile} | sed -e 's/\(\.\)*$//g') + relfile=$(echo ${origfile} | sed -e 's/\(\.\)*$//g' | tr -s ' ') + mp3file=$(echo "${relfile}" | sed -e 's/.flac/.mp3/') + VerboseOutput 0 "Checking ${origfile}" if [[ -d "${1}/$origfile" ]]; then - newdir="${3}/$relfile" + VerboseOutput 0 "${origfile} is folder" + newdir="${3}/$origfile" newdir=${newdir#"/"} VerboseOutput 1 "Entering $newdir" CreateFileList "${1}/$origfile" "${2}/$relfile" "$newdir" - elif [[ "${1}/$origfile" != *".m3u" ]] && [[ ! -f "${2}/$relfile" || "${1}/$origfile" -nt "${2}/$relfile" ]]; then - echo ${3}/$relfile >> /tmp/music-sync-filelist - VerboseOutput 2 "Added: ${3}/${relfile}" + elif [[ "${1}/$origfile" != *".m3u" ]] && [[ "${1}/$origfile" -nt "${2}/$mp3file" ]]; then + echo ${3}/$origfile >> /tmp/music-sync-filelist + VerboseOutput 0 "${origfile} is newer in source" + VerboseOutput 2 "Added: ${3}/${origfile}" fi + if [[ "${1}/$origfile" == *".m3u" ]]; then + VerboseOutput 0 "${origfile} is playlist" + fi + done } -ConvertFiles() { +Execute() { curline=0 percentage=0 - while read -r line - do + IFS=" +"; + if [[ $multithread == true ]]; then + VerboseOutput 3 "Running in parallel. Mind your load" + export -f ProcessFile + export -f ConvertFile + export -f CopyFile + export -f VerboseOutput + export -f ExecTime + export dest + export source + export bitrate + export convert + export temp + export verbose + export coverartsize + export begin + parallel --jobs ${jobcount} --will-cite --line-buffer --arg-file "/tmp/music-sync-filelist" ProcessFile + else + for line in $(cat "/tmp/music-sync-filelist") + do + curline=$(expr ${curline} + 1) + total=$(cat /tmp/music-sync-filelist | wc -l) + percentage=$(echo "scale=4;${curline}/${total}" | bc) + percentage=$(echo "scale=2;${percentage}*100" | bc) + VerboseOutput 2 "Current file: $line" + VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%)" + ProcessFile $line + done + fi + VerboseOutput 2 "Done!" +} +ProcessFile() { + line=${1} + curline="$(grep -n "$line" /tmp/music-sync-filelist | head -n 1 | cut -d: -f1)" + total=$(cat /tmp/music-sync-filelist | wc -l) + percentage=$(echo "scale=4;${curline}/${total}" | bc) + percentage=$(echo "scale=2;${percentage}*100" | bc) + VerboseOutput 2 "Current File: $line" + VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%)" + if [[ $convert == true ]]; then + ConvertFile "$line" + fi + CopyFile "$line" +} - if [[ ! -f "${source}/$line" ]]; then - VerboseOutput 5 "Source-file ${source}/$line Unreachable" - ExecTime - exit 3 +ConvertFile() { + line=${1} + if [[ ! -f "${source}/$line" ]]; then + VerboseOutput 5 "Source-file ${source}/$line Unreachable" + ExecTime + exit 3 + fi + + if [[ "$temp/$line" = */* ]]; then + VerboseOutput 0 "Creating folder $temp/${line%/*}" + mkdir -p "$temp/${line%/*}"; + fi; + mp3line=$(echo "$line" | sed -e 's/.flac/.mp3/') + if [[ ! -f "$temp/$mp3line" || "${source}/$mp3line" -nt "$temp/$mp3line" ]]; then + if [[ $line != *".mp3" ]]; then + VerboseOutput 3 "${line} will be converted to mp3-file ${mp3line}" fi - - curline=$(expr ${curline} + 1) - total=$(cat /tmp/music-sync-filelist | wc -l) - percentage=$(echo "scale=4;${curline}/${total}" | bc) - percentage=$(echo "scale=2;${percentage}*100" | bc) - VerboseOutput 1 "Converting: $line" - VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 1 of 2" - - if [[ "$temp/$line" = */* ]]; then - mkdir -p "$temp/${line%/*}"; - fi; - if [[ ! -f "$temp/$line" || "${source}/$line" -nt "$temp/$line" ]]; then - lame -b ${bitrate} $source/$line $temp/$line 1>/dev/null 2>/dev/null - VerboseOutput 2 "Converted: $line" - if [[ $convertart == true ]]; then - mkdir -p "$temp/$line-images/" - eyeD3 --write-images "$temp/$line-images/" "$temp/$line" 1>/dev/null 2>/dev/null - convert "$temp/$line-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/$line-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null - eyeD3 --remove-all-images "$temp/$line" 1>/dev/null 2>/dev/null - eyeD3 --add-image "$temp/$line-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/$line" 1>/dev/null 2>/dev/null - VerboseOutput 2 "Converted cover art: $line" + VerboseOutput 0 "Converting MP3-file $line" + ffmpeg -y -i "$source/$line" -acodec libmp3lame -map_metadata 0 -id3v2_version 3 -b:a ${bitrate}k "$temp/${mp3line}" 1>/dev/null 2>/dev/null + if [[ $convertart == true ]]; then + VerboseOutput 0 "Creating folder $temp/${mp3line}-images/" + mkdir -p "$temp/${mp3line}-images/" + VerboseOutput 0 "Extracted albumart" + eyeD3 --write-images "$temp/${mp3line}-images/" "$temp/${mp3line}" 1>/dev/null 2>/dev/null + frontcovers=(${temp}/${mp3line}-images/FRONT_COVER.*) + if [ -e "$frontcovers" ]; then + VerboseOutput 0 "Converting albumart" + convert "$temp/${mp3line}-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/${mp3line}-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null + eyeD3 --remove-all-images "$temp/${mp3line}" 1>/dev/null 2>/dev/null + VerboseOutput 0 "Embedding albumart" + eyeD3 --add-image "$temp/${mp3line}-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/${mp3line}" 1>/dev/null 2>/dev/null + VerboseOutput 1 "Converted cover art: ${mp3line}" + else + VerboseOutput 4 "No front cover art found for ${mp3line}" fi - - else - VerboseOutput 3 "$line already converted" - fi; - - done < "/tmp/music-sync-filelist" + fi + VerboseOutput 1 "Converted: $line" + else + VerboseOutput 3 "$line already converted" + fi; } -CopyFiles() { - curline=0 - percentage=0 - while read -r line - do - if [[ ! -d "$dest" ]]; then - VerboseOutput 5 "Destination unreachable" - ExecTime - exit 4 - fi +CopyFile() { + line=${1} + if [[ ! -d "$dest" ]]; then + VerboseOutput 5 "Destination unreachable" + ExecTime + exit 4 + fi - if [[ ! -f "${temp}/$line" ]]; then - VerboseOutput 5 "Source-file ${temp}/$line Unreachable" - ExecTime - exit 3 - fi + if [[ $convert == true ]]; then + mp3line=$(echo "$line" | sed -e 's/.flac/.mp3/') + else + mp3line=$(echo "$line") + fi - curline=$(expr ${curline} + 1) - total=$(cat /tmp/music-sync-filelist | wc -l) - percentage=$(echo "scale=4;${curline}/${total}" | bc) - percentage=$(echo "scale=2;${percentage}*100" | bc) + if [[ ! -f "${temp}/$mp3line" ]]; then + VerboseOutput 5 "Source-file ${temp}/$mp3line Unreachable" + ExecTime + exit 3 + fi - VerboseOutput 1 "Copying: $line" - VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 2 of 2" - - if [[ "$dest/$line" = */* ]]; then - mkdir -p "$dest/${line%/*}"; - fi; + destline=$(echo $mp3line | sed -e 's/\.*\//\//g'| tr -s ' ') + VerboseOutput 1 "Copying: $line" + + if [[ "$dest/$line" = */* ]]; then + VerboseOutput 0 "Creating folder $dest/${line%/*}" + mkdir -p "$dest/${destline%/*}"; + fi; - cp -f $temp/$line $dest/$line 1>/dev/null 2>/dev/null + cp -fv "$temp/$mp3line" "$dest/${destline}" 1>/dev/null 2>/dev/null - VerboseOutput 2 "Copied: $line" - - done < "/tmp/music-sync-filelist" + VerboseOutput 1 "Copied: $line" } + CleanUp() { VerboseOutput 1 "Cleaning Up" if [[ -f /tmp/music-sync-filelist ]]; then + VerboseOutput 1 "Removing filelist" rm "/tmp/music-sync-filelist" fi VerboseOutput 1 "Done" @@ -338,7 +454,7 @@ ExitHandler() { trap 'ExitHandler' SIGINT trap 'ErrorHandler $BASH_COMMAND' ERR -CheckDeps 0 +CheckDeps 2 GetOptions $@ CheckDeps 1 if [[ $help == true ]]; then @@ -348,20 +464,19 @@ fi if [[ -f /tmp/music-sync-filelist ]]; then rm /tmp/music-sync-filelist fi +VerboseOutput 2 "Scanning for new or updated files" CreateFileList $source $dest "" if [[ ! -f /tmp/music-sync-filelist ]]; then VerboseOutput 2 "Nothing to do!" CleanUp exit 0 fi -if [[ $convert == true ]]; then - ConvertFiles -else +if [[ $convert == false ]]; then if [[ $temp != "/tmp/converted" ]]; then VerboseOutput 2 "Conversion not enabled. Ignoring cache folder" fi temp=$source -fi -CopyFiles +fi +Execute CleanUp ExecTime