Merge branch 'develop'

This commit is contained in:
Jeroen De Meerleer 2020-02-12 18:56:13 +01:00
commit e664de89b4
2 changed files with 281 additions and 164 deletions

View File

@ -1,45 +1,47 @@
# Bash Music sync
(c) 2018-2019 Jeroen De Meerleer <me@jeroened.be>
(c) 2018-2020 Jeroen De Meerleer <me@jeroened.be>
Script to sync music from one folder to another
## Usage:
```
music-sync <options> -s|--source <source> -d|--dest <destination>
music-sync <options> <source> <destination>
Syncronises music from one folder to another.
Syncronises music from one folder to another.
Usage:
music-sync <options> -s|--source <source> -d|--dest <destination>
music-sync <options> <source> <destination>
Options:
-s, --source <source> The source folder of the music
-d, --dest <destination> The destination folder of the music
-t, --temp <folder> The temporary cache for converted files (default: /tmp/converted)
-c, --convert <bitrate> Convert files to a given bitrate in kbps before syncing (default: 192)
-a, --resize-art <width> Resize album-art before syncing (default width: 200)
-v, --verbose <0-6> Set log level (default: 2)
-h, --help Display this help text
Options:
-s, --source <source> The source folder of the music
-d, --dest <destination> The destination folder of the music
-t, --temp <folder> The temporary cache for converted files (default: /tmp/converted)
-c, --convert <bitrate> Convert files to a given bitrate in kbps before syncing (default: 192)
-a, --resize-art <width> Resize album-art before syncing (default width: 200)
-j, --jobs <nproc> Number of processes to use in multi-threading (default: nproc - 2)
-v, --verbose <0-6> Set log level (default: 2)
-h, --help Display this help text
Log levels:
0 | Verbose
1 | Debug
2 | Info
3 | Warning
4 | Error
5 | Fatal
6 | Silence
Log levels:
0 | Verbose
1 | Debug
2 | Info
3 | Warning
4 | Error
5 | Fatal
6 | Silence
Exit Codes:
1 Dependencies not met
2 Invalid Argument
3 Source Unreachable
4 Destination Unreachable
5 Command failed
Exit Codes:
1 Dependencies not met
2 Invalid Argument
3 Source Unreachable
4 Destination Unreachable
5 Command failed
```
## Licence
Copyright 2018-2019 Jeroen De Meerleer <me@jeroened.be>
Copyright 2018-2020 Jeroen De Meerleer <me@jeroened.be>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

View File

@ -1,4 +1,38 @@
#!/bin/bash
#
#/
#/ Usage:
#/ music-sync <options> -s|--source <source> -d|--dest <destination>
#/ music-sync <options> <source> <destination>
#/
#/ Syncronises music from one folder to another.
#/
#/ Options:
#/ -s, --source <source> The source folder of the music
#/ -d, --dest <destination> The destination folder of the music
#/ -t, --temp <folder> The temporary cache for converted files (default: /tmp/converted)
#/ -c, --convert <bitrate> Convert files to a given bitrate in kbps before syncing (default: 192)
#/ -a, --resize-art <width> Resize album-art before syncing (default width: 200)
#/ -j, --jobs <nproc> Number of processes to use in multi-threading (default: nproc - 2)
#/ -v, --verbose <0-6> Set log level (default: 2)
#/ -h, --help Display this help text
#/
#/ Log levels:
#/ 0 | Verbose
#/ 1 | Debug
#/ 2 | Info
#/ 3 | Warning
#/ 4 | Error
#/ 5 | Fatal
#/ 6 | Silence
#/
#/ Exit Codes:
#/ 1 Dependencies not met
#/ 2 Invalid Argument
#/ 3 Source Unreachable
#/ 4 Destination Unreachable
#/ 5 Command failed
#/
begin=$(date +"%s")
# saner programming env: these switches turn some bugs into errors
@ -12,9 +46,13 @@ help=false
temp="/tmp/converted"
convertart=false
coverartsize=200
jobcount=$(expr $(nproc) - 2)
multithread=false
script_name=$(basename "${0}")
script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
CheckDeps() {
if [[ $1 == 0 ]]; then
if [[ $1 == 2 ]]; then
# Check getopt
! getopt --test > /dev/null
if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
@ -24,44 +62,70 @@ CheckDeps() {
ExecTime
exit 1
fi
VerboseOutput 0 "\`getopt --test\` succeeded"
fi
if [[ $1 == 1 ]]; then
# Check lame
if [[ $convert == true && ! $(lame --version 2>/dev/null) ]]; then
VerboseOutput 5 "\`lame --version\` failed"
VerboseOutput 5 "Sorry, It seems that lame is not installed on your system"
VerboseOutput 5 "Please install lame from your repositories and make sure it is available in your \$PATH"
VerboseOutput 5 "Otherwise disable conversion"
ExecTime
exit 1
# Check ffmpeg
if [[ $convert == true ]]; then
if [[ ! $(ffmpeg -h 2>/dev/null) ]]; then
VerboseOutput 5 "\`ffmpeg -h\` failed"
VerboseOutput 5 "Sorry, It seems that ffmpeg is not installed on your system"
VerboseOutput 5 "Please install ffmpeg from your repositories and make sure it is available in your \$PATH"
VerboseOutput 5 "Otherwise disable conversion"
ExecTime
exit 1
fi
VerboseOutput 0 "\`ffmpeg -h\` succeeded"
fi
# Check EyeD3
if [[ $convertart == true && ! $(eyeD3 --version 2>/dev/null) ]]; then
VerboseOutput 5 "\`eyeD3 --version\` failed"
VerboseOutput 5 "Sorry, It seems that eyeD3 is not installed on your system"
VerboseOutput 5 "Please install eyeD3 from your repositories and make sure it is available in your \$PATH"
VerboseOutput 5 "Otherwise disable albumart conversion"
ExecTime
exit 1
if [[ $convertart == true ]]; then
if [[ ! $(eyeD3 --version 2>/dev/null) ]]; then
VerboseOutput 5 "\`eyeD3 --version\` failed"
VerboseOutput 5 "Sorry, It seems that eyeD3 is not installed on your system"
VerboseOutput 5 "Please install eyeD3 from your repositories and make sure it is available in your \$PATH"
VerboseOutput 5 "Otherwise disable albumart conversion"
ExecTime
exit 1
fi
VerboseOutput 0 "\`eyeD3 --version\` succeeded"
fi
# Check ImageMagick
if [[ $convertart == true && ! $(convert --version 2>/dev/null) ]]; then
VerboseOutput 5 "\`convert --version\` failed"
VerboseOutput 5 "Sorry, It seems that ImageMagick is not installed on your system"
VerboseOutput 5 "Please install ImageMagick from your repositories and make sure it is available in your \$PATH"
VerboseOutput 5 "Otherwise disable albumart conversion"
ExecTime
exit 1
if [[ $convertart == true ]]; then
if [[ ! $(convert --version 2>/dev/null) ]]; then
VerboseOutput 5 "\`convert --version\` failed"
VerboseOutput 5 "Sorry, It seems that ImageMagick is not installed on your system"
VerboseOutput 5 "Please install ImageMagick from your repositories and make sure it is available in your \$PATH"
VerboseOutput 5 "Otherwise disable albumart conversion"
ExecTime
exit 1
fi
VerboseOutput 0 "\`convert --version\` succeeded"
fi
# Check Gnu parallel
if [[ $multithread == true ]]; then
if [[ ! $(parallel -h 2>/dev/null) ]]; then
VerboseOutput 5 "\`parallel -h\` failed"
VerboseOutput 5 "Sorry, It seems that parallel is not installed on your system"
VerboseOutput 5 "Please install gnu-parallel and env_parallel from your repositories and make sure it is available in your \$PATH"
VerboseOutput 5 "Otherwise disable multithreading"
ExecTime
exit 1
fi
VerboseOutput 0 "\`parallel -h \` succeeded"
fi
fi
VerboseOutput 1 "Dependency test OK"
}
GetOptions() {
# https://stackoverflow.com/a/29754866
OPTIONS=s:d:t:c::a::v::h
LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help
OPTIONS=s:d:t:c::a::v::hj::
LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help,jobs::
# -use ! and PIPESTATUS to get exit code with errexit set
# -temporarily store output to be able to check for errors
@ -85,43 +149,60 @@ GetOptions() {
verbose=2
if [[ $2 != "" ]]; then
verbose=${2}
shift
fi
shift
shift 2
;;
-h|--help)
help=true
shift
;;
-s|--source)
VerboseOutput 0 "--source given."
VerboseOutput 1 "Source is ${2}"
source="$2"
shift 2
;;
-d|--dest)
VerboseOutput 0 "--dest given."
VerboseOutput 1 "Destination is ${2}"
dest="$2"
shift 2
;;
-t|--temp)
VerboseOutput 0 "--temp given"
VerboseOutput 1 "Tempfolder is ${2}"
temp="$2"
shift 2
;;
-c|--convert)
VerboseOutput 0 "--convert given"
convert=true
bitrate=192
if [[ $2 != "" ]]; then
bitrate=${2}
shift
fi
shift
VerboseOutput 1 "Converted bitrate is ${bitrate}"
shift 2
;;
-a|--resize-art)
VerboseOutput 0 "--resize-art given"
convertart=true
coverartsize=200
if [[ $2 != "" ]]; then
coverartsize=${2}
shift
fi
shift
shift 2
VerboseOutput 1 "Album art will ${coverartsize}px wide"
;;
-j|--jobs)
VerboseOutput 0 "--jobs given"
multithread=true
jobcount=$(expr $(nproc) - 2)
if [[ $2 != "" ]]; then
jobcount=${2}
fi
shift 2
VerboseOutput 1 "Multithreading will use ${jobcount} threads"
;;
--)
shift
@ -135,51 +216,29 @@ GetOptions() {
done
if [[ ! -z "${1+x}" ]]; then
if [[ ${source} != "-" ]]; then
VerboseOutput 4 "Source provided twice. Continueing with ${1}"
fi
source="$1"
VerboseOutput 1 "Source is ${1}"
fi
if [[ ! -z "${2+x}" ]]; then
if [[ ${dest} != "-" ]]; then
VerboseOutput 4 "Destination provided twice. Continueing with ${2}"
fi
dest="$2"
VerboseOutput 1 "Destination is ${2}"
fi
if [[ $dest == "-" ]] || [[ $source == "-" ]]; then
help=true
fi
VerboseOutput 1 "Checks OK. Going on"
}
Usage() {
echo ""
echo "Usage:"
echo "music-sync <options> -s|--source <source> -d|--dest <destination>"
echo "music-sync <options> <source> <destination>"
echo ""
echo "Syncronises music from one folder to another."
echo ""
echo "Options:"
echo " -s, --source <source> The source folder of the music"
echo " -d, --dest <destination> The destination folder of the music"
echo " -t, --temp <folder> The temporary cache for converted files (default: /tmp/converted)"
echo " -c, --convert <bitrate> Convert files to a given bitrate in kbps before syncing (default: 192)"
echo " -a, --resize-art <width> Resize album-art before syncing (default width: 200)"
echo " -v, --verbose <0-6> Set log level (default: 2)"
echo " -h, --help Display this help text"
echo ""
echo "Log levels:"
echo " 0 | Verbose"
echo " 1 | Debug"
echo " 2 | Info"
echo " 3 | Warning"
echo " 4 | Error"
echo " 5 | Fatal"
echo " 6 | Silence"
echo ""
echo "Exit Codes:"
echo " 1 Dependencies not met"
echo " 2 Invalid Argument"
echo " 3 Source Unreachable"
echo " 4 Destination Unreachable"
echo " 5 Command failed"
echo ""
grep '^#/' "${script_dir}/${script_name}" | sed 's/^#\/\w*//'
}
VerboseOutput() {
@ -219,99 +278,156 @@ CreateFileList() {
sourcepath="${sourcepath/\]/\\\]}"
for file in $sourcepath; do
origfile="${file#"$1/"}"
relfile=$(echo ${origfile} | sed -e 's/\(\.\)*$//g')
relfile=$(echo ${origfile} | sed -e 's/\(\.\)*$//g' | tr -s ' ')
mp3file=$(echo "${relfile}" | sed -e 's/.flac/.mp3/')
VerboseOutput 0 "Checking ${origfile}"
if [[ -d "${1}/$origfile" ]]; then
newdir="${3}/$relfile"
VerboseOutput 0 "${origfile} is folder"
newdir="${3}/$origfile"
newdir=${newdir#"/"}
VerboseOutput 1 "Entering $newdir"
CreateFileList "${1}/$origfile" "${2}/$relfile" "$newdir"
elif [[ "${1}/$origfile" != *".m3u" ]] && [[ ! -f "${2}/$relfile" || "${1}/$origfile" -nt "${2}/$relfile" ]]; then
echo ${3}/$relfile >> /tmp/music-sync-filelist
VerboseOutput 2 "Added: ${3}/${relfile}"
elif [[ "${1}/$origfile" != *".m3u" ]] && [[ "${1}/$origfile" -nt "${2}/$mp3file" ]]; then
echo ${3}/$origfile >> /tmp/music-sync-filelist
VerboseOutput 0 "${origfile} is newer in source"
VerboseOutput 2 "Added: ${3}/${origfile}"
fi
if [[ "${1}/$origfile" == *".m3u" ]]; then
VerboseOutput 0 "${origfile} is playlist"
fi
done
}
ConvertFiles() {
Execute() {
curline=0
percentage=0
while read -r line
do
IFS="
";
if [[ $multithread == true ]]; then
VerboseOutput 3 "Running in parallel. Mind your load"
export -f ProcessFile
export -f ConvertFile
export -f CopyFile
export -f VerboseOutput
export -f ExecTime
export dest
export source
export bitrate
export convert
export temp
export verbose
export coverartsize
export begin
parallel --jobs ${jobcount} --will-cite --line-buffer --arg-file "/tmp/music-sync-filelist" ProcessFile
else
for line in $(cat "/tmp/music-sync-filelist")
do
curline=$(expr ${curline} + 1)
total=$(cat /tmp/music-sync-filelist | wc -l)
percentage=$(echo "scale=4;${curline}/${total}" | bc)
percentage=$(echo "scale=2;${percentage}*100" | bc)
VerboseOutput 2 "Current file: $line"
VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%)"
ProcessFile $line
done
fi
VerboseOutput 2 "Done!"
}
ProcessFile() {
line=${1}
curline="$(grep -n "$line" /tmp/music-sync-filelist | head -n 1 | cut -d: -f1)"
total=$(cat /tmp/music-sync-filelist | wc -l)
percentage=$(echo "scale=4;${curline}/${total}" | bc)
percentage=$(echo "scale=2;${percentage}*100" | bc)
VerboseOutput 2 "Current File: $line"
VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%)"
if [[ $convert == true ]]; then
ConvertFile "$line"
fi
CopyFile "$line"
}
if [[ ! -f "${source}/$line" ]]; then
VerboseOutput 5 "Source-file ${source}/$line Unreachable"
ExecTime
exit 3
ConvertFile() {
line=${1}
if [[ ! -f "${source}/$line" ]]; then
VerboseOutput 5 "Source-file ${source}/$line Unreachable"
ExecTime
exit 3
fi
if [[ "$temp/$line" = */* ]]; then
VerboseOutput 0 "Creating folder $temp/${line%/*}"
mkdir -p "$temp/${line%/*}";
fi;
mp3line=$(echo "$line" | sed -e 's/.flac/.mp3/')
if [[ ! -f "$temp/$mp3line" || "${source}/$mp3line" -nt "$temp/$mp3line" ]]; then
if [[ $line != *".mp3" ]]; then
VerboseOutput 3 "${line} will be converted to mp3-file ${mp3line}"
fi
curline=$(expr ${curline} + 1)
total=$(cat /tmp/music-sync-filelist | wc -l)
percentage=$(echo "scale=4;${curline}/${total}" | bc)
percentage=$(echo "scale=2;${percentage}*100" | bc)
VerboseOutput 1 "Converting: $line"
VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 1 of 2"
if [[ "$temp/$line" = */* ]]; then
mkdir -p "$temp/${line%/*}";
fi;
if [[ ! -f "$temp/$line" || "${source}/$line" -nt "$temp/$line" ]]; then
lame -b ${bitrate} $source/$line $temp/$line 1>/dev/null 2>/dev/null
VerboseOutput 2 "Converted: $line"
if [[ $convertart == true ]]; then
mkdir -p "$temp/$line-images/"
eyeD3 --write-images "$temp/$line-images/" "$temp/$line" 1>/dev/null 2>/dev/null
convert "$temp/$line-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/$line-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null
eyeD3 --remove-all-images "$temp/$line" 1>/dev/null 2>/dev/null
eyeD3 --add-image "$temp/$line-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/$line" 1>/dev/null 2>/dev/null
VerboseOutput 2 "Converted cover art: $line"
VerboseOutput 0 "Converting MP3-file $line"
ffmpeg -y -i "$source/$line" -acodec libmp3lame -map_metadata 0 -id3v2_version 3 -b:a ${bitrate}k "$temp/${mp3line}" 1>/dev/null 2>/dev/null
if [[ $convertart == true ]]; then
VerboseOutput 0 "Creating folder $temp/${mp3line}-images/"
mkdir -p "$temp/${mp3line}-images/"
VerboseOutput 0 "Extracted albumart"
eyeD3 --write-images "$temp/${mp3line}-images/" "$temp/${mp3line}" 1>/dev/null 2>/dev/null
frontcovers=(${temp}/${mp3line}-images/FRONT_COVER.*)
if [ -e "$frontcovers" ]; then
VerboseOutput 0 "Converting albumart"
convert "$temp/${mp3line}-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/${mp3line}-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null
eyeD3 --remove-all-images "$temp/${mp3line}" 1>/dev/null 2>/dev/null
VerboseOutput 0 "Embedding albumart"
eyeD3 --add-image "$temp/${mp3line}-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/${mp3line}" 1>/dev/null 2>/dev/null
VerboseOutput 1 "Converted cover art: ${mp3line}"
else
VerboseOutput 4 "No front cover art found for ${mp3line}"
fi
else
VerboseOutput 3 "$line already converted"
fi;
done < "/tmp/music-sync-filelist"
fi
VerboseOutput 1 "Converted: $line"
else
VerboseOutput 3 "$line already converted"
fi;
}
CopyFiles() {
curline=0
percentage=0
while read -r line
do
if [[ ! -d "$dest" ]]; then
VerboseOutput 5 "Destination unreachable"
ExecTime
exit 4
fi
CopyFile() {
line=${1}
if [[ ! -d "$dest" ]]; then
VerboseOutput 5 "Destination unreachable"
ExecTime
exit 4
fi
if [[ ! -f "${temp}/$line" ]]; then
VerboseOutput 5 "Source-file ${temp}/$line Unreachable"
ExecTime
exit 3
fi
if [[ $convert == true ]]; then
mp3line=$(echo "$line" | sed -e 's/.flac/.mp3/')
else
mp3line=$(echo "$line")
fi
curline=$(expr ${curline} + 1)
total=$(cat /tmp/music-sync-filelist | wc -l)
percentage=$(echo "scale=4;${curline}/${total}" | bc)
percentage=$(echo "scale=2;${percentage}*100" | bc)
if [[ ! -f "${temp}/$mp3line" ]]; then
VerboseOutput 5 "Source-file ${temp}/$mp3line Unreachable"
ExecTime
exit 3
fi
VerboseOutput 1 "Copying: $line"
VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 2 of 2"
if [[ "$dest/$line" = */* ]]; then
mkdir -p "$dest/${line%/*}";
fi;
destline=$(echo $mp3line | sed -e 's/\.*\//\//g'| tr -s ' ')
VerboseOutput 1 "Copying: $line"
if [[ "$dest/$line" = */* ]]; then
VerboseOutput 0 "Creating folder $dest/${line%/*}"
mkdir -p "$dest/${destline%/*}";
fi;
cp -f $temp/$line $dest/$line 1>/dev/null 2>/dev/null
cp -fv "$temp/$mp3line" "$dest/${destline}" 1>/dev/null 2>/dev/null
VerboseOutput 2 "Copied: $line"
done < "/tmp/music-sync-filelist"
VerboseOutput 1 "Copied: $line"
}
CleanUp() {
VerboseOutput 1 "Cleaning Up"
if [[ -f /tmp/music-sync-filelist ]]; then
VerboseOutput 1 "Removing filelist"
rm "/tmp/music-sync-filelist"
fi
VerboseOutput 1 "Done"
@ -338,7 +454,7 @@ ExitHandler() {
trap 'ExitHandler' SIGINT
trap 'ErrorHandler $BASH_COMMAND' ERR
CheckDeps 0
CheckDeps 2
GetOptions $@
CheckDeps 1
if [[ $help == true ]]; then
@ -348,20 +464,19 @@ fi
if [[ -f /tmp/music-sync-filelist ]]; then
rm /tmp/music-sync-filelist
fi
VerboseOutput 2 "Scanning for new or updated files"
CreateFileList $source $dest ""
if [[ ! -f /tmp/music-sync-filelist ]]; then
VerboseOutput 2 "Nothing to do!"
CleanUp
exit 0
fi
if [[ $convert == true ]]; then
ConvertFiles
else
if [[ $convert == false ]]; then
if [[ $temp != "/tmp/converted" ]]; then
VerboseOutput 2 "Conversion not enabled. Ignoring cache folder"
fi
temp=$source
fi
CopyFiles
fi
Execute
CleanUp
ExecTime