Merge branch 'develop'

README.MD

@@ -1,22 +1,24 @@
 # Bash Music sync
-(c) 2018-2019 Jeroen De Meerleer <me@jeroened.be>
+(c) 2018-2020 Jeroen De Meerleer <me@jeroened.be>
 
 Script to sync music from one folder to another
 
 ## Usage:
 
 ```
+ Syncronises music from one folder to another.
+
+ Usage:
  music-sync <options> -s|--source <source> -d|--dest <destination>
  music-sync <options> <source> <destination>
 
-Syncronises music from one folder to another.
-
  Options:
   -s, --source <source>      The source folder of the music
   -d, --dest <destination>   The destination folder of the music
   -t, --temp <folder>        The temporary cache for converted files (default: /tmp/converted)
   -c, --convert <bitrate>    Convert files to a given bitrate in kbps before syncing (default: 192)
   -a, --resize-art <width>   Resize album-art before syncing (default width: 200)
+  -j, --jobs <nproc>         Number of processes to use in multi-threading (default: nproc - 2)
   -v, --verbose <0-6>        Set log level (default: 2)
   -h, --help                 Display this help text
 
@@ -39,7 +41,7 @@ Exit Codes:
 ```
 
 ## Licence
-Copyright 2018-2019 Jeroen De Meerleer <me@jeroened.be>
+Copyright 2018-2020 Jeroen De Meerleer <me@jeroened.be>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

music-sync.sh

@@ -1,4 +1,38 @@
 #!/bin/bash
+#
+#/
+#/ Usage:
+#/ music-sync <options> -s|--source <source> -d|--dest <destination>
+#/ music-sync <options> <source> <destination>
+#/ 
+#/ Syncronises music from one folder to another.
+#/ 
+#/ Options:
+#/  -s, --source <source>      The source folder of the music
+#/  -d, --dest <destination>   The destination folder of the music
+#/  -t, --temp <folder>        The temporary cache for converted files (default: /tmp/converted)
+#/  -c, --convert <bitrate>    Convert files to a given bitrate in kbps before syncing (default: 192)
+#/  -a, --resize-art <width>   Resize album-art before syncing (default width: 200)
+#/  -j, --jobs <nproc>         Number of processes to use in multi-threading (default: nproc - 2)
+#/  -v, --verbose <0-6>        Set log level (default: 2)
+#/  -h, --help                 Display this help text
+#/ 
+#/ Log levels:
+#/  0 | Verbose
+#/  1 | Debug
+#/  2 | Info
+#/  3 | Warning
+#/  4 | Error
+#/  5 | Fatal
+#/  6 | Silence
+#/ 
+#/ Exit Codes:
+#/  1 Dependencies not met
+#/  2 Invalid Argument
+#/  3 Source Unreachable
+#/  4 Destination Unreachable
+#/  5 Command failed
+#/
 
 begin=$(date +"%s")
 # saner programming env: these switches turn some bugs into errors
@@ -12,9 +46,13 @@ help=false
 temp="/tmp/converted"
 convertart=false
 coverartsize=200
+jobcount=$(expr $(nproc) - 2)
+multithread=false
+script_name=$(basename "${0}")
+script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 
 CheckDeps() {
-	if [[ $1 == 0 ]]; then
+	if [[ $1 == 2 ]]; then
 		# Check getopt
 		! getopt --test > /dev/null 
 		if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
@@ -24,20 +62,26 @@ CheckDeps() {
 			ExecTime
 			exit 1
 		fi
+		VerboseOutput 0 "\`getopt --test\` succeeded"
 	fi
 
 	if [[ $1 == 1 ]]; then
-		# Check lame
+		# Check ffmpeg
-		if [[ $convert == true  && ! $(lame --version 2>/dev/null) ]]; then
+		if [[ $convert == true ]]; then
-			VerboseOutput 5 "\`lame --version\` failed"
+			if [[ ! $(ffmpeg -h 2>/dev/null) ]]; then
-			VerboseOutput 5 "Sorry, It seems that lame is not installed on your system"
+				VerboseOutput 5 "\`ffmpeg -h\` failed"
-			VerboseOutput 5 "Please install lame from your repositories and make sure it is available in your \$PATH"
+				VerboseOutput 5 "Sorry, It seems that ffmpeg is not installed on your system"
+				VerboseOutput 5 "Please install ffmpeg from your repositories and make sure it is available in your \$PATH"
 				VerboseOutput 5 "Otherwise disable conversion"
 				ExecTime
 				exit 1
 			fi
+			VerboseOutput 0 "\`ffmpeg -h\` succeeded"
+		fi
+
 		# Check EyeD3
-		if [[ $convertart == true  && ! $(eyeD3 --version 2>/dev/null) ]]; then
+		if [[ $convertart == true ]]; then
+			if [[ ! $(eyeD3 --version 2>/dev/null) ]]; then
 				VerboseOutput 5 "\`eyeD3 --version\` failed"
 				VerboseOutput 5 "Sorry, It seems that eyeD3 is not installed on your system"
 				VerboseOutput 5 "Please install eyeD3 from your repositories and make sure it is available in your \$PATH"
@@ -45,8 +89,12 @@ CheckDeps() {
 				ExecTime
 				exit 1
 			fi
+			VerboseOutput 0 "\`eyeD3 --version\` succeeded"
+		fi
+
 		# Check ImageMagick
-		if [[ $convertart == true  && ! $(convert --version 2>/dev/null) ]]; then
+		if [[ $convertart == true ]]; then
+			if [[ ! $(convert --version 2>/dev/null) ]]; then
 				VerboseOutput 5 "\`convert --version\` failed"
 				VerboseOutput 5 "Sorry, It seems that ImageMagick is not installed on your system"
 				VerboseOutput 5 "Please install ImageMagick from your repositories and make sure it is available in your \$PATH"
@@ -54,14 +102,30 @@ CheckDeps() {
 				ExecTime
 				exit 1
 			fi
+			VerboseOutput 0 "\`convert --version\` succeeded"
 		fi
+
+		# Check Gnu parallel
+		if [[ $multithread == true ]]; then
+			if [[ ! $(parallel -h 2>/dev/null) ]]; then
+				VerboseOutput 5 "\`parallel -h\` failed"
+				VerboseOutput 5 "Sorry, It seems that parallel is not installed on your system"
+				VerboseOutput 5 "Please install gnu-parallel and env_parallel from your repositories and make sure it is available in your \$PATH"
+				VerboseOutput 5 "Otherwise disable multithreading"
+				ExecTime
+				exit 1
+			fi
+			VerboseOutput 0 "\`parallel -h \` succeeded"
+		fi
+	fi
+	VerboseOutput 1 "Dependency test OK"
 }
 
 GetOptions() {
 
 	# https://stackoverflow.com/a/29754866
-	OPTIONS=s:d:t:c::a::v::h
+	OPTIONS=s:d:t:c::a::v::hj::
-	LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help
+	LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help,jobs::
 
 	# -use ! and PIPESTATUS to get exit code with errexit set
 	# -temporarily store output to be able to check for errors
@@ -85,43 +149,60 @@ GetOptions() {
 				verbose=2
 				if [[ $2 != "" ]]; then
 					verbose=${2}
-					shift
 				fi
-				shift
+				shift 2
 				;;
 			-h|--help)
 				help=true
 				shift
 				;;
 			-s|--source)
+				VerboseOutput 0 "--source given."
+				VerboseOutput 1 "Source is ${2}"
 				source="$2"
 				shift 2
 				;;
 			-d|--dest)
+				VerboseOutput 0 "--dest given."
+				VerboseOutput 1 "Destination is ${2}"
 				dest="$2"
 				shift 2
 				;;
 			-t|--temp)
+				VerboseOutput 0 "--temp given"
+				VerboseOutput 1 "Tempfolder is ${2}"
 				temp="$2"
 				shift 2
 				;;
 			-c|--convert)
+				VerboseOutput 0 "--convert given"
 				convert=true
 				bitrate=192
 				if [[ $2 != "" ]]; then
 					bitrate=${2}
-					shift
 				fi
-				shift
+				VerboseOutput 1 "Converted bitrate is ${bitrate}"
+				shift 2
 				;;
 			-a|--resize-art)
+				VerboseOutput 0 "--resize-art given"
 				convertart=true
 				coverartsize=200
 				if [[ $2 != "" ]]; then
 					coverartsize=${2}
-					shift
 				fi
-				shift
+				shift 2
+				VerboseOutput 1 "Album art will ${coverartsize}px wide"
+				;;
+			-j|--jobs)
+				VerboseOutput 0 "--jobs given"
+				multithread=true
+				jobcount=$(expr $(nproc) - 2)
+				if [[ $2 != "" ]]; then
+					jobcount=${2}
+				fi
+				shift 2
+				VerboseOutput 1 "Multithreading will use ${jobcount} threads"
 				;;
 			--)
 				shift
@@ -135,51 +216,29 @@ GetOptions() {
 	done
 
 	if [[ ! -z "${1+x}" ]]; then
+		if [[ ${source} != "-" ]]; then
+			VerboseOutput 4 "Source provided twice. Continueing with ${1}"
+		fi
 		source="$1"
+		VerboseOutput 1 "Source is ${1}"
 	fi
 
 	if [[ ! -z "${2+x}" ]]; then
+		if [[ ${dest} != "-" ]]; then
+			VerboseOutput 4 "Destination provided twice. Continueing with ${2}"
+		fi
 		dest="$2"
+		VerboseOutput 1 "Destination is ${2}"
 	fi
 
 	if [[ $dest == "-" ]] || [[ $source == "-" ]]; then
 		help=true
 	fi
+	VerboseOutput 1 "Checks OK. Going on"
 }
 
 Usage() {
-	echo ""
+	grep '^#/' "${script_dir}/${script_name}" | sed 's/^#\/\w*//'
-	echo "Usage:"
-	echo "music-sync <options> -s|--source <source> -d|--dest <destination>"
-	echo "music-sync <options> <source> <destination>"
-	echo ""
-	echo "Syncronises music from one folder to another."
-	echo ""
-	echo "Options:"
-	echo " -s, --source <source>      The source folder of the music"
-	echo " -d, --dest <destination>   The destination folder of the music"
-	echo " -t, --temp <folder>        The temporary cache for converted files (default: /tmp/converted)"
-	echo " -c, --convert <bitrate>    Convert files to a given bitrate in kbps before syncing (default: 192)"
-	echo " -a, --resize-art <width>   Resize album-art before syncing (default width: 200)"
-	echo " -v, --verbose <0-6>        Set log level (default: 2)"
-	echo " -h, --help                 Display this help text"
-	echo ""
-	echo "Log levels:"
-	echo " 0 | Verbose"
-	echo " 1 | Debug"
-	echo " 2 | Info"
-	echo " 3 | Warning"
-	echo " 4 | Error"
-	echo " 5 | Fatal"
-	echo " 6 | Silence"
-	echo ""
-	echo "Exit Codes:"
-	echo " 1 Dependencies not met"
-	echo " 2 Invalid Argument"
-	echo " 3 Source Unreachable"
-	echo " 4 Destination Unreachable"
-	echo " 5 Command failed"
-	echo ""
 }
 
 VerboseOutput() {
@@ -219,99 +278,156 @@ CreateFileList() {
 	sourcepath="${sourcepath/\]/\\\]}"
 	for file in $sourcepath; do
 		origfile="${file#"$1/"}"
-		relfile=$(echo ${origfile} | sed -e 's/\(\.\)*$//g')
+		relfile=$(echo ${origfile} | sed -e 's/\(\.\)*$//g' | tr -s ' ')
+		mp3file=$(echo "${relfile}" | sed -e 's/.flac/.mp3/')
+		VerboseOutput 0 "Checking ${origfile}"
 		if [[ -d "${1}/$origfile" ]]; then
-			newdir="${3}/$relfile"
+			VerboseOutput 0 "${origfile} is folder"
+			newdir="${3}/$origfile"
 			newdir=${newdir#"/"}
 			VerboseOutput 1 "Entering $newdir"
 			CreateFileList "${1}/$origfile" "${2}/$relfile" "$newdir"
-		elif [[ "${1}/$origfile" != *".m3u" ]] && [[ ! -f "${2}/$relfile" || "${1}/$origfile" -nt "${2}/$relfile" ]]; then
+		elif [[ "${1}/$origfile" != *".m3u" ]] && [[ "${1}/$origfile" -nt "${2}/$mp3file" ]]; then
-			echo ${3}/$relfile >> /tmp/music-sync-filelist
+			echo ${3}/$origfile >> /tmp/music-sync-filelist	
-			VerboseOutput 2 "Added: ${3}/${relfile}"
+			VerboseOutput 0 "${origfile} is newer in source"
+			VerboseOutput 2 "Added: ${3}/${origfile}"
 		fi
+		if [[ "${1}/$origfile" == *".m3u" ]]; then
+			VerboseOutput 0 "${origfile} is playlist"
+		fi
+
 	done
 }
 
-ConvertFiles() {
+Execute() {
 	curline=0
 	percentage=0
-	while read -r line
+	IFS="
+";
+	if [[ $multithread == true ]]; then
+		VerboseOutput 3 "Running in parallel. Mind your load"
+		export -f ProcessFile
+		export -f ConvertFile
+		export -f CopyFile
+		export -f VerboseOutput
+		export -f ExecTime
+		export dest
+		export source
+		export bitrate
+		export convert
+		export temp
+		export verbose
+		export coverartsize
+		export begin
+		parallel --jobs ${jobcount} --will-cite --line-buffer --arg-file "/tmp/music-sync-filelist" ProcessFile 
+	else
+		for line in $(cat "/tmp/music-sync-filelist")
 		do
+			curline=$(expr ${curline} + 1)
+			total=$(cat /tmp/music-sync-filelist | wc -l)
+			percentage=$(echo "scale=4;${curline}/${total}" | bc)
+			percentage=$(echo "scale=2;${percentage}*100" | bc)
+			VerboseOutput 2 "Current file: $line"
+			VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%)"
+			ProcessFile $line
+		done
+	fi
+	VerboseOutput 2 "Done!"
+}
+ProcessFile() {
+	line=${1}
+	curline="$(grep -n "$line" /tmp/music-sync-filelist | head -n 1 | cut -d: -f1)"
+	total=$(cat /tmp/music-sync-filelist | wc -l)
+	percentage=$(echo "scale=4;${curline}/${total}" | bc)
+	percentage=$(echo "scale=2;${percentage}*100" | bc)
+	VerboseOutput 2 "Current File: $line"
+	VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%)"
+	if [[ $convert == true ]]; then
+		ConvertFile "$line"
+	fi
+	CopyFile "$line"
+}
 
+ConvertFile() {
+	line=${1}
 	if [[ ! -f "${source}/$line" ]]; then
 		VerboseOutput 5 "Source-file ${source}/$line Unreachable"
 		ExecTime
 		exit 3
 	fi
 
-		curline=$(expr ${curline} + 1)
-		total=$(cat /tmp/music-sync-filelist | wc -l)
-		percentage=$(echo "scale=4;${curline}/${total}" | bc)
-		percentage=$(echo "scale=2;${percentage}*100" | bc)
-		VerboseOutput 1 "Converting: $line"
-		VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 1 of 2"
-
 	if [[ "$temp/$line" = */* ]]; then
+		VerboseOutput 0 "Creating folder $temp/${line%/*}"
 		mkdir -p "$temp/${line%/*}";
 	fi;
-		if [[ ! -f "$temp/$line" || "${source}/$line" -nt "$temp/$line" ]]; then
+	mp3line=$(echo "$line" | sed -e 's/.flac/.mp3/')
-			lame -b ${bitrate} $source/$line $temp/$line 1>/dev/null 2>/dev/null
+	if [[ ! -f "$temp/$mp3line" || "${source}/$mp3line" -nt "$temp/$mp3line" ]]; then
-			VerboseOutput 2 "Converted: $line"
+		if [[ $line != *".mp3" ]]; then
-			if [[ $convertart == true ]]; then
+			VerboseOutput 3 "${line} will be converted to mp3-file ${mp3line}"
-				mkdir -p "$temp/$line-images/"
-				eyeD3 --write-images "$temp/$line-images/" "$temp/$line" 1>/dev/null 2>/dev/null
-				convert "$temp/$line-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/$line-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null
-				eyeD3 --remove-all-images "$temp/$line" 1>/dev/null 2>/dev/null
-				eyeD3 --add-image "$temp/$line-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/$line" 1>/dev/null 2>/dev/null
-				VerboseOutput 2 "Converted cover art: $line"
 		fi
-			
+		VerboseOutput 0 "Converting MP3-file $line"
+		ffmpeg -y -i "$source/$line" -acodec libmp3lame -map_metadata 0 -id3v2_version 3 -b:a ${bitrate}k "$temp/${mp3line}" 1>/dev/null 2>/dev/null
+		if [[ $convertart == true ]]; then
+			VerboseOutput 0 "Creating folder $temp/${mp3line}-images/"
+			mkdir -p "$temp/${mp3line}-images/"
+			VerboseOutput 0 "Extracted albumart"
+			eyeD3 --write-images "$temp/${mp3line}-images/" "$temp/${mp3line}" 1>/dev/null 2>/dev/null
+			frontcovers=(${temp}/${mp3line}-images/FRONT_COVER.*)
+			if [ -e "$frontcovers" ]; then
+				VerboseOutput 0 "Converting albumart"
+				convert "$temp/${mp3line}-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/${mp3line}-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null
+				eyeD3 --remove-all-images "$temp/${mp3line}" 1>/dev/null 2>/dev/null
+				VerboseOutput 0 "Embedding albumart"
+				eyeD3 --add-image "$temp/${mp3line}-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/${mp3line}" 1>/dev/null 2>/dev/null
+				VerboseOutput 1 "Converted cover art: ${mp3line}"
+			else
+				VerboseOutput 4 "No front cover art found for ${mp3line}"
+			fi
+		fi
+		VerboseOutput 1 "Converted: $line"
 	else
 		VerboseOutput 3 "$line already converted"
 	fi;
-
-	done < "/tmp/music-sync-filelist"
 }
 
-CopyFiles() {
+CopyFile() {
-	curline=0
+	line=${1}
-	percentage=0
-	while read -r line
-	do
 	if [[ ! -d "$dest" ]]; then
 		VerboseOutput 5 "Destination unreachable"
 		ExecTime
 		exit 4
 	fi
 
-		if [[ ! -f "${temp}/$line" ]]; then
+	if [[ $convert == true ]]; then
-			VerboseOutput 5 "Source-file ${temp}/$line Unreachable"
+		mp3line=$(echo "$line" | sed -e 's/.flac/.mp3/')
+	else 
+		mp3line=$(echo "$line")
+	fi
+
+	if [[ ! -f "${temp}/$mp3line" ]]; then
+		VerboseOutput 5 "Source-file ${temp}/$mp3line Unreachable"
 		ExecTime
 		exit 3
 	fi
 
-		curline=$(expr ${curline} + 1)
+	destline=$(echo $mp3line | sed -e 's/\.*\//\//g'| tr -s ' ')
-		total=$(cat /tmp/music-sync-filelist | wc -l)
-		percentage=$(echo "scale=4;${curline}/${total}" | bc)
-		percentage=$(echo "scale=2;${percentage}*100" | bc)
-
 	VerboseOutput 1 "Copying: $line"
-		VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 2 of 2"
 	
 	if [[ "$dest/$line" = */* ]]; then
-			mkdir -p "$dest/${line%/*}";
+		VerboseOutput 0 "Creating folder $dest/${line%/*}"
+		mkdir -p "$dest/${destline%/*}";
 	fi;
 
-		cp -f $temp/$line $dest/$line 1>/dev/null 2>/dev/null
+	cp -fv "$temp/$mp3line" "$dest/${destline}" 1>/dev/null 2>/dev/null
 
-		VerboseOutput 2 "Copied: $line"
+	VerboseOutput 1 "Copied: $line"
-
-	done < "/tmp/music-sync-filelist"
 }
 
+
 CleanUp() {
 	VerboseOutput 1 "Cleaning Up"
 	if [[ -f /tmp/music-sync-filelist ]]; then
+		VerboseOutput 1 "Removing filelist"
 		rm "/tmp/music-sync-filelist"
 	fi
 	VerboseOutput 1 "Done"
@@ -338,7 +454,7 @@ ExitHandler() {
 trap 'ExitHandler' SIGINT
 trap 'ErrorHandler $BASH_COMMAND' ERR
 
-CheckDeps 0
+CheckDeps 2
 GetOptions $@
 CheckDeps 1
 if [[ $help == true ]]; then
@@ -348,20 +464,19 @@ fi
 if [[ -f /tmp/music-sync-filelist ]]; then
 	rm /tmp/music-sync-filelist
 fi
+VerboseOutput 2 "Scanning for new or updated files"
 CreateFileList $source $dest ""
 if [[ ! -f /tmp/music-sync-filelist ]]; then
 	VerboseOutput 2 "Nothing to do!"
 	CleanUp
 	exit 0
 fi
-if [[ $convert == true ]]; then
+if [[ $convert == false ]]; then
-	ConvertFiles
-else
 	if [[ $temp != "/tmp/converted" ]]; then
 		VerboseOutput 2 "Conversion not enabled. Ignoring cache folder"
 	fi
 	temp=$source
 fi	
-CopyFiles
+Execute
 CleanUp
 ExecTime