Added progress to the parallel output

I will not lie

README.MD

@@ -1,45 +1,47 @@
 # Bash Music sync
-(c) 2018-2019 Jeroen De Meerleer <me@jeroened.be>
+(c) 2018-2020 Jeroen De Meerleer <me@jeroened.be>
 
 Script to sync music from one folder to another
 
 ## Usage:
 
 ```
-music-sync <options> -s|--source <source> -d|--dest <destination>
+ Syncronises music from one folder to another.
-music-sync <options> <source> <destination>
 
-Syncronises music from one folder to another.
+ Usage:
+ music-sync <options> -s|--source <source> -d|--dest <destination>
+ music-sync <options> <source> <destination>
 
-Options:
+ Options:
- -s, --source <source>      The source folder of the music
+  -s, --source <source>      The source folder of the music
- -d, --dest <destination>   The destination folder of the music
+  -d, --dest <destination>   The destination folder of the music
- -t, --temp <folder>        The temporary cache for converted files (default: /tmp/converted)
+  -t, --temp <folder>        The temporary cache for converted files (default: /tmp/converted)
- -c, --convert <bitrate>    Convert files to a given bitrate in kbps before syncing (default: 192)
+  -c, --convert <bitrate>    Convert files to a given bitrate in kbps before syncing (default: 192)
- -a, --resize-art <width>   Resize album-art before syncing (default width: 200)
+  -a, --resize-art <width>   Resize album-art before syncing (default width: 200)
- -v, --verbose <0-6>        Set log level (default: 2)
+  -j, --jobs <nproc>         Number of processes to use in multi-threading (default: nproc - 2)
- -h, --help                 Display this help text
+  -v, --verbose <0-6>        Set log level (default: 2)
+  -h, --help                 Display this help text
 
-Log levels:
+ Log levels:
- 0 | Verbose
+  0 | Verbose
- 1 | Debug
+  1 | Debug
- 2 | Info
+  2 | Info
- 3 | Warning
+  3 | Warning
- 4 | Error
+  4 | Error
- 5 | Fatal
+  5 | Fatal
- 6 | Silence
+  6 | Silence
 
-Exit Codes:
+ Exit Codes:
- 1 Dependencies not met
+  1 Dependencies not met
- 2 Invalid Argument
+  2 Invalid Argument
- 3 Source Unreachable
+  3 Source Unreachable
- 4 Destination Unreachable
+  4 Destination Unreachable
- 5 Command failed
+  5 Command failed
 
 ```
 
 ## Licence
-Copyright 2018-2019 Jeroen De Meerleer <me@jeroened.be>
+Copyright 2018-2020 Jeroen De Meerleer <me@jeroened.be>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

music-sync.sh

@@ -1,4 +1,38 @@
 #!/bin/bash
+#
+#/
+#/ Usage:
+#/ music-sync <options> -s|--source <source> -d|--dest <destination>
+#/ music-sync <options> <source> <destination>
+#/ 
+#/ Syncronises music from one folder to another.
+#/ 
+#/ Options:
+#/  -s, --source <source>      The source folder of the music
+#/  -d, --dest <destination>   The destination folder of the music
+#/  -t, --temp <folder>        The temporary cache for converted files (default: /tmp/converted)
+#/  -c, --convert <bitrate>    Convert files to a given bitrate in kbps before syncing (default: 192)
+#/  -a, --resize-art <width>   Resize album-art before syncing (default width: 200)
+#/  -j, --jobs <nproc>         Number of processes to use in multi-threading (default: nproc - 2)
+#/  -v, --verbose <0-6>        Set log level (default: 2)
+#/  -h, --help                 Display this help text
+#/ 
+#/ Log levels:
+#/  0 | Verbose
+#/  1 | Debug
+#/  2 | Info
+#/  3 | Warning
+#/  4 | Error
+#/  5 | Fatal
+#/  6 | Silence
+#/ 
+#/ Exit Codes:
+#/  1 Dependencies not met
+#/  2 Invalid Argument
+#/  3 Source Unreachable
+#/  4 Destination Unreachable
+#/  5 Command failed
+#/
 
 begin=$(date +"%s")
 # saner programming env: these switches turn some bugs into errors
@@ -12,9 +46,13 @@ help=false
 temp="/tmp/converted"
 convertart=false
 coverartsize=200
+jobcount=$(expr $(nproc) - 2)
+multithread=false
+script_name=$(basename "${0}")
+script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 
 CheckDeps() {
-	if [[ $1 == 0 ]]; then
+	if [[ $1 == 2 ]]; then
 		# Check getopt
 		! getopt --test > /dev/null 
 		if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
@@ -28,38 +66,57 @@ CheckDeps() {
 	fi
 
 	if [[ $1 == 1 ]]; then
-		# Check lame
+		# Check ffmpeg
-		if [[ $convert == true  && ! $(lame --version 2>/dev/null) ]]; then
+		if [[ $convert == true ]]; then
-			VerboseOutput 5 "\`lame --version\` failed"
+			if [[ ! $(ffmpeg -h 2>/dev/null) ]]; then
-			VerboseOutput 5 "Sorry, It seems that lame is not installed on your system"
+				VerboseOutput 5 "\`ffmpeg -h\` failed"
-			VerboseOutput 5 "Please install lame from your repositories and make sure it is available in your \$PATH"
+				VerboseOutput 5 "Sorry, It seems that ffmpeg is not installed on your system"
-			VerboseOutput 5 "Otherwise disable conversion"
+				VerboseOutput 5 "Please install ffmpeg from your repositories and make sure it is available in your \$PATH"
-			ExecTime
+				VerboseOutput 5 "Otherwise disable conversion"
-			exit 1
+				ExecTime
+				exit 1
+			fi
+			VerboseOutput 0 "\`ffmpeg -h\` succeeded"
 		fi
-		VerboseOutput 0 "\`lame --version\` succeeded"
 
 		# Check EyeD3
-		if [[ $convertart == true  && ! $(eyeD3 --version 2>/dev/null) ]]; then
+		if [[ $convertart == true ]]; then
-			VerboseOutput 5 "\`eyeD3 --version\` failed"
+			if [[ ! $(eyeD3 --version 2>/dev/null) ]]; then
-			VerboseOutput 5 "Sorry, It seems that eyeD3 is not installed on your system"
+				VerboseOutput 5 "\`eyeD3 --version\` failed"
-			VerboseOutput 5 "Please install eyeD3 from your repositories and make sure it is available in your \$PATH"
+				VerboseOutput 5 "Sorry, It seems that eyeD3 is not installed on your system"
-			VerboseOutput 5 "Otherwise disable albumart conversion"
+				VerboseOutput 5 "Please install eyeD3 from your repositories and make sure it is available in your \$PATH"
-			ExecTime
+				VerboseOutput 5 "Otherwise disable albumart conversion"
-			exit 1
+				ExecTime
+				exit 1
+			fi
+			VerboseOutput 0 "\`eyeD3 --version\` succeeded"
 		fi
-		VerboseOutput 0 "\`eyeD3 --version\` succeeded"
 
 		# Check ImageMagick
-		if [[ $convertart == true  && ! $(convert --version 2>/dev/null) ]]; then
+		if [[ $convertart == true ]]; then
-			VerboseOutput 5 "\`convert --version\` failed"
+			if [[ ! $(convert --version 2>/dev/null) ]]; then
-			VerboseOutput 5 "Sorry, It seems that ImageMagick is not installed on your system"
+				VerboseOutput 5 "\`convert --version\` failed"
-			VerboseOutput 5 "Please install ImageMagick from your repositories and make sure it is available in your \$PATH"
+				VerboseOutput 5 "Sorry, It seems that ImageMagick is not installed on your system"
-			VerboseOutput 5 "Otherwise disable albumart conversion"
+				VerboseOutput 5 "Please install ImageMagick from your repositories and make sure it is available in your \$PATH"
-			ExecTime
+				VerboseOutput 5 "Otherwise disable albumart conversion"
-			exit 1
+				ExecTime
+				exit 1
+			fi
+			VerboseOutput 0 "\`convert --version\` succeeded"
+		fi
+
+		# Check Gnu parallel
+		if [[ $multithread == true ]]; then
+			if [[ ! $(parallel -h 2>/dev/null) ]]; then
+				VerboseOutput 5 "\`parallel -h\` failed"
+				VerboseOutput 5 "Sorry, It seems that parallel is not installed on your system"
+				VerboseOutput 5 "Please install gnu-parallel and env_parallel from your repositories and make sure it is available in your \$PATH"
+				VerboseOutput 5 "Otherwise disable multithreading"
+				ExecTime
+				exit 1
+			fi
+			VerboseOutput 0 "\`parallel -h \` succeeded"
 		fi
-		VerboseOutput 0 "\`convert --version\` succeeded"
 	fi
 	VerboseOutput 1 "Dependency test OK"
 }
@@ -67,8 +124,8 @@ CheckDeps() {
 GetOptions() {
 
 	# https://stackoverflow.com/a/29754866
-	OPTIONS=s:d:t:c::a::v::h
+	OPTIONS=s:d:t:c::a::v::hj::
-	LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help
+	LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help,jobs::
 
 	# -use ! and PIPESTATUS to get exit code with errexit set
 	# -temporarily store output to be able to check for errors
@@ -92,9 +149,8 @@ GetOptions() {
 				verbose=2
 				if [[ $2 != "" ]]; then
 					verbose=${2}
-					shift
 				fi
-				shift
+				shift 2
 				;;
 			-h|--help)
 				help=true
@@ -124,10 +180,9 @@ GetOptions() {
 				bitrate=192
 				if [[ $2 != "" ]]; then
 					bitrate=${2}
-					shift
 				fi
 				VerboseOutput 1 "Converted bitrate is ${bitrate}"
-				shift
+				shift 2
 				;;
 			-a|--resize-art)
 				VerboseOutput 0 "--resize-art given"
@@ -135,11 +190,20 @@ GetOptions() {
 				coverartsize=200
 				if [[ $2 != "" ]]; then
 					coverartsize=${2}
-					shift
 				fi
-				shift
+				shift 2
 				VerboseOutput 1 "Album art will ${coverartsize}px wide"
 				;;
+			-j|--jobs)
+				VerboseOutput 0 "--jobs given"
+				multithread=true
+				jobcount=$(expr $(nproc) - 2)
+				if [[ $2 != "" ]]; then
+					jobcount=${2}
+				fi
+				shift 2
+				VerboseOutput 1 "Multithreading will use ${jobcount} threads"
+				;;
 			--)
 				shift
 				break
@@ -170,43 +234,11 @@ GetOptions() {
 	if [[ $dest == "-" ]] || [[ $source == "-" ]]; then
 		help=true
 	fi
-
 	VerboseOutput 1 "Checks OK. Going on"
 }
 
 Usage() {
-	echo ""
+	grep '^#/' "${script_dir}/${script_name}" | sed 's/^#\/\w*//'
-	echo "Usage:"
-	echo "music-sync <options> -s|--source <source> -d|--dest <destination>"
-	echo "music-sync <options> <source> <destination>"
-	echo ""
-	echo "Syncronises music from one folder to another."
-	echo ""
-	echo "Options:"
-	echo " -s, --source <source>      The source folder of the music"
-	echo " -d, --dest <destination>   The destination folder of the music"
-	echo " -t, --temp <folder>        The temporary cache for converted files (default: /tmp/converted)"
-	echo " -c, --convert <bitrate>    Convert files to a given bitrate in kbps before syncing (default: 192)"
-	echo " -a, --resize-art <width>   Resize album-art before syncing (default width: 200)"
-	echo " -v, --verbose <0-6>        Set log level (default: 2)"
-	echo " -h, --help                 Display this help text"
-	echo ""
-	echo "Log levels:"
-	echo " 0 | Verbose"
-	echo " 1 | Debug"
-	echo " 2 | Info"
-	echo " 3 | Warning"
-	echo " 4 | Error"
-	echo " 5 | Fatal"
-	echo " 6 | Silence"
-	echo ""
-	echo "Exit Codes:"
-	echo " 1 Dependencies not met"
-	echo " 2 Invalid Argument"
-	echo " 3 Source Unreachable"
-	echo " 4 Destination Unreachable"
-	echo " 5 Command failed"
-	echo ""
 }
 
 VerboseOutput() {
@@ -246,7 +278,8 @@ CreateFileList() {
 	sourcepath="${sourcepath/\]/\\\]}"
 	for file in $sourcepath; do
 		origfile="${file#"$1/"}"
-		relfile=$(echo ${origfile} | sed -e 's/\(\.\)*$//g')
+		relfile=$(echo ${origfile} | sed -e 's/\(\.\)*$//g' | tr -s ' ')
+		mp3file=$(echo "${relfile}" | sed -e 's/.flac/.mp3/')
 		VerboseOutput 0 "Checking ${origfile}"
 		if [[ -d "${1}/$origfile" ]]; then
 			VerboseOutput 0 "${origfile} is folder"
@@ -254,7 +287,7 @@ CreateFileList() {
 			newdir=${newdir#"/"}
 			VerboseOutput 1 "Entering $newdir"
 			CreateFileList "${1}/$origfile" "${2}/$relfile" "$newdir"
-		elif [[ "${1}/$origfile" != *".m3u" ]] && [[ ! -f "${2}/$origfile" || "${1}/$origfile" -nt "${2}/$relfile" ]]; then
+		elif [[ "${1}/$origfile" != *".m3u" ]] && [[ "${1}/$origfile" -nt "${2}/$mp3file" ]]; then
 			echo ${3}/$origfile >> /tmp/music-sync-filelist	
 			VerboseOutput 0 "${origfile} is newer in source"
 			VerboseOutput 2 "Added: ${3}/${origfile}"
@@ -266,91 +299,131 @@ CreateFileList() {
 	done
 }
 
-ConvertFiles() {
+Execute() {
 	curline=0
 	percentage=0
-	while read -r line
+	IFS="
-	do
+";
+	if [[ $multithread == true ]]; then
+		VerboseOutput 3 "Running in parallel. Mind your load"
+		export -f ProcessFile
+		export -f ConvertFile
+		export -f CopyFile
+		export -f VerboseOutput
+		export -f ExecTime
+		export dest
+		export source
+		export bitrate
+		export convert
+		export temp
+		export verbose
+		export coverartsize
+		export begin
+		parallel --jobs ${jobcount} --will-cite --line-buffer --arg-file "/tmp/music-sync-filelist" ProcessFile 
+	else
+		for line in $(cat "/tmp/music-sync-filelist")
+		do
+			curline=$(expr ${curline} + 1)
+			total=$(cat /tmp/music-sync-filelist | wc -l)
+			percentage=$(echo "scale=4;${curline}/${total}" | bc)
+			percentage=$(echo "scale=2;${percentage}*100" | bc)
+			VerboseOutput 2 "Current file: $line"
+			VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%)"
+			ProcessFile $line
+		done
+	fi
+	VerboseOutput 2 "Done!"
+}
+ProcessFile() {
+	line=${1}
+	curline="$(grep -n "$line" /tmp/music-sync-filelist | head -n 1 | cut -d: -f1)"
+	total=$(cat /tmp/music-sync-filelist | wc -l)
+	percentage=$(echo "scale=4;${curline}/${total}" | bc)
+	percentage=$(echo "scale=2;${percentage}*100" | bc)
+	VerboseOutput 2 "Current File: $line"
+	VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%)"
+	if [[ $convert == true ]]; then
+		ConvertFile "$line"
+	fi
+	CopyFile "$line"
+}
 
-		if [[ ! -f "${source}/$line" ]]; then
+ConvertFile() {
-			VerboseOutput 5 "Source-file ${source}/$line Unreachable"
+	line=${1}
-			ExecTime
+	if [[ ! -f "${source}/$line" ]]; then
-			exit 3
+		VerboseOutput 5 "Source-file ${source}/$line Unreachable"
+		ExecTime
+		exit 3
+	fi
+
+	if [[ "$temp/$line" = */* ]]; then
+		VerboseOutput 0 "Creating folder $temp/${line%/*}"
+		mkdir -p "$temp/${line%/*}";
+	fi;
+	mp3line=$(echo "$line" | sed -e 's/.flac/.mp3/')
+	if [[ ! -f "$temp/$mp3line" || "${source}/$mp3line" -nt "$temp/$mp3line" ]]; then
+		if [[ $line != *".mp3" ]]; then
+			VerboseOutput 3 "${line} will be converted to mp3-file ${mp3line}"
 		fi
-
+		VerboseOutput 0 "Converting MP3-file $line"
-		curline=$(expr ${curline} + 1)
+		ffmpeg -y -i "$source/$line" -acodec libmp3lame -map_metadata 0 -id3v2_version 3 -b:a ${bitrate}k "$temp/${mp3line}" 1>/dev/null 2>/dev/null
-		total=$(cat /tmp/music-sync-filelist | wc -l)
+		if [[ $convertart == true ]]; then
-		percentage=$(echo "scale=4;${curline}/${total}" | bc)
+			VerboseOutput 0 "Creating folder $temp/${mp3line}-images/"
-		percentage=$(echo "scale=2;${percentage}*100" | bc)
+			mkdir -p "$temp/${mp3line}-images/"
-		VerboseOutput 1 "Converting: $line"
+			VerboseOutput 0 "Extracted albumart"
-		VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 1 of 2"
+			eyeD3 --write-images "$temp/${mp3line}-images/" "$temp/${mp3line}" 1>/dev/null 2>/dev/null
-
+			frontcovers=(${temp}/${mp3line}-images/FRONT_COVER.*)
-		if [[ "$temp/$line" = */* ]]; then
+			if [ -e "$frontcovers" ]; then
-			VerboseOutput 0 "Creating folder $temp/${line%/*}"
-			mkdir -p "$temp/${line%/*}";
-		fi;
-		if [[ ! -f "$temp/$line" || "${source}/$line" -nt "$temp/$line" ]]; then
-			VerboseOutput 0 "Converting MP3-file $temp/${line%/*}"
-			lame -b ${bitrate} $source/$line $temp/$line 1>/dev/null 2>/dev/null
-			if [[ $convertart == true ]]; then
-				VerboseOutput 0 "Creating folder $temp/$line-images/"
-				mkdir -p "$temp/$line-images/"
-				VerboseOutput 0 "Extracted albumart"
-				eyeD3 --write-images "$temp/$line-images/" "$temp/$line" 1>/dev/null 2>/dev/null
 				VerboseOutput 0 "Converting albumart"
-				convert "$temp/$line-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/$line-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null
+				convert "$temp/${mp3line}-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/${mp3line}-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null
-				eyeD3 --remove-all-images "$temp/$line" 1>/dev/null 2>/dev/null
+				eyeD3 --remove-all-images "$temp/${mp3line}" 1>/dev/null 2>/dev/null
 				VerboseOutput 0 "Embedding albumart"
-				eyeD3 --add-image "$temp/$line-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/$line" 1>/dev/null 2>/dev/null
+				eyeD3 --add-image "$temp/${mp3line}-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/${mp3line}" 1>/dev/null 2>/dev/null
-				VerboseOutput 1 "Converted cover art: $line"
+				VerboseOutput 1 "Converted cover art: ${mp3line}"
+			else
+				VerboseOutput 4 "No front cover art found for ${mp3line}"
 			fi
-			VerboseOutput 2 "Converted: $line"
+		fi
-		else
+		VerboseOutput 1 "Converted: $line"
-			VerboseOutput 3 "$line already converted"
+	else
-		fi;
+		VerboseOutput 3 "$line already converted"
-
+	fi;
-	done < "/tmp/music-sync-filelist"
-	VerboseOutput 2 "Done converting files"
 }
 
-CopyFiles() {
+CopyFile() {
-	curline=0
+	line=${1}
-	percentage=0
+	if [[ ! -d "$dest" ]]; then
-	while read -r line
+		VerboseOutput 5 "Destination unreachable"
-	do
+		ExecTime
-		if [[ ! -d "$dest" ]]; then
+		exit 4
-			VerboseOutput 5 "Destination unreachable"
+	fi
-			ExecTime
-			exit 4
-		fi
 
-		if [[ ! -f "${temp}/$line" ]]; then
+	if [[ $convert == true ]]; then
-			VerboseOutput 5 "Source-file ${temp}/$line Unreachable"
+		mp3line=$(echo "$line" | sed -e 's/.flac/.mp3/')
-			ExecTime
+	else 
-			exit 3
+		mp3line=$(echo "$line")
-		fi
+	fi
 
-		curline=$(expr ${curline} + 1)
+	if [[ ! -f "${temp}/$mp3line" ]]; then
-		total=$(cat /tmp/music-sync-filelist | wc -l)
+		VerboseOutput 5 "Source-file ${temp}/$mp3line Unreachable"
-		percentage=$(echo "scale=4;${curline}/${total}" | bc)
+		ExecTime
-		percentage=$(echo "scale=2;${percentage}*100" | bc)
+		exit 3
-		destline=$(echo $line | sed -e 's/\.*\//\//g')
+	fi
-		VerboseOutput 1 "Copying: $line"
-		VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 2 of 2"
-		
-		if [[ "$dest/$line" = */* ]]; then
-			VerboseOutput 0 "Creating folder $dest/${line%/*}"
-			mkdir -p "$dest/${destline%/*}";
-		fi;
 
-		cp -f $temp/$line $dest/$destline 1>/dev/null 2>/dev/null
+	destline=$(echo $mp3line | sed -e 's/\.*\//\//g'| tr -s ' ')
+	VerboseOutput 1 "Copying: $line"
+	
+	if [[ "$dest/$line" = */* ]]; then
+		VerboseOutput 0 "Creating folder $dest/${line%/*}"
+		mkdir -p "$dest/${destline%/*}";
+	fi;
 
-		VerboseOutput 2 "Copied: $line"
+	cp -fv "$temp/$mp3line" "$dest/${destline}" 1>/dev/null 2>/dev/null
 
-	done < "/tmp/music-sync-filelist"
+	VerboseOutput 1 "Copied: $line"
-	VerboseOutput 2 "Done copying files"
 }
 
+
 CleanUp() {
 	VerboseOutput 1 "Cleaning Up"
 	if [[ -f /tmp/music-sync-filelist ]]; then
@@ -381,7 +454,7 @@ ExitHandler() {
 trap 'ExitHandler' SIGINT
 trap 'ErrorHandler $BASH_COMMAND' ERR
 
-CheckDeps 0
+CheckDeps 2
 GetOptions $@
 CheckDeps 1
 if [[ $help == true ]]; then
@@ -398,14 +471,12 @@ if [[ ! -f /tmp/music-sync-filelist ]]; then
 	CleanUp
 	exit 0
 fi
-if [[ $convert == true ]]; then
+if [[ $convert == false ]]; then
-	ConvertFiles
-else
 	if [[ $temp != "/tmp/converted" ]]; then
 		VerboseOutput 2 "Conversion not enabled. Ignoring cache folder"
 	fi
 	temp=$source
-fi
+fi	
-CopyFiles
+Execute
 CleanUp
 ExecTime