From 72d8a2dcfff25b33efea2a0e06c62fb6d6852890 Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Sat, 16 Mar 2019 18:31:17 +0100 Subject: [PATCH 01/16] Added some more output --- music-sync.sh | 53 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/music-sync.sh b/music-sync.sh index 0b71f64..010f41a 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -24,6 +24,7 @@ CheckDeps() { ExecTime exit 1 fi + VerboseOutput 0 "\`getopt --test\` succeeded" fi if [[ $1 == 1 ]]; then @@ -36,6 +37,8 @@ CheckDeps() { ExecTime exit 1 fi + VerboseOutput 0 "\`lame --version\` succeeded" + # Check EyeD3 if [[ $convertart == true && ! $(eyeD3 --version 2>/dev/null) ]]; then VerboseOutput 5 "\`eyeD3 --version\` failed" @@ -45,6 +48,8 @@ CheckDeps() { ExecTime exit 1 fi + VerboseOutput 0 "\`eyeD3 --version\` succeeded" + # Check ImageMagick if [[ $convertart == true && ! $(convert --version 2>/dev/null) ]]; then VerboseOutput 5 "\`convert --version\` failed" @@ -54,7 +59,9 @@ CheckDeps() { ExecTime exit 1 fi + VerboseOutput 0 "\`convert --version\` succeeded" fi + VerboseOutput 1 "Dependency test OK" } GetOptions() { @@ -94,27 +101,36 @@ GetOptions() { shift ;; -s|--source) + VerboseOutput 0 "--source given." + VerboseOutput 1 "Source is ${2}" source="$2" shift 2 ;; -d|--dest) + VerboseOutput 0 "--dest given." + VerboseOutput 1 "Destination is ${2}" dest="$2" shift 2 ;; -t|--temp) + VerboseOutput 0 "--temp given" + VerboseOutput 1 "Tempfolder is ${2}" temp="$2" shift 2 ;; -c|--convert) + VerboseOutput 0 "--convert given" convert=true bitrate=192 if [[ $2 != "" ]]; then bitrate=${2} shift fi + VerboseOutput 1 "Converted bitrate is ${bitrate}" shift ;; -a|--resize-art) + VerboseOutput 0 "--resize-art given" convertart=true coverartsize=200 if [[ $2 != "" ]]; then @@ -122,6 +138,7 @@ GetOptions() { shift fi shift + VerboseOutput 1 "Album art will ${coverartsize}px wide" ;; --) shift @@ -135,16 +152,26 @@ GetOptions() { done if [[ ! -z "${1+x}" ]]; then + if [[ ${source} != "-" ]]; then + VerboseOutput 4 "Source provided twice. Continueing with ${1}" + fi source="$1" + VerboseOutput 1 "Source is ${1}" fi if [[ ! -z "${2+x}" ]]; then + if [[ ${dest} != "-" ]]; then + VerboseOutput 4 "Destination provided twice. Continueing with ${2}" + fi dest="$2" + VerboseOutput 1 "Destination is ${2}" fi if [[ $dest == "-" ]] || [[ $source == "-" ]]; then help=true fi + + VerboseOutput 1 "Checks OK. Going on" } Usage() { @@ -220,15 +247,22 @@ CreateFileList() { for file in $sourcepath; do origfile="${file#"$1/"}" relfile=$(echo ${origfile} | sed -e 's/\(\.\)*$//g') + VerboseOutput 0 "Checking ${origfile}" if [[ -d "${1}/$origfile" ]]; then + VerboseOutput 0 "${origfile} is folder" newdir="${3}/$relfile" newdir=${newdir#"/"} VerboseOutput 1 "Entering $newdir" CreateFileList "${1}/$origfile" "${2}/$relfile" "$newdir" elif [[ "${1}/$origfile" != *".m3u" ]] && [[ ! -f "${2}/$relfile" || "${1}/$origfile" -nt "${2}/$relfile" ]]; then - echo ${3}/$relfile >> /tmp/music-sync-filelist - VerboseOutput 2 "Added: ${3}/${relfile}" + echo ${3}/$relfile >> /tmp/music-sync-filelist + VerboseOutput 0 "${origfile} is newer in source" + VerboseOutput 1 "Added: ${3}/${origfile}" fi + if [[ "${1}/$origfile" == *".m3u" ]]; then + VerboseOutput 0 "${origfile} is playlist" + fi + done } @@ -252,25 +286,31 @@ ConvertFiles() { VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 1 of 2" if [[ "$temp/$line" = */* ]]; then + VerboseOutput 0 "Creating folder $temp/${line%/*}" mkdir -p "$temp/${line%/*}"; fi; if [[ ! -f "$temp/$line" || "${source}/$line" -nt "$temp/$line" ]]; then + VerboseOutput 0 "Converting MP3-file $temp/${line%/*}" lame -b ${bitrate} $source/$line $temp/$line 1>/dev/null 2>/dev/null - VerboseOutput 2 "Converted: $line" if [[ $convertart == true ]]; then + VerboseOutput 0 "Creating folder $temp/$line-images/" mkdir -p "$temp/$line-images/" + VerboseOutput 0 "Extracted albumart" eyeD3 --write-images "$temp/$line-images/" "$temp/$line" 1>/dev/null 2>/dev/null + VerboseOutput 0 "Converting albumart" convert "$temp/$line-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/$line-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null eyeD3 --remove-all-images "$temp/$line" 1>/dev/null 2>/dev/null + VerboseOutput 0 "Embedding albumart" eyeD3 --add-image "$temp/$line-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/$line" 1>/dev/null 2>/dev/null - VerboseOutput 2 "Converted cover art: $line" + VerboseOutput 1 "Converted cover art: $line" fi - + VerboseOutput 2 "Converted: $line" else VerboseOutput 3 "$line already converted" fi; done < "/tmp/music-sync-filelist" + VerboseOutput 2 "Done converting files" } CopyFiles() { @@ -299,6 +339,7 @@ CopyFiles() { VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 2 of 2" if [[ "$dest/$line" = */* ]]; then + VerboseOutput 0 "Creating folder $dest/${line%/*}" mkdir -p "$dest/${line%/*}"; fi; @@ -307,11 +348,13 @@ CopyFiles() { VerboseOutput 2 "Copied: $line" done < "/tmp/music-sync-filelist" + VerboseOutput 2 "Done copying files" } CleanUp() { VerboseOutput 1 "Cleaning Up" if [[ -f /tmp/music-sync-filelist ]]; then + VerboseOutput 1 "Removing filelist" rm "/tmp/music-sync-filelist" fi VerboseOutput 1 "Done" From 99a2ab4784261457322fb0fdcab2369cb978764d Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Mon, 1 Jul 2019 12:34:43 +0200 Subject: [PATCH 02/16] Removing final dots in dest --- music-sync.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/music-sync.sh b/music-sync.sh index 010f41a..0c44a03 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -250,14 +250,14 @@ CreateFileList() { VerboseOutput 0 "Checking ${origfile}" if [[ -d "${1}/$origfile" ]]; then VerboseOutput 0 "${origfile} is folder" - newdir="${3}/$relfile" + newdir="${3}/$origfile" newdir=${newdir#"/"} VerboseOutput 1 "Entering $newdir" CreateFileList "${1}/$origfile" "${2}/$relfile" "$newdir" - elif [[ "${1}/$origfile" != *".m3u" ]] && [[ ! -f "${2}/$relfile" || "${1}/$origfile" -nt "${2}/$relfile" ]]; then - echo ${3}/$relfile >> /tmp/music-sync-filelist + elif [[ "${1}/$origfile" != *".m3u" ]] && [[ ! -f "${2}/$origfile" || "${1}/$origfile" -nt "${2}/$relfile" ]]; then + echo ${3}/$origfile >> /tmp/music-sync-filelist VerboseOutput 0 "${origfile} is newer in source" - VerboseOutput 1 "Added: ${3}/${origfile}" + VerboseOutput 2 "Added: ${3}/${origfile}" fi if [[ "${1}/$origfile" == *".m3u" ]]; then VerboseOutput 0 "${origfile} is playlist" @@ -334,16 +334,16 @@ CopyFiles() { total=$(cat /tmp/music-sync-filelist | wc -l) percentage=$(echo "scale=4;${curline}/${total}" | bc) percentage=$(echo "scale=2;${percentage}*100" | bc) - + destline=$(echo $line | sed -e 's/\.*\//\//g') VerboseOutput 1 "Copying: $line" VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 2 of 2" if [[ "$dest/$line" = */* ]]; then VerboseOutput 0 "Creating folder $dest/${line%/*}" - mkdir -p "$dest/${line%/*}"; + mkdir -p "$dest/${destline%/*}"; fi; - cp -f $temp/$line $dest/$line 1>/dev/null 2>/dev/null + cp -f $temp/$line $dest/$destline 1>/dev/null 2>/dev/null VerboseOutput 2 "Copied: $line" From c50bcb3468aff0338a5432c6056a87d6615458d4 Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Mon, 1 Jul 2019 12:35:01 +0200 Subject: [PATCH 03/16] Add notification about scanning folders --- music-sync.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/music-sync.sh b/music-sync.sh index 0c44a03..7283d25 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -391,6 +391,7 @@ fi if [[ -f /tmp/music-sync-filelist ]]; then rm /tmp/music-sync-filelist fi +VerboseOutput 2 "Scanning for new or updated files" CreateFileList $source $dest "" if [[ ! -f /tmp/music-sync-filelist ]]; then VerboseOutput 2 "Nothing to do!" From dd6dbd898e0540ce294483f0dd0d1bdab686fe27 Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Sat, 8 Feb 2020 13:50:50 +0100 Subject: [PATCH 04/16] Added flac-support --- music-sync.sh | 64 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/music-sync.sh b/music-sync.sh index 7283d25..730d2d6 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -11,6 +11,7 @@ bitrate=192 help=false temp="/tmp/converted" convertart=false +noflac=false coverartsize=200 CheckDeps() { @@ -39,6 +40,17 @@ CheckDeps() { fi VerboseOutput 0 "\`lame --version\` succeeded" + # Check ffmpeg + if [[ $noflac == true && ! $(ffmpeg -h 2>/dev/null) ]]; then + VerboseOutput 5 "\`ffmpeg -h\` failed" + VerboseOutput 5 "Sorry, It seems that ffmpeg is not installed on your system" + VerboseOutput 5 "Please install ffmpeg from your repositories and make sure it is available in your \$PATH" + VerboseOutput 5 "If you don't have any FLAC-files use --no-flac" + ExecTime + exit 1 + fi + VerboseOutput 0 "\`ffmpeg -h\` succeeded" + # Check EyeD3 if [[ $convertart == true && ! $(eyeD3 --version 2>/dev/null) ]]; then VerboseOutput 5 "\`eyeD3 --version\` failed" @@ -67,8 +79,8 @@ CheckDeps() { GetOptions() { # https://stackoverflow.com/a/29754866 - OPTIONS=s:d:t:c::a::v::h - LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help + OPTIONS=s:d:t:c::a::v::hF + LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help,no-flac # -use ! and PIPESTATUS to get exit code with errexit set # -temporarily store output to be able to check for errors @@ -129,6 +141,12 @@ GetOptions() { VerboseOutput 1 "Converted bitrate is ${bitrate}" shift ;; + -F|--no-flac) + VerboseOutput 0 "--no-flac given" + noflac=true + VerboseOutput 1 "FLAC conversion is disabled" + shift + ;; -a|--resize-art) VerboseOutput 0 "--resize-art given" convertart=true @@ -269,9 +287,11 @@ CreateFileList() { ConvertFiles() { curline=0 percentage=0 - while read -r line + IFS=" +"; + for line in $(cat "/tmp/music-sync-filelist") do - + VerboseOutput 0 "${source}/$line" if [[ ! -f "${source}/$line" ]]; then VerboseOutput 5 "Source-file ${source}/$line Unreachable" ExecTime @@ -289,34 +309,42 @@ ConvertFiles() { VerboseOutput 0 "Creating folder $temp/${line%/*}" mkdir -p "$temp/${line%/*}"; fi; - if [[ ! -f "$temp/$line" || "${source}/$line" -nt "$temp/$line" ]]; then - VerboseOutput 0 "Converting MP3-file $temp/${line%/*}" - lame -b ${bitrate} $source/$line $temp/$line 1>/dev/null 2>/dev/null + mp3line=$(echo "$line" | sed -e 's/.flac/.mp3/') + if [[ ! -f "$temp/$mp3line" || "${source}/$mp3line" -nt "$temp/$mp3line" ]]; then + if [[ $line == *".flac" ]]; then + VerboseOutput 0 "Converting FLAC-file $line to mp3" + ffmpeg -y -i "$source/$line" -acodec libmp3lame -map_metadata 0 -id3v2_version 3 -b:a ${bitrate}k "$temp/${mp3line}" 1>/dev/null 2>/dev/null + else + VerboseOutput 0 "Converting MP3-file $line" + lame -b ${bitrate} "$source/$line" "$temp/$mp3line" 1>/dev/null 2>/dev/null + fi if [[ $convertart == true ]]; then - VerboseOutput 0 "Creating folder $temp/$line-images/" - mkdir -p "$temp/$line-images/" + VerboseOutput 0 "Creating folder $temp/${mp3line}-images/" + mkdir -p "$temp/${mp3line}-images/" VerboseOutput 0 "Extracted albumart" - eyeD3 --write-images "$temp/$line-images/" "$temp/$line" 1>/dev/null 2>/dev/null + eyeD3 --write-images "$temp/${mp3line}-images/" "$temp/${mp3line}" 1>/dev/null 2>/dev/null VerboseOutput 0 "Converting albumart" - convert "$temp/$line-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/$line-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null - eyeD3 --remove-all-images "$temp/$line" 1>/dev/null 2>/dev/null + convert "$temp/${mp3line}-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/${mp3line}-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null + eyeD3 --remove-all-images "$temp/${mp3line}" 1>/dev/null 2>/dev/null VerboseOutput 0 "Embedding albumart" - eyeD3 --add-image "$temp/$line-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/$line" 1>/dev/null 2>/dev/null - VerboseOutput 1 "Converted cover art: $line" + eyeD3 --add-image "$temp/${mp3line}-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/${mp3line}" 1>/dev/null 2>/dev/null + VerboseOutput 1 "Converted cover art: ${mp3line}" fi VerboseOutput 2 "Converted: $line" else VerboseOutput 3 "$line already converted" fi; - done < "/tmp/music-sync-filelist" + done VerboseOutput 2 "Done converting files" } CopyFiles() { curline=0 percentage=0 - while read -r line + IFS=" +"; + for line in $(cat "/tmp/music-sync-filelist") do if [[ ! -d "$dest" ]]; then VerboseOutput 5 "Destination unreachable" @@ -343,11 +371,11 @@ CopyFiles() { mkdir -p "$dest/${destline%/*}"; fi; - cp -f $temp/$line $dest/$destline 1>/dev/null 2>/dev/null + cp -f $temp/$line $dest/${destline/.flac/.mp3} 1>/dev/null 2>/dev/null VerboseOutput 2 "Copied: $line" - done < "/tmp/music-sync-filelist" + done VerboseOutput 2 "Done copying files" } From 535a75b04d554b7158d26a7d90ddb8f9974daac0 Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Sun, 9 Feb 2020 14:42:20 +0100 Subject: [PATCH 05/16] Always use ffmpeg --- music-sync.sh | 71 +++++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 39 deletions(-) diff --git a/music-sync.sh b/music-sync.sh index 730d2d6..c2a8cf1 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -11,7 +11,6 @@ bitrate=192 help=false temp="/tmp/converted" convertart=false -noflac=false coverartsize=200 CheckDeps() { @@ -29,23 +28,12 @@ CheckDeps() { fi if [[ $1 == 1 ]]; then - # Check lame - if [[ $convert == true && ! $(lame --version 2>/dev/null) ]]; then - VerboseOutput 5 "\`lame --version\` failed" - VerboseOutput 5 "Sorry, It seems that lame is not installed on your system" - VerboseOutput 5 "Please install lame from your repositories and make sure it is available in your \$PATH" - VerboseOutput 5 "Otherwise disable conversion" - ExecTime - exit 1 - fi - VerboseOutput 0 "\`lame --version\` succeeded" - # Check ffmpeg - if [[ $noflac == true && ! $(ffmpeg -h 2>/dev/null) ]]; then + if [[ $convert == true && ! $(ffmpeg -h 2>/dev/null) ]]; then VerboseOutput 5 "\`ffmpeg -h\` failed" VerboseOutput 5 "Sorry, It seems that ffmpeg is not installed on your system" VerboseOutput 5 "Please install ffmpeg from your repositories and make sure it is available in your \$PATH" - VerboseOutput 5 "If you don't have any FLAC-files use --no-flac" + VerboseOutput 5 "Otherwise disable conversion" ExecTime exit 1 fi @@ -79,8 +67,8 @@ CheckDeps() { GetOptions() { # https://stackoverflow.com/a/29754866 - OPTIONS=s:d:t:c::a::v::hF - LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help,no-flac + OPTIONS=s:d:t:c::a::v::h + LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help # -use ! and PIPESTATUS to get exit code with errexit set # -temporarily store output to be able to check for errors @@ -141,12 +129,6 @@ GetOptions() { VerboseOutput 1 "Converted bitrate is ${bitrate}" shift ;; - -F|--no-flac) - VerboseOutput 0 "--no-flac given" - noflac=true - VerboseOutput 1 "FLAC conversion is disabled" - shift - ;; -a|--resize-art) VerboseOutput 0 "--resize-art given" convertart=true @@ -265,6 +247,7 @@ CreateFileList() { for file in $sourcepath; do origfile="${file#"$1/"}" relfile=$(echo ${origfile} | sed -e 's/\(\.\)*$//g') + mp3file=$(echo "${relfile}" | sed -e 's/.flac/.mp3/') VerboseOutput 0 "Checking ${origfile}" if [[ -d "${1}/$origfile" ]]; then VerboseOutput 0 "${origfile} is folder" @@ -272,10 +255,11 @@ CreateFileList() { newdir=${newdir#"/"} VerboseOutput 1 "Entering $newdir" CreateFileList "${1}/$origfile" "${2}/$relfile" "$newdir" - elif [[ "${1}/$origfile" != *".m3u" ]] && [[ ! -f "${2}/$origfile" || "${1}/$origfile" -nt "${2}/$relfile" ]]; then + elif [[ "${1}/$origfile" != *".m3u" ]] && [[ "${1}/$origfile" -nt "${2}/$mp3file" ]]; then echo ${3}/$origfile >> /tmp/music-sync-filelist VerboseOutput 0 "${origfile} is newer in source" VerboseOutput 2 "Added: ${3}/${origfile}" + echo "${1}/$origfile newer then ${2}/$mp3file" >> /tmp/poot.txt fi if [[ "${1}/$origfile" == *".m3u" ]]; then VerboseOutput 0 "${origfile} is playlist" @@ -311,24 +295,27 @@ ConvertFiles() { fi; mp3line=$(echo "$line" | sed -e 's/.flac/.mp3/') if [[ ! -f "$temp/$mp3line" || "${source}/$mp3line" -nt "$temp/$mp3line" ]]; then - if [[ $line == *".flac" ]]; then - VerboseOutput 0 "Converting FLAC-file $line to mp3" - ffmpeg -y -i "$source/$line" -acodec libmp3lame -map_metadata 0 -id3v2_version 3 -b:a ${bitrate}k "$temp/${mp3line}" 1>/dev/null 2>/dev/null - else - VerboseOutput 0 "Converting MP3-file $line" - lame -b ${bitrate} "$source/$line" "$temp/$mp3line" 1>/dev/null 2>/dev/null + if [[ $line != *".mp3" ]]; then + VerboseOutput 3 "${line} will be converted to mp3-file ${mp3line}" fi + VerboseOutput 0 "Converting MP3-file $line" + ffmpeg -y -i "$source/$line" -acodec libmp3lame -map_metadata 0 -id3v2_version 3 -b:a ${bitrate}k "$temp/${mp3line}" 1>/dev/null 2>/dev/null if [[ $convertart == true ]]; then VerboseOutput 0 "Creating folder $temp/${mp3line}-images/" mkdir -p "$temp/${mp3line}-images/" VerboseOutput 0 "Extracted albumart" eyeD3 --write-images "$temp/${mp3line}-images/" "$temp/${mp3line}" 1>/dev/null 2>/dev/null - VerboseOutput 0 "Converting albumart" - convert "$temp/${mp3line}-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/${mp3line}-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null - eyeD3 --remove-all-images "$temp/${mp3line}" 1>/dev/null 2>/dev/null - VerboseOutput 0 "Embedding albumart" - eyeD3 --add-image "$temp/${mp3line}-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/${mp3line}" 1>/dev/null 2>/dev/null - VerboseOutput 1 "Converted cover art: ${mp3line}" + frontcovers=(${temp}/${mp3line}-images/FRONT_COVER.*) + if [ -e "$frontcovers" ]; then + VerboseOutput 0 "Converting albumart" + convert "$temp/${mp3line}-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/${mp3line}-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null + eyeD3 --remove-all-images "$temp/${mp3line}" 1>/dev/null 2>/dev/null + VerboseOutput 0 "Embedding albumart" + eyeD3 --add-image "$temp/${mp3line}-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/${mp3line}" 1>/dev/null 2>/dev/null + VerboseOutput 1 "Converted cover art: ${mp3line}" + else + VerboseOutput 4 "No front cover art found for ${mp3line}" + fi fi VerboseOutput 2 "Converted: $line" else @@ -352,8 +339,14 @@ CopyFiles() { exit 4 fi - if [[ ! -f "${temp}/$line" ]]; then - VerboseOutput 5 "Source-file ${temp}/$line Unreachable" + if [[ $convert == true ]]; then + mp3line=$(echo "$line" | sed -e 's/.flac/.mp3/') + else + mp3line=$(echo "$line") + fi + + if [[ ! -f "${temp}/$mp3line" ]]; then + VerboseOutput 5 "Source-file ${temp}/$mp3line Unreachable" ExecTime exit 3 fi @@ -362,7 +355,7 @@ CopyFiles() { total=$(cat /tmp/music-sync-filelist | wc -l) percentage=$(echo "scale=4;${curline}/${total}" | bc) percentage=$(echo "scale=2;${percentage}*100" | bc) - destline=$(echo $line | sed -e 's/\.*\//\//g') + destline=$(echo $mp3line | sed -e 's/\.*\//\//g') VerboseOutput 1 "Copying: $line" VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 2 of 2" @@ -371,7 +364,7 @@ CopyFiles() { mkdir -p "$dest/${destline%/*}"; fi; - cp -f $temp/$line $dest/${destline/.flac/.mp3} 1>/dev/null 2>/dev/null + cp -f $temp/$mp3line $dest/${destline} 1>/dev/null 2>/dev/null VerboseOutput 2 "Copied: $line" From 623e83a46d26cd8e294161959f176aca4ff683ac Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Tue, 11 Feb 2020 20:07:57 +0100 Subject: [PATCH 06/16] No debug in prod --- music-sync.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/music-sync.sh b/music-sync.sh index c2a8cf1..89b3fdc 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -259,7 +259,6 @@ CreateFileList() { echo ${3}/$origfile >> /tmp/music-sync-filelist VerboseOutput 0 "${origfile} is newer in source" VerboseOutput 2 "Added: ${3}/${origfile}" - echo "${1}/$origfile newer then ${2}/$mp3file" >> /tmp/poot.txt fi if [[ "${1}/$origfile" == *".m3u" ]]; then VerboseOutput 0 "${origfile} is playlist" From 14f1dd54bd257e1bcef9581143be75fe4c4435a6 Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Wed, 12 Feb 2020 00:13:48 +0100 Subject: [PATCH 07/16] Added multithreading option --- music-sync.sh | 237 +++++++++++++++++++++++++++++--------------------- 1 file changed, 137 insertions(+), 100 deletions(-) diff --git a/music-sync.sh b/music-sync.sh index 89b3fdc..b34b709 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -12,6 +12,8 @@ help=false temp="/tmp/converted" convertart=false coverartsize=200 +jobcount="-2" +multithread=false CheckDeps() { if [[ $1 == 0 ]]; then @@ -60,6 +62,17 @@ CheckDeps() { exit 1 fi VerboseOutput 0 "\`convert --version\` succeeded" + + # Check Gnu parallel + if [[ $multithread == true && ! $(parallel -h 2>/dev/null) ]]; then + VerboseOutput 5 "\`parallel -h\` failed" + VerboseOutput 5 "Sorry, It seems that parallel is not installed on your system" + VerboseOutput 5 "Please install gnu-parallel and env_parallel from your repositories and make sure it is available in your \$PATH" + VerboseOutput 5 "Otherwise disable multithreading" + ExecTime + exit 1 + fi + VerboseOutput 0 "\`parallel -h \` succeeded" fi VerboseOutput 1 "Dependency test OK" } @@ -67,8 +80,8 @@ CheckDeps() { GetOptions() { # https://stackoverflow.com/a/29754866 - OPTIONS=s:d:t:c::a::v::h - LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help + OPTIONS=s:d:t:c::a::v::hj:: + LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help,jobs:: # -use ! and PIPESTATUS to get exit code with errexit set # -temporarily store output to be able to check for errors @@ -140,6 +153,17 @@ GetOptions() { shift VerboseOutput 1 "Album art will ${coverartsize}px wide" ;; + -j|--jobs) + VerboseOutput 0 "--jobs given" + multithread=true + jobcount="-2" + if [[ $2 != "" ]]; then + jobcount=${2} + shift + fi + shift + VerboseOutput 1 "multithreading will use ${jobcount} threads" + ;; --) shift break @@ -170,7 +194,6 @@ GetOptions() { if [[ $dest == "-" ]] || [[ $source == "-" ]]; then help=true fi - VerboseOutput 1 "Checks OK. Going on" } @@ -267,109 +290,125 @@ CreateFileList() { done } -ConvertFiles() { +Execute() { curline=0 percentage=0 IFS=" "; - for line in $(cat "/tmp/music-sync-filelist") - do - VerboseOutput 0 "${source}/$line" - if [[ ! -f "${source}/$line" ]]; then - VerboseOutput 5 "Source-file ${source}/$line Unreachable" - ExecTime - exit 3 + if [[ $multithread == true ]]; then + VerboseOutput 3 "Running in parallel. Mind your load" + export -f ProcessFile + export -f ConvertFile + export -f CopyFile + export -f VerboseOutput + export -f ExecTime + export dest + export source + export bitrate + export convert + export temp + export coverartsize + export begin + parallel --jobs ${jobcount} --line-buffer --arg-file "/tmp/music-sync-filelist" ProcessFile + else + for line in $(cat "/tmp/music-sync-filelist") + do + + curline=$(expr ${curline} + 1) + total=$(cat /tmp/music-sync-filelist | wc -l) + percentage=$(echo "scale=4;${curline}/${total}" | bc) + percentage=$(echo "scale=2;${percentage}*100" | bc) + VerboseOutput 2 "Current file: $line" + VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%)" + ProcessFile $line + done + fi + VerboseOutput 2 "Done!" +} +ProcessFile() { + line=${1} + VerboseOutput 2 "Current File: $line" + if [[ $convert == true ]]; then + ConvertFile "$line" + fi + CopyFile "$line" +} + +ConvertFile() { + line=${1} + if [[ ! -f "${source}/$line" ]]; then + VerboseOutput 5 "Source-file ${source}/$line Unreachable" + ExecTime + exit 3 + fi + + if [[ "$temp/$line" = */* ]]; then + VerboseOutput 0 "Creating folder $temp/${line%/*}" + mkdir -p "$temp/${line%/*}"; + fi; + mp3line=$(echo "$line" | sed -e 's/.flac/.mp3/') + if [[ ! -f "$temp/$mp3line" || "${source}/$mp3line" -nt "$temp/$mp3line" ]]; then + if [[ $line != *".mp3" ]]; then + VerboseOutput 3 "${line} will be converted to mp3-file ${mp3line}" fi + VerboseOutput 0 "Converting MP3-file $line" + ffmpeg -y -i "$source/$line" -acodec libmp3lame -map_metadata 0 -id3v2_version 3 -b:a ${bitrate}k "$temp/${mp3line}" 1>/dev/null 2>/dev/null + if [[ $convertart == true ]]; then + VerboseOutput 0 "Creating folder $temp/${mp3line}-images/" + mkdir -p "$temp/${mp3line}-images/" + VerboseOutput 0 "Extracted albumart" + eyeD3 --write-images "$temp/${mp3line}-images/" "$temp/${mp3line}" 1>/dev/null 2>/dev/null + frontcovers=(${temp}/${mp3line}-images/FRONT_COVER.*) + if [ -e "$frontcovers" ]; then + VerboseOutput 0 "Converting albumart" + convert "$temp/${mp3line}-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/${mp3line}-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null + eyeD3 --remove-all-images "$temp/${mp3line}" 1>/dev/null 2>/dev/null + VerboseOutput 0 "Embedding albumart" + eyeD3 --add-image "$temp/${mp3line}-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/${mp3line}" 1>/dev/null 2>/dev/null + VerboseOutput 1 "Converted cover art: ${mp3line}" + else + VerboseOutput 4 "No front cover art found for ${mp3line}" + fi + fi + VerboseOutput 1 "Converted: $line" + else + VerboseOutput 3 "$line already converted" + fi; +} - curline=$(expr ${curline} + 1) - total=$(cat /tmp/music-sync-filelist | wc -l) - percentage=$(echo "scale=4;${curline}/${total}" | bc) - percentage=$(echo "scale=2;${percentage}*100" | bc) - VerboseOutput 1 "Converting: $line" - VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 1 of 2" +CopyFile() { + line=${1} + if [[ ! -d "$dest" ]]; then + VerboseOutput 5 "Destination unreachable" + ExecTime + exit 4 + fi - if [[ "$temp/$line" = */* ]]; then - VerboseOutput 0 "Creating folder $temp/${line%/*}" - mkdir -p "$temp/${line%/*}"; - fi; + if [[ $convert == true ]]; then mp3line=$(echo "$line" | sed -e 's/.flac/.mp3/') - if [[ ! -f "$temp/$mp3line" || "${source}/$mp3line" -nt "$temp/$mp3line" ]]; then - if [[ $line != *".mp3" ]]; then - VerboseOutput 3 "${line} will be converted to mp3-file ${mp3line}" - fi - VerboseOutput 0 "Converting MP3-file $line" - ffmpeg -y -i "$source/$line" -acodec libmp3lame -map_metadata 0 -id3v2_version 3 -b:a ${bitrate}k "$temp/${mp3line}" 1>/dev/null 2>/dev/null - if [[ $convertart == true ]]; then - VerboseOutput 0 "Creating folder $temp/${mp3line}-images/" - mkdir -p "$temp/${mp3line}-images/" - VerboseOutput 0 "Extracted albumart" - eyeD3 --write-images "$temp/${mp3line}-images/" "$temp/${mp3line}" 1>/dev/null 2>/dev/null - frontcovers=(${temp}/${mp3line}-images/FRONT_COVER.*) - if [ -e "$frontcovers" ]; then - VerboseOutput 0 "Converting albumart" - convert "$temp/${mp3line}-images/FRONT_COVER.*" -resize ${coverartsize}x${coverartsize} "$temp/${mp3line}-images/FRONT_COVER.jpg" 1>/dev/null 2>/dev/null - eyeD3 --remove-all-images "$temp/${mp3line}" 1>/dev/null 2>/dev/null - VerboseOutput 0 "Embedding albumart" - eyeD3 --add-image "$temp/${mp3line}-images/FRONT_COVER.jpg:FRONT_COVER" "$temp/${mp3line}" 1>/dev/null 2>/dev/null - VerboseOutput 1 "Converted cover art: ${mp3line}" - else - VerboseOutput 4 "No front cover art found for ${mp3line}" - fi - fi - VerboseOutput 2 "Converted: $line" - else - VerboseOutput 3 "$line already converted" - fi; + else + mp3line=$(echo "$line") + fi - done - VerboseOutput 2 "Done converting files" + if [[ ! -f "${temp}/$mp3line" ]]; then + VerboseOutput 5 "Source-file ${temp}/$mp3line Unreachable" + ExecTime + exit 3 + fi + + destline=$(echo $mp3line | sed -e 's/\.*\//\//g') + VerboseOutput 1 "Copying: $line" + + if [[ "$dest/$line" = */* ]]; then + VerboseOutput 0 "Creating folder $dest/${line%/*}" + mkdir -p "$dest/${destline%/*}"; + fi; + + cp -f $temp/$mp3line $dest/${destline} 1>/dev/null 2>/dev/null + + VerboseOutput 1 "Copied: $line" } -CopyFiles() { - curline=0 - percentage=0 - IFS=" -"; - for line in $(cat "/tmp/music-sync-filelist") - do - if [[ ! -d "$dest" ]]; then - VerboseOutput 5 "Destination unreachable" - ExecTime - exit 4 - fi - - if [[ $convert == true ]]; then - mp3line=$(echo "$line" | sed -e 's/.flac/.mp3/') - else - mp3line=$(echo "$line") - fi - - if [[ ! -f "${temp}/$mp3line" ]]; then - VerboseOutput 5 "Source-file ${temp}/$mp3line Unreachable" - ExecTime - exit 3 - fi - - curline=$(expr ${curline} + 1) - total=$(cat /tmp/music-sync-filelist | wc -l) - percentage=$(echo "scale=4;${curline}/${total}" | bc) - percentage=$(echo "scale=2;${percentage}*100" | bc) - destline=$(echo $mp3line | sed -e 's/\.*\//\//g') - VerboseOutput 1 "Copying: $line" - VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 2 of 2" - - if [[ "$dest/$line" = */* ]]; then - VerboseOutput 0 "Creating folder $dest/${line%/*}" - mkdir -p "$dest/${destline%/*}"; - fi; - - cp -f $temp/$mp3line $dest/${destline} 1>/dev/null 2>/dev/null - - VerboseOutput 2 "Copied: $line" - - done - VerboseOutput 2 "Done copying files" -} CleanUp() { VerboseOutput 1 "Cleaning Up" @@ -418,14 +457,12 @@ if [[ ! -f /tmp/music-sync-filelist ]]; then CleanUp exit 0 fi -if [[ $convert == true ]]; then - ConvertFiles -else +if [[ $convert == false ]]; then if [[ $temp != "/tmp/converted" ]]; then VerboseOutput 2 "Conversion not enabled. Ignoring cache folder" fi temp=$source -fi -CopyFiles +fi +Execute CleanUp ExecTime From b2f66fc0570932e8c096e045967ec359275515e5 Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Wed, 12 Feb 2020 00:21:50 +0100 Subject: [PATCH 08/16] Updated docs --- music-sync.sh | 69 +++++++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/music-sync.sh b/music-sync.sh index b34b709..e685ac3 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -1,4 +1,38 @@ #!/bin/bash +# +#/ +#/ Usage: +#/ music-sync -s|--source -d|--dest +#/ music-sync +#/ +#/ Syncronises music from one folder to another. +#/ +#/ Options: +#/ -s, --source The source folder of the music +#/ -d, --dest The destination folder of the music +#/ -t, --temp The temporary cache for converted files (default: /tmp/converted) +#/ -c, --convert Convert files to a given bitrate in kbps before syncing (default: 192) +#/ -a, --resize-art Resize album-art before syncing (default width: 200) +#/ -j, --jobs Number of processes to use in multi-threading (default: CPU_CORES - 2) +#/ -v, --verbose <0-6> Set log level (default: 2) +#/ -h, --help Display this help text +#/ +#/ Log levels: +#/ 0 | Verbose +#/ 1 | Debug +#/ 2 | Info +#/ 3 | Warning +#/ 4 | Error +#/ 5 | Fatal +#/ 6 | Silence +#/ +#/ Exit Codes: +#/ 1 Dependencies not met +#/ 2 Invalid Argument +#/ 3 Source Unreachable +#/ 4 Destination Unreachable +#/ 5 Command failed +#/ begin=$(date +"%s") # saner programming env: these switches turn some bugs into errors @@ -14,6 +48,8 @@ convertart=false coverartsize=200 jobcount="-2" multithread=false +script_name=$(basename "${0}") +script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) CheckDeps() { if [[ $1 == 0 ]]; then @@ -198,38 +234,7 @@ GetOptions() { } Usage() { - echo "" - echo "Usage:" - echo "music-sync -s|--source -d|--dest " - echo "music-sync " - echo "" - echo "Syncronises music from one folder to another." - echo "" - echo "Options:" - echo " -s, --source The source folder of the music" - echo " -d, --dest The destination folder of the music" - echo " -t, --temp The temporary cache for converted files (default: /tmp/converted)" - echo " -c, --convert Convert files to a given bitrate in kbps before syncing (default: 192)" - echo " -a, --resize-art Resize album-art before syncing (default width: 200)" - echo " -v, --verbose <0-6> Set log level (default: 2)" - echo " -h, --help Display this help text" - echo "" - echo "Log levels:" - echo " 0 | Verbose" - echo " 1 | Debug" - echo " 2 | Info" - echo " 3 | Warning" - echo " 4 | Error" - echo " 5 | Fatal" - echo " 6 | Silence" - echo "" - echo "Exit Codes:" - echo " 1 Dependencies not met" - echo " 2 Invalid Argument" - echo " 3 Source Unreachable" - echo " 4 Destination Unreachable" - echo " 5 Command failed" - echo "" + grep '^#/' "${script_dir}/${script_name}" | sed 's/^#\/\w*//' } VerboseOutput() { From 9f617b3c55491b8362c1d79522c6a0f18c93e3e5 Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Wed, 12 Feb 2020 00:24:57 +0100 Subject: [PATCH 09/16] Exporting verbose --- music-sync.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/music-sync.sh b/music-sync.sh index e685ac3..19e2041 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -312,6 +312,7 @@ Execute() { export bitrate export convert export temp + export verbose export coverartsize export begin parallel --jobs ${jobcount} --line-buffer --arg-file "/tmp/music-sync-filelist" ProcessFile From c6be37116d9fc8fd3fd0cc7846d942005061efb4 Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Wed, 12 Feb 2020 00:38:34 +0100 Subject: [PATCH 10/16] Fixing quotes --- music-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/music-sync.sh b/music-sync.sh index 19e2041..38beaf9 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -410,7 +410,7 @@ CopyFile() { mkdir -p "$dest/${destline%/*}"; fi; - cp -f $temp/$mp3line $dest/${destline} 1>/dev/null 2>/dev/null + cp -f "$temp/$mp3line" "$dest/${destline}" 1>/dev/null 2>/dev/null VerboseOutput 1 "Copied: $line" } From fa23c4d99206c1f64b0e6fafae0ce3dbe95114be Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Wed, 12 Feb 2020 13:09:56 +0100 Subject: [PATCH 11/16] Fixed double spaces --- music-sync.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/music-sync.sh b/music-sync.sh index 38beaf9..ea0462f 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -274,7 +274,7 @@ CreateFileList() { sourcepath="${sourcepath/\]/\\\]}" for file in $sourcepath; do origfile="${file#"$1/"}" - relfile=$(echo ${origfile} | sed -e 's/\(\.\)*$//g') + relfile=$(echo ${origfile} | sed -e 's/\(\.\)*$//g' | tr -s ' ') mp3file=$(echo "${relfile}" | sed -e 's/.flac/.mp3/') VerboseOutput 0 "Checking ${origfile}" if [[ -d "${1}/$origfile" ]]; then @@ -402,7 +402,7 @@ CopyFile() { exit 3 fi - destline=$(echo $mp3line | sed -e 's/\.*\//\//g') + destline=$(echo $mp3line | sed -e 's/\.*\//\//g'| tr -s ' ') VerboseOutput 1 "Copying: $line" if [[ "$dest/$line" = */* ]]; then @@ -410,7 +410,7 @@ CopyFile() { mkdir -p "$dest/${destline%/*}"; fi; - cp -f "$temp/$mp3line" "$dest/${destline}" 1>/dev/null 2>/dev/null + cp -fv "$temp/$mp3line" "$dest/${destline}" 1>/dev/null 2>/dev/null VerboseOutput 1 "Copied: $line" } From 1d5c6b45ffc31874e552edf42e5aeda872f7d1f4 Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Wed, 12 Feb 2020 13:32:48 +0100 Subject: [PATCH 12/16] Updated readme --- README.MD | 56 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/README.MD b/README.MD index cb0869a..89b6a22 100644 --- a/README.MD +++ b/README.MD @@ -1,45 +1,47 @@ # Bash Music sync -(c) 2018-2019 Jeroen De Meerleer +(c) 2018-2020 Jeroen De Meerleer Script to sync music from one folder to another ## Usage: ``` -music-sync -s|--source -d|--dest -music-sync + Syncronises music from one folder to another. -Syncronises music from one folder to another. + Usage: + music-sync -s|--source -d|--dest + music-sync -Options: - -s, --source The source folder of the music - -d, --dest The destination folder of the music - -t, --temp The temporary cache for converted files (default: /tmp/converted) - -c, --convert Convert files to a given bitrate in kbps before syncing (default: 192) - -a, --resize-art Resize album-art before syncing (default width: 200) - -v, --verbose <0-6> Set log level (default: 2) - -h, --help Display this help text + Options: + -s, --source The source folder of the music + -d, --dest The destination folder of the music + -t, --temp The temporary cache for converted files (default: /tmp/converted) + -c, --convert Convert files to a given bitrate in kbps before syncing (default: 192) + -a, --resize-art Resize album-art before syncing (default width: 200) + -j, --jobs Number of processes to use in multi-threading (default: CPU_CORES - 2) + -v, --verbose <0-6> Set log level (default: 2) + -h, --help Display this help text -Log levels: - 0 | Verbose - 1 | Debug - 2 | Info - 3 | Warning - 4 | Error - 5 | Fatal - 6 | Silence + Log levels: + 0 | Verbose + 1 | Debug + 2 | Info + 3 | Warning + 4 | Error + 5 | Fatal + 6 | Silence -Exit Codes: - 1 Dependencies not met - 2 Invalid Argument - 3 Source Unreachable - 4 Destination Unreachable - 5 Command failed + Exit Codes: + 1 Dependencies not met + 2 Invalid Argument + 3 Source Unreachable + 4 Destination Unreachable + 5 Command failed ``` ## Licence -Copyright 2018-2019 Jeroen De Meerleer +Copyright 2018-2020 Jeroen De Meerleer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: From ad724374025947e7245b5432eefde86f6fb74a48 Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Wed, 12 Feb 2020 13:53:58 +0100 Subject: [PATCH 13/16] Parameter handling --- music-sync.sh | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/music-sync.sh b/music-sync.sh index ea0462f..597202f 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -141,9 +141,8 @@ GetOptions() { verbose=2 if [[ $2 != "" ]]; then verbose=${2} - shift fi - shift + shift 2 ;; -h|--help) help=true @@ -173,10 +172,9 @@ GetOptions() { bitrate=192 if [[ $2 != "" ]]; then bitrate=${2} - shift fi VerboseOutput 1 "Converted bitrate is ${bitrate}" - shift + shift 2 ;; -a|--resize-art) VerboseOutput 0 "--resize-art given" @@ -184,9 +182,8 @@ GetOptions() { coverartsize=200 if [[ $2 != "" ]]; then coverartsize=${2} - shift fi - shift + shift 2 VerboseOutput 1 "Album art will ${coverartsize}px wide" ;; -j|--jobs) @@ -195,9 +192,8 @@ GetOptions() { jobcount="-2" if [[ $2 != "" ]]; then jobcount=${2} - shift fi - shift + shift 2 VerboseOutput 1 "multithreading will use ${jobcount} threads" ;; --) From 53a732dd29a87a9a165886765fc0199f13520a91 Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Wed, 12 Feb 2020 13:58:57 +0100 Subject: [PATCH 14/16] Better way to calculate threads --- README.MD | 2 +- music-sync.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.MD b/README.MD index 89b6a22..a38b1d0 100644 --- a/README.MD +++ b/README.MD @@ -18,7 +18,7 @@ Script to sync music from one folder to another -t, --temp The temporary cache for converted files (default: /tmp/converted) -c, --convert Convert files to a given bitrate in kbps before syncing (default: 192) -a, --resize-art Resize album-art before syncing (default width: 200) - -j, --jobs Number of processes to use in multi-threading (default: CPU_CORES - 2) + -j, --jobs Number of processes to use in multi-threading (default: nproc - 2) -v, --verbose <0-6> Set log level (default: 2) -h, --help Display this help text diff --git a/music-sync.sh b/music-sync.sh index 597202f..cc494b2 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -13,7 +13,7 @@ #/ -t, --temp The temporary cache for converted files (default: /tmp/converted) #/ -c, --convert Convert files to a given bitrate in kbps before syncing (default: 192) #/ -a, --resize-art Resize album-art before syncing (default width: 200) -#/ -j, --jobs Number of processes to use in multi-threading (default: CPU_CORES - 2) +#/ -j, --jobs Number of processes to use in multi-threading (default: nproc - 2) #/ -v, --verbose <0-6> Set log level (default: 2) #/ -h, --help Display this help text #/ @@ -46,7 +46,7 @@ help=false temp="/tmp/converted" convertart=false coverartsize=200 -jobcount="-2" +jobcount=$(expr $(nproc) - 2) multithread=false script_name=$(basename "${0}") script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) @@ -189,12 +189,12 @@ GetOptions() { -j|--jobs) VerboseOutput 0 "--jobs given" multithread=true - jobcount="-2" + jobcount=$(expr $(nproc) - 2) if [[ $2 != "" ]]; then jobcount=${2} fi shift 2 - VerboseOutput 1 "multithreading will use ${jobcount} threads" + VerboseOutput 1 "Multithreading will use ${jobcount} threads" ;; --) shift From 2f39dc88e4f5bad789fd0cfa7ca30ef2983cb0b7 Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Wed, 12 Feb 2020 18:54:28 +0100 Subject: [PATCH 15/16] Don't say you checked deps when you don't need them I will not lie --- music-sync.sh | 76 ++++++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/music-sync.sh b/music-sync.sh index cc494b2..0e26cab 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -52,7 +52,7 @@ script_name=$(basename "${0}") script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) CheckDeps() { - if [[ $1 == 0 ]]; then + if [[ $1 == 2 ]]; then # Check getopt ! getopt --test > /dev/null if [[ ${PIPESTATUS[0]} -ne 4 ]]; then @@ -67,48 +67,56 @@ CheckDeps() { if [[ $1 == 1 ]]; then # Check ffmpeg - if [[ $convert == true && ! $(ffmpeg -h 2>/dev/null) ]]; then - VerboseOutput 5 "\`ffmpeg -h\` failed" - VerboseOutput 5 "Sorry, It seems that ffmpeg is not installed on your system" - VerboseOutput 5 "Please install ffmpeg from your repositories and make sure it is available in your \$PATH" - VerboseOutput 5 "Otherwise disable conversion" - ExecTime - exit 1 + if [[ $convert == true ]]; then + if [[ ! $(ffmpeg -h 2>/dev/null) ]]; then + VerboseOutput 5 "\`ffmpeg -h\` failed" + VerboseOutput 5 "Sorry, It seems that ffmpeg is not installed on your system" + VerboseOutput 5 "Please install ffmpeg from your repositories and make sure it is available in your \$PATH" + VerboseOutput 5 "Otherwise disable conversion" + ExecTime + exit 1 + fi + VerboseOutput 0 "\`ffmpeg -h\` succeeded" fi - VerboseOutput 0 "\`ffmpeg -h\` succeeded" # Check EyeD3 - if [[ $convertart == true && ! $(eyeD3 --version 2>/dev/null) ]]; then - VerboseOutput 5 "\`eyeD3 --version\` failed" - VerboseOutput 5 "Sorry, It seems that eyeD3 is not installed on your system" - VerboseOutput 5 "Please install eyeD3 from your repositories and make sure it is available in your \$PATH" - VerboseOutput 5 "Otherwise disable albumart conversion" - ExecTime - exit 1 + if [[ $convertart == true ]]; then + if [[ ! $(eyeD3 --version 2>/dev/null) ]]; then + VerboseOutput 5 "\`eyeD3 --version\` failed" + VerboseOutput 5 "Sorry, It seems that eyeD3 is not installed on your system" + VerboseOutput 5 "Please install eyeD3 from your repositories and make sure it is available in your \$PATH" + VerboseOutput 5 "Otherwise disable albumart conversion" + ExecTime + exit 1 + fi + VerboseOutput 0 "\`eyeD3 --version\` succeeded" fi - VerboseOutput 0 "\`eyeD3 --version\` succeeded" # Check ImageMagick - if [[ $convertart == true && ! $(convert --version 2>/dev/null) ]]; then - VerboseOutput 5 "\`convert --version\` failed" - VerboseOutput 5 "Sorry, It seems that ImageMagick is not installed on your system" - VerboseOutput 5 "Please install ImageMagick from your repositories and make sure it is available in your \$PATH" - VerboseOutput 5 "Otherwise disable albumart conversion" - ExecTime - exit 1 + if [[ $convertart == true ]]; then + if [[ ! $(convert --version 2>/dev/null) ]]; then + VerboseOutput 5 "\`convert --version\` failed" + VerboseOutput 5 "Sorry, It seems that ImageMagick is not installed on your system" + VerboseOutput 5 "Please install ImageMagick from your repositories and make sure it is available in your \$PATH" + VerboseOutput 5 "Otherwise disable albumart conversion" + ExecTime + exit 1 + fi + VerboseOutput 0 "\`convert --version\` succeeded" fi - VerboseOutput 0 "\`convert --version\` succeeded" # Check Gnu parallel - if [[ $multithread == true && ! $(parallel -h 2>/dev/null) ]]; then - VerboseOutput 5 "\`parallel -h\` failed" - VerboseOutput 5 "Sorry, It seems that parallel is not installed on your system" - VerboseOutput 5 "Please install gnu-parallel and env_parallel from your repositories and make sure it is available in your \$PATH" - VerboseOutput 5 "Otherwise disable multithreading" - ExecTime - exit 1 + if [[ $multithread == true ]]; then + if [[ ! $(parallel -h 2>/dev/null) ]]; then + VerboseOutput 5 "\`parallel -h\` failed" + VerboseOutput 5 "Sorry, It seems that parallel is not installed on your system" + VerboseOutput 5 "Please install gnu-parallel and env_parallel from your repositories and make sure it is available in your \$PATH" + VerboseOutput 5 "Otherwise disable multithreading" + ExecTime + exit 1 + fi + VerboseOutput 0 "\`parallel -h \` succeeded" fi - VerboseOutput 0 "\`parallel -h \` succeeded" fi VerboseOutput 1 "Dependency test OK" } @@ -442,7 +450,7 @@ ExitHandler() { trap 'ExitHandler' SIGINT trap 'ErrorHandler $BASH_COMMAND' ERR -CheckDeps 0 +CheckDeps 2 GetOptions $@ CheckDeps 1 if [[ $help == true ]]; then From 9e2b515bed91d95cf2d6b2bb1449cb263ea09ae2 Mon Sep 17 00:00:00 2001 From: Jeroen De Meerleer Date: Wed, 12 Feb 2020 18:55:04 +0100 Subject: [PATCH 16/16] Added progress to the parallel output --- music-sync.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/music-sync.sh b/music-sync.sh index 0e26cab..8b3cc36 100755 --- a/music-sync.sh +++ b/music-sync.sh @@ -319,11 +319,10 @@ Execute() { export verbose export coverartsize export begin - parallel --jobs ${jobcount} --line-buffer --arg-file "/tmp/music-sync-filelist" ProcessFile + parallel --jobs ${jobcount} --will-cite --line-buffer --arg-file "/tmp/music-sync-filelist" ProcessFile else for line in $(cat "/tmp/music-sync-filelist") do - curline=$(expr ${curline} + 1) total=$(cat /tmp/music-sync-filelist | wc -l) percentage=$(echo "scale=4;${curline}/${total}" | bc) @@ -337,7 +336,12 @@ Execute() { } ProcessFile() { line=${1} + curline="$(grep -n "$line" /tmp/music-sync-filelist | head -n 1 | cut -d: -f1)" + total=$(cat /tmp/music-sync-filelist | wc -l) + percentage=$(echo "scale=4;${curline}/${total}" | bc) + percentage=$(echo "scale=2;${percentage}*100" | bc) VerboseOutput 2 "Current File: $line" + VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%)" if [[ $convert == true ]]; then ConvertFile "$line" fi