Added multithreading option

This commit is contained in:
Jeroen De Meerleer 2020-02-12 00:13:48 +01:00
parent 623e83a46d
commit 14f1dd54bd

View File

@ -12,6 +12,8 @@ help=false
temp="/tmp/converted" temp="/tmp/converted"
convertart=false convertart=false
coverartsize=200 coverartsize=200
jobcount="-2"
multithread=false
CheckDeps() { CheckDeps() {
if [[ $1 == 0 ]]; then if [[ $1 == 0 ]]; then
@ -60,6 +62,17 @@ CheckDeps() {
exit 1 exit 1
fi fi
VerboseOutput 0 "\`convert --version\` succeeded" VerboseOutput 0 "\`convert --version\` succeeded"
# Check Gnu parallel
if [[ $multithread == true && ! $(parallel -h 2>/dev/null) ]]; then
VerboseOutput 5 "\`parallel -h\` failed"
VerboseOutput 5 "Sorry, It seems that parallel is not installed on your system"
VerboseOutput 5 "Please install gnu-parallel and env_parallel from your repositories and make sure it is available in your \$PATH"
VerboseOutput 5 "Otherwise disable multithreading"
ExecTime
exit 1
fi
VerboseOutput 0 "\`parallel -h \` succeeded"
fi fi
VerboseOutput 1 "Dependency test OK" VerboseOutput 1 "Dependency test OK"
} }
@ -67,8 +80,8 @@ CheckDeps() {
GetOptions() { GetOptions() {
# https://stackoverflow.com/a/29754866 # https://stackoverflow.com/a/29754866
OPTIONS=s:d:t:c::a::v::h OPTIONS=s:d:t:c::a::v::hj::
LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help LONGOPTS=source:,dest:,temp:,convert::,resize-art::,verbose::,help,jobs::
# -use ! and PIPESTATUS to get exit code with errexit set # -use ! and PIPESTATUS to get exit code with errexit set
# -temporarily store output to be able to check for errors # -temporarily store output to be able to check for errors
@ -140,6 +153,17 @@ GetOptions() {
shift shift
VerboseOutput 1 "Album art will ${coverartsize}px wide" VerboseOutput 1 "Album art will ${coverartsize}px wide"
;; ;;
-j|--jobs)
VerboseOutput 0 "--jobs given"
multithread=true
jobcount="-2"
if [[ $2 != "" ]]; then
jobcount=${2}
shift
fi
shift
VerboseOutput 1 "multithreading will use ${jobcount} threads"
;;
--) --)
shift shift
break break
@ -170,7 +194,6 @@ GetOptions() {
if [[ $dest == "-" ]] || [[ $source == "-" ]]; then if [[ $dest == "-" ]] || [[ $source == "-" ]]; then
help=true help=true
fi fi
VerboseOutput 1 "Checks OK. Going on" VerboseOutput 1 "Checks OK. Going on"
} }
@ -267,26 +290,57 @@ CreateFileList() {
done done
} }
ConvertFiles() { Execute() {
curline=0 curline=0
percentage=0 percentage=0
IFS=" IFS="
"; ";
if [[ $multithread == true ]]; then
VerboseOutput 3 "Running in parallel. Mind your load"
export -f ProcessFile
export -f ConvertFile
export -f CopyFile
export -f VerboseOutput
export -f ExecTime
export dest
export source
export bitrate
export convert
export temp
export coverartsize
export begin
parallel --jobs ${jobcount} --line-buffer --arg-file "/tmp/music-sync-filelist" ProcessFile
else
for line in $(cat "/tmp/music-sync-filelist") for line in $(cat "/tmp/music-sync-filelist")
do do
VerboseOutput 0 "${source}/$line"
if [[ ! -f "${source}/$line" ]]; then
VerboseOutput 5 "Source-file ${source}/$line Unreachable"
ExecTime
exit 3
fi
curline=$(expr ${curline} + 1) curline=$(expr ${curline} + 1)
total=$(cat /tmp/music-sync-filelist | wc -l) total=$(cat /tmp/music-sync-filelist | wc -l)
percentage=$(echo "scale=4;${curline}/${total}" | bc) percentage=$(echo "scale=4;${curline}/${total}" | bc)
percentage=$(echo "scale=2;${percentage}*100" | bc) percentage=$(echo "scale=2;${percentage}*100" | bc)
VerboseOutput 1 "Converting: $line" VerboseOutput 2 "Current file: $line"
VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 1 of 2" VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%)"
ProcessFile $line
done
fi
VerboseOutput 2 "Done!"
}
ProcessFile() {
line=${1}
VerboseOutput 2 "Current File: $line"
if [[ $convert == true ]]; then
ConvertFile "$line"
fi
CopyFile "$line"
}
ConvertFile() {
line=${1}
if [[ ! -f "${source}/$line" ]]; then
VerboseOutput 5 "Source-file ${source}/$line Unreachable"
ExecTime
exit 3
fi
if [[ "$temp/$line" = */* ]]; then if [[ "$temp/$line" = */* ]]; then
VerboseOutput 0 "Creating folder $temp/${line%/*}" VerboseOutput 0 "Creating folder $temp/${line%/*}"
@ -316,22 +370,14 @@ ConvertFiles() {
VerboseOutput 4 "No front cover art found for ${mp3line}" VerboseOutput 4 "No front cover art found for ${mp3line}"
fi fi
fi fi
VerboseOutput 2 "Converted: $line" VerboseOutput 1 "Converted: $line"
else else
VerboseOutput 3 "$line already converted" VerboseOutput 3 "$line already converted"
fi; fi;
done
VerboseOutput 2 "Done converting files"
} }
CopyFiles() { CopyFile() {
curline=0 line=${1}
percentage=0
IFS="
";
for line in $(cat "/tmp/music-sync-filelist")
do
if [[ ! -d "$dest" ]]; then if [[ ! -d "$dest" ]]; then
VerboseOutput 5 "Destination unreachable" VerboseOutput 5 "Destination unreachable"
ExecTime ExecTime
@ -350,13 +396,8 @@ CopyFiles() {
exit 3 exit 3
fi fi
curline=$(expr ${curline} + 1)
total=$(cat /tmp/music-sync-filelist | wc -l)
percentage=$(echo "scale=4;${curline}/${total}" | bc)
percentage=$(echo "scale=2;${percentage}*100" | bc)
destline=$(echo $mp3line | sed -e 's/\.*\//\//g') destline=$(echo $mp3line | sed -e 's/\.*\//\//g')
VerboseOutput 1 "Copying: $line" VerboseOutput 1 "Copying: $line"
VerboseOutput 2 "Progress: $curline / $total (${percentage%00}%) Step 2 of 2"
if [[ "$dest/$line" = */* ]]; then if [[ "$dest/$line" = */* ]]; then
VerboseOutput 0 "Creating folder $dest/${line%/*}" VerboseOutput 0 "Creating folder $dest/${line%/*}"
@ -365,12 +406,10 @@ CopyFiles() {
cp -f $temp/$mp3line $dest/${destline} 1>/dev/null 2>/dev/null cp -f $temp/$mp3line $dest/${destline} 1>/dev/null 2>/dev/null
VerboseOutput 2 "Copied: $line" VerboseOutput 1 "Copied: $line"
done
VerboseOutput 2 "Done copying files"
} }
CleanUp() { CleanUp() {
VerboseOutput 1 "Cleaning Up" VerboseOutput 1 "Cleaning Up"
if [[ -f /tmp/music-sync-filelist ]]; then if [[ -f /tmp/music-sync-filelist ]]; then
@ -418,14 +457,12 @@ if [[ ! -f /tmp/music-sync-filelist ]]; then
CleanUp CleanUp
exit 0 exit 0
fi fi
if [[ $convert == true ]]; then if [[ $convert == false ]]; then
ConvertFiles
else
if [[ $temp != "/tmp/converted" ]]; then if [[ $temp != "/tmp/converted" ]]; then
VerboseOutput 2 "Conversion not enabled. Ignoring cache folder" VerboseOutput 2 "Conversion not enabled. Ignoring cache folder"
fi fi
temp=$source temp=$source
fi fi
CopyFiles Execute
CleanUp CleanUp
ExecTime ExecTime