From 9f6c068cdec1a31943a8f1d785f84d1476ebb749 Mon Sep 17 00:00:00 2001 From: Clinton Hall Date: Thu, 20 Jun 2019 12:56:02 +1200 Subject: [PATCH] Transcode patch 1 (#1627) * Add Piping of stderr to capture transcoding failures. #1619 * Allow passing absolute nice command. #1619 * Change .cfg description for niceness * Fix errors due to VM packages out of date (ffmpeg) * Fix Sqlite import error on tests * Fix Azure issues https://developercommunity.visualstudio.com/content/problem/598264/known-issue-azure-pipelines-images-missing-sqlite3.html --- autoProcessMedia.cfg.spec | 4 +++- azure-pipelines.yml | 44 +++++++++++++++++++++++++++++++++++---- core/__init__.py | 5 ++++- core/transcoder.py | 10 +++++---- nzbToCouchPotato.py | 6 ++++-- nzbToGamez.py | 6 ++++-- nzbToHeadPhones.py | 6 ++++-- nzbToLazyLibrarian.py | 6 ++++-- nzbToLidarr.py | 6 ++++-- nzbToMedia.py | 6 ++++-- nzbToMylar.py | 6 ++++-- nzbToNzbDrone.py | 6 ++++-- nzbToRadarr.py | 6 ++++-- nzbToSickBeard.py | 6 ++++-- 14 files changed, 93 insertions(+), 30 deletions(-) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index 7969831d0..e3d11cb66 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -36,7 +36,9 @@ [Posix] ### Process priority setting for External commands (Extractor and Transcoder) on Posix (Unix/Linux/OSX) systems. # Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). - niceness = 0 + # If entering an integer e.g 'niceness = 4', this is added to the nice command and passed as 'nice -n4' (Default). + # If entering a comma separated list e.g. 'niceness = nice,4' this will be passed as 'nice 4' (Safer). + niceness = nice,-n0 # Set the ionice scheduling class. 0 for none, 1 for real time, 2 for best-effort, 3 for idle. ionice_class = 0 # Set the ionice scheduling class data. This defines the class data, if the class accepts an argument. For real time and best-effort, 0-7 is valid data. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 96321a5bb..bc64c2d4a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,6 +24,42 @@ jobs: maxParallel: 4 steps: + - script: | + # Make sure all packages are pulled from latest + sudo apt-get update + + # Fail out if any setups fail + set -e + + # Delete old Pythons + rm -rf $AGENT_TOOLSDIRECTORY/Python/2.7.16 + rm -rf $AGENT_TOOLSDIRECTORY/Python/3.5.7 + rm -rf $AGENT_TOOLSDIRECTORY/Python/3.7.3 + + # Download new Pythons + azcopy --recursive \ + --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/2.7.15 \ + --destination $AGENT_TOOLSDIRECTORY/Python/2.7.15 + + azcopy --recursive \ + --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.5.5 \ + --destination $AGENT_TOOLSDIRECTORY/Python/3.5.5 + + azcopy --recursive \ + --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.7.2 \ + --destination $AGENT_TOOLSDIRECTORY/Python/3.7.2 + + # Install new Pythons + original_directory=$PWD + setups=$(find $AGENT_TOOLSDIRECTORY/Python -name setup.sh) + for setup in $setups; do + chmod +x $setup; + cd $(dirname $setup); + ./$(basename $setup); + cd $original_directory; + done; + displayName: 'Workaround: update apt and roll back Python versions' + - task: UsePythonVersion@0 inputs: versionSpec: '$(python.version)' @@ -41,10 +77,10 @@ jobs: displayName: 'pytest' - script: | - rm -rf .git - python cleanup.py - python TorrentToMedia.py - python nzbToMedia.py + rm -rf .git + python cleanup.py + python TorrentToMedia.py + python nzbToMedia.py displayName: 'Test source install cleanup' - task: PublishTestResults@2 diff --git a/core/__init__.py b/core/__init__.py index b3c643fbe..a7a4ce249 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -457,7 +457,10 @@ def configure_niceness(): with open(os.devnull, 'w') as devnull: try: subprocess.Popen(['nice'], stdout=devnull, stderr=devnull).communicate() - NICENESS.extend(['nice', '-n{0}'.format(int(CFG['Posix']['niceness']))]) + if len(CFG['Posix']['niceness'].split(',')) > 1: #Allow passing of absolute command, not just value. + NICENESS.extend(CFG['Posix']['niceness'].split(',')) + else: + NICENESS.extend(['nice', '-n{0}'.format(int(CFG['Posix']['niceness']))]) except Exception: pass try: diff --git a/core/transcoder.py b/core/transcoder.py index fcbeb43f6..f0da8dd45 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -607,7 +607,7 @@ def extract_subs(file, newfile_path, bitbucket): result = 1 # set result to failed in case call fails. try: proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) - proc.communicate() + out, err = proc.communicate() result = proc.returncode except Exception: logger.error('Extracting subtitle has failed') @@ -930,17 +930,19 @@ def transcode_directory(dir_name): result = 1 # set result to failed in case call fails. try: if isinstance(file, string_types): - proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) + proc = subprocess.Popen(command, stdout=bitbucket, stderr=subprocess.PIPE) else: img, data = next(iteritems(file)) - proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket, stdin=subprocess.PIPE) + proc = subprocess.Popen(command, stdout=bitbucket, stderr=subprocess.PIPE, stdin=subprocess.PIPE) for vob in data['files']: procin = zip_out(vob, img, bitbucket) if procin: logger.debug('Feeding in file: {0} to Transcoder'.format(vob)) shutil.copyfileobj(procin.stdout, proc.stdin) procin.stdout.close() - proc.communicate() + out, err = proc.communicate() + if err: + logger.error('Transcoder returned:{0} has failed'.format(err)) result = proc.returncode except Exception: logger.error('Transcoding of video {0} has failed'.format(newfile_path)) diff --git a/nzbToCouchPotato.py b/nzbToCouchPotato.py index 862e7f786..60638200f 100755 --- a/nzbToCouchPotato.py +++ b/nzbToCouchPotato.py @@ -112,8 +112,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToGamez.py b/nzbToGamez.py index 390234215..92237f07f 100755 --- a/nzbToGamez.py +++ b/nzbToGamez.py @@ -68,8 +68,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToHeadPhones.py b/nzbToHeadPhones.py index a1917f109..d875d1516 100755 --- a/nzbToHeadPhones.py +++ b/nzbToHeadPhones.py @@ -82,8 +82,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToLazyLibrarian.py b/nzbToLazyLibrarian.py index dd7607091..ad8d2534e 100755 --- a/nzbToLazyLibrarian.py +++ b/nzbToLazyLibrarian.py @@ -76,8 +76,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToLidarr.py b/nzbToLidarr.py index c38f59754..da89ab46d 100755 --- a/nzbToLidarr.py +++ b/nzbToLidarr.py @@ -97,8 +97,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToMedia.py b/nzbToMedia.py index 12a72ae6f..f2a5a57fd 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -468,8 +468,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToMylar.py b/nzbToMylar.py index e26cfe6d5..cd408dcba 100755 --- a/nzbToMylar.py +++ b/nzbToMylar.py @@ -73,8 +73,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToNzbDrone.py b/nzbToNzbDrone.py index 680074cd5..b99ed190c 100755 --- a/nzbToNzbDrone.py +++ b/nzbToNzbDrone.py @@ -102,8 +102,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToRadarr.py b/nzbToRadarr.py index f75ca3508..a7e02f93a 100755 --- a/nzbToRadarr.py +++ b/nzbToRadarr.py @@ -107,8 +107,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). # diff --git a/nzbToSickBeard.py b/nzbToSickBeard.py index 00f180773..b6d8b42e3 100755 --- a/nzbToSickBeard.py +++ b/nzbToSickBeard.py @@ -113,8 +113,10 @@ # Niceness for external tasks Extractor and Transcoder. # -# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). -#niceness=10 +# Set the Niceness value for the nice command. These range from -20 (most favorable to the process) to 19 (least favorable to the process). +# If entering an integer e.g 'niceness=4', this is added to the nice command and passed as 'nice -n4' (Default). +# If entering a comma separated list e.g. 'niceness=nice,4' this will be passed as 'nice 4' (Safer). +#niceness=nice,-n0 # ionice scheduling class (0, 1, 2, 3). #