Merge branch 'processing' into nightly

* processing:
  Add `processor` folder to folder structure
  Streamline `core.processor.nzbget.parse_status`
  Streamline `core.processor.nzbget._parse_unpack_status`
  Streamline `core.processor.nzbget._parse_health_status`
  Extract health status parsing from `core.processor.nzbget.parse_status` -> `_parse_health_status`
  Extract unpack status parsing from `core.processor.nzbget.parse_status` -> `_parse_unpack_status`
  Streamline `core.processor.nzbget._parse_par_status`
  Extract par status parsing from `core.processor.nzbget.parse_status` -> `_parse_par_status`
  Streamline `core.processor.nzbget._parse_total_status`
  Extract total status parsing from `core.processor.nzbget.parse_status` -> `_parse_total_status`
  Streamline `core.processor.nzbget.check_version`
  Streamline `core.processor.nzbget.parse_failure_link`
  Streamline `core.processor.nzbget.parse_download_id`
  Standardize processing
  Extract version checks from `core.processor.nzbget.process` -> `check_version`
  Extract status parsing from `core.processor.nzbget.process` -> `parse_status`
  Extract failure_link parsing from `core.processor.nzbget.process` -> `parse_failure_link`
  Extract download_id parsing from `core.processor.nzbget.process` -> `parse_download_id`
  Standardize processing
  Merge legacy sab parsing with 0.7.17+
  Extract manual processing from `nzbToMedia.main` -> `core.processor.manual`
  Extract sabnzb processing from `nzbToMedia.main` -> `core.processor.sabnzbd`
  Extract nzbget processing from `nzbToMedia.main` -> `core.processor.nzbget`
  Refactor `nzbToMedia.process` -> `core.processor.nzb.process`

cleanup.py

@@ -25,6 +25,7 @@
         'auto_process',
         'extractor',
         'plugins',
+        'processor',
         'utils',
     ],
 }

core/__init__.py

@@ -89,10 +89,6 @@
 NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual']
 TORRENT_CLIENTS = ['transmission', 'deluge', 'utorrent', 'rtorrent', 'qbittorrent', 'other', 'manual']
 
-# sabnzbd constants
-SABNZB_NO_OF_ARGUMENTS = 8
-SABNZB_0717_NO_OF_ARGUMENTS = 9
-
 # sickbeard fork/branch constants
 FORK_DEFAULT = 'default'
 FORK_FAILED = 'failed'

core/processor/manual.py

@@ -0,0 +1,72 @@
+import os
+
+import core
+from core import logger
+from core.auto_process.common import ProcessResult
+from core.processor import nzb
+from core.utils import (
+    get_dirs,
+    get_download_info,
+)
+
+try:
+    text_type = unicode
+except NameError:
+    text_type = str
+
+
+def process():
+    # Perform Manual Post-Processing
+    logger.warning(
+        'Invalid number of arguments received from client, Switching to manual run mode ...')
+
+    # Post-Processing Result
+    result = ProcessResult(
+        message='',
+        status_code=0,
+    )
+
+    for section, subsections in core.SECTIONS.items():
+        for subsection in subsections:
+            if not core.CFG[section][subsection].isenabled():
+                continue
+            for dir_name in get_dirs(section, subsection, link='move'):
+                logger.info(
+                    'Starting manual run for {0}:{1} - Folder: {2}'.format(
+                        section, subsection, dir_name))
+                logger.info(
+                    'Checking database for download info for {0} ...'.format(
+                        os.path.basename(dir_name)))
+
+                core.DOWNLOAD_INFO = get_download_info(
+                    os.path.basename(dir_name), 0)
+                if core.DOWNLOAD_INFO:
+                    logger.info('Found download info for {0}, '
+                                'setting variables now ...'.format
+                                (os.path.basename(dir_name)))
+                    client_agent = text_type(
+                        core.DOWNLOAD_INFO[0]['client_agent']) or 'manual'
+                    download_id = text_type(
+                        core.DOWNLOAD_INFO[0]['input_id']) or ''
+                else:
+                    logger.info('Unable to locate download info for {0}, '
+                                'continuing to try and process this release ...'.format
+                                (os.path.basename(dir_name)))
+                    client_agent = 'manual'
+                    download_id = ''
+
+                if client_agent and client_agent.lower() not in core.NZB_CLIENTS:
+                    continue
+
+                input_name = os.path.basename(dir_name)
+
+                results = nzb.process(dir_name, input_name, 0,
+                                  client_agent=client_agent,
+                                  download_id=download_id or None,
+                                  input_category=subsection)
+                if results.status_code != 0:
+                    logger.error(
+                        'A problem was reported when trying to perform a manual run for {0}:{1}.'.format
+                        (section, subsection))
+                    result = results
+    return result

core/processor/nzb.py

@@ -0,0 +1,154 @@
+import datetime
+
+import core
+from core import logger, main_db
+from core.auto_process import comics, games, movies, music, tv, books
+from core.auto_process.common import ProcessResult
+from core.plugins.downloaders.nzb.utils import get_nzoid
+from core.plugins.plex import plex_update
+from core.user_scripts import external_script
+from core.utils import (
+    char_replace,
+    clean_dir,
+    convert_to_ascii,
+    extract_files,
+    update_download_info_status,
+)
+
+try:
+    text_type = unicode
+except NameError:
+    text_type = str
+
+
+def process(input_directory, input_name=None, status=0, client_agent='manual', download_id=None, input_category=None, failure_link=None):
+    if core.SAFE_MODE and input_directory == core.NZB_DEFAULT_DIRECTORY:
+        logger.error(
+            'The input directory:[{0}] is the Default Download Directory. Please configure category directories to prevent processing of other media.'.format(
+                input_directory))
+        return ProcessResult(
+            message='',
+            status_code=-1,
+        )
+
+    if not download_id and client_agent == 'sabnzbd':
+        download_id = get_nzoid(input_name)
+
+    if client_agent != 'manual' and not core.DOWNLOAD_INFO:
+        logger.debug('Adding NZB download info for directory {0} to database'.format(input_directory))
+
+        my_db = main_db.DBConnection()
+
+        input_directory1 = input_directory
+        input_name1 = input_name
+
+        try:
+            encoded, input_directory1 = char_replace(input_directory)
+            encoded, input_name1 = char_replace(input_name)
+        except Exception:
+            pass
+
+        control_value_dict = {'input_directory': text_type(input_directory1)}
+        new_value_dict = {
+            'input_name': text_type(input_name1),
+            'input_hash': text_type(download_id),
+            'input_id': text_type(download_id),
+            'client_agent': text_type(client_agent),
+            'status': 0,
+            'last_update': datetime.date.today().toordinal(),
+        }
+        my_db.upsert('downloads', new_value_dict, control_value_dict)
+
+    # auto-detect section
+    if input_category is None:
+        input_category = 'UNCAT'
+    usercat = input_category
+    section = core.CFG.findsection(input_category).isenabled()
+    if section is None:
+        section = core.CFG.findsection('ALL').isenabled()
+        if section is None:
+            logger.error(
+                'Category:[{0}] is not defined or is not enabled. Please rename it or ensure it is enabled for the appropriate section in your autoProcessMedia.cfg and try again.'.format(
+                    input_category))
+            return ProcessResult(
+                message='',
+                status_code=-1,
+            )
+        else:
+            usercat = 'ALL'
+
+    if len(section) > 1:
+        logger.error(
+            'Category:[{0}] is not unique, {1} are using it. Please rename it or disable all other sections using the same category name in your autoProcessMedia.cfg and try again.'.format(
+                input_category, section.keys()))
+        return ProcessResult(
+            message='',
+            status_code=-1,
+        )
+
+    if section:
+        section_name = section.keys()[0]
+        logger.info('Auto-detected SECTION:{0}'.format(section_name))
+    else:
+        logger.error('Unable to locate a section with subsection:{0} enabled in your autoProcessMedia.cfg, exiting!'.format(
+            input_category))
+        return ProcessResult(
+            status_code=-1,
+            message='',
+        )
+
+    cfg = dict(core.CFG[section_name][usercat])
+
+    extract = int(cfg.get('extract', 0))
+
+    try:
+        if int(cfg.get('remote_path')) and not core.REMOTE_PATHS:
+            logger.error('Remote Path is enabled for {0}:{1} but no Network mount points are defined. Please check your autoProcessMedia.cfg, exiting!'.format(
+                section_name, input_category))
+            return ProcessResult(
+                status_code=-1,
+                message='',
+            )
+    except Exception:
+        logger.error('Remote Path {0} is not valid for {1}:{2} Please set this to either 0 to disable or 1 to enable!'.format(
+            cfg.get('remote_path'), section_name, input_category))
+
+    input_name, input_directory = convert_to_ascii(input_name, input_directory)
+
+    if extract == 1 and not (status > 0 and core.NOEXTRACTFAILED):
+        logger.debug('Checking for archives to extract in directory: {0}'.format(input_directory))
+        extract_files(input_directory)
+
+    logger.info('Calling {0}:{1} to post-process:{2}'.format(section_name, input_category, input_name))
+
+    if section_name in ['CouchPotato', 'Radarr', 'Watcher3']:
+        result = movies.process(section_name, input_directory, input_name, status, client_agent, download_id, input_category, failure_link)
+    elif section_name in ['SickBeard', 'SiCKRAGE', 'NzbDrone', 'Sonarr']:
+        result = tv.process(section_name, input_directory, input_name, status, client_agent, download_id, input_category, failure_link)
+    elif section_name in ['HeadPhones', 'Lidarr']:
+        result = music.process(section_name, input_directory, input_name, status, client_agent, input_category)
+    elif section_name == 'Mylar':
+        result = comics.process(section_name, input_directory, input_name, status, client_agent, input_category)
+    elif section_name == 'Gamez':
+        result = games.process(section_name, input_directory, input_name, status, client_agent, input_category)
+    elif section_name == 'LazyLibrarian':
+        result = books.process(section_name, input_directory, input_name, status, client_agent, input_category)
+    elif section_name == 'UserScript':
+        result = external_script(input_directory, input_name, input_category, section[usercat])
+    else:
+        result = ProcessResult(
+            message='',
+            status_code=-1,
+        )
+
+    plex_update(input_category)
+
+    if result.status_code == 0:
+        if client_agent != 'manual':
+            # update download status in our DB
+            update_download_info_status(input_name, 1)
+        if section_name not in ['UserScript', 'NzbDrone', 'Sonarr', 'Radarr', 'Lidarr']:
+            # cleanup our processing folders of any misc unwanted files and empty directories
+            clean_dir(input_directory, section_name, input_category)
+
+    return result

core/processor/nzbget.py

@@ -0,0 +1,108 @@
+import os
+import sys
+
+import core
+from core import logger
+from core.processor import nzb
+
+
+def parse_download_id():
+    """Parse nzbget download_id from environment."""
+    download_id_keys = [
+        'NZBPR_COUCHPOTATO',
+        'NZBPR_DRONE',
+        'NZBPR_SONARR',
+        'NZBPR_RADARR',
+        'NZBPR_LIDARR',
+    ]
+    for download_id_key in download_id_keys:
+        try:
+            return os.environ[download_id_key]
+        except KeyError:
+            pass
+    else:
+        return ''
+
+
+def parse_failure_link():
+    """Parse nzbget failure_link from environment."""
+    return os.environ.get('NZBPR__DNZB_FAILURE')
+
+
+def _parse_total_status():
+    status_summary = os.environ['NZBPP_TOTALSTATUS']
+    if status_summary != 'SUCCESS':
+        status = os.environ['NZBPP_STATUS']
+        logger.info('Download failed with status {0}.'.format(status))
+        return 1
+    return 0
+
+
+def _parse_par_status():
+    """Parse nzbget par status from environment."""
+    par_status = os.environ['NZBPP_PARSTATUS']
+    if par_status == '1' or par_status == '4':
+        logger.warning('Par-repair failed, setting status \'failed\'')
+        return 1
+    return 0
+
+
+def _parse_unpack_status():
+    if os.environ['NZBPP_UNPACKSTATUS'] == '1':
+        logger.warning('Unpack failed, setting status \'failed\'')
+        return 1
+    return 0
+
+
+def _parse_health_status():
+    """Parse nzbget download health from environment."""
+    status = 0
+    unpack_status_value = os.environ['NZBPP_UNPACKSTATUS']
+    par_status_value = os.environ['NZBPP_PARSTATUS']
+    if unpack_status_value == '0' and par_status_value == '0':
+        # Unpack was skipped due to nzb-file properties
+        # or due to errors during par-check
+        if int(os.environ['NZBPP_HEALTH']) < 1000:
+            logger.warning('Download health is compromised and Par-check/repair disabled or no .par2 files found. Setting status \'failed\'')
+            status = 1
+        else:
+            logger.info('Par-check/repair disabled or no .par2 files found, and Unpack not required. Health is ok so handle as though download successful')
+        logger.info('Please check your Par-check/repair settings for future downloads.')
+    return status
+
+
+def parse_status():
+    if 'NZBPP_TOTALSTATUS' in os.environ:  # Called from nzbget 13.0 or later
+        status = _parse_total_status()
+    else:
+        par_status = _parse_par_status()
+        unpack_status = _parse_unpack_status()
+        health_status = _parse_health_status()
+        status = par_status or unpack_status or health_status
+    return status
+
+
+def check_version():
+    """Check nzbget version and if version is unsupported, exit."""
+    version = os.environ['NZBOP_VERSION']
+    # Check if the script is called from nzbget 11.0 or later
+    if version[0:5] < '11.0':
+        logger.error('NZBGet Version {0} is not supported. Please update NZBGet.'.format(version))
+        sys.exit(core.NZBGET_POSTPROCESS_ERROR)
+    logger.info('Script triggered from NZBGet Version {0}.'.format(version))
+
+
+def process():
+    check_version()
+    status = parse_status()
+    download_id = parse_download_id()
+    failure_link = parse_failure_link()
+    return nzb.process(
+        input_directory=os.environ['NZBPP_DIRECTORY'],
+        input_name=os.environ['NZBPP_NZBNAME'],
+        status=status,
+        client_agent='nzbget',
+        download_id=download_id,
+        input_category=os.environ['NZBPP_CATEGORY'],
+        failure_link=failure_link,
+    )