Merge pull request #6 from mara/download-timeout-retry

Download timeout retry

README.md

@@ -116,29 +116,39 @@ For all options, see
       config.py are used.
 
     Options:
-      --developer_token TEXT       The developer token that is used to access the
-                                   BingAds API. Default: "012345679ABCDEF"
-      --oauth2_client_id TEXT      The Oauth client id obtained from the BingAds
-                                   developer center. Default:
-                                   "abc1234-1234-1234-abc-abcd1234"
-      --oauth2_client_secret TEXT  The Oauth client secret obtained from the
-                                   BingAds developer center. Default:
-                                   "ABCDefgh!1234567890"
-      --oauth2_refresh_token TEXT  The Oauth refresh token returned from the
-                                   adwords-downloader-refresh-oauth2-token script.
-                                   Default: "ABCDefgh!1234567890ABCDefgh!123456789
-                                   0ABCDefgh!1234567890ABCDefgh!1234567890ABCDefgh
-                                   !1234567890ABCDefgh!1234567890ABCDefgh!12345678
-                                   90"
-      --data_dir TEXT              The directory where result data is written to.
-                                   Default: "/tmp/bingads/"
-      --data_file TEXT             The name of the file the result is written to.
-                                   Default: "ad_performance.csv.gz"
-      --first_date TEXT            The first day from which on data will be
-                                   downloaded. Default: "2015-01-01"
-      --environment TEXT           The deployment environment. Default:
-                                   "production"
-      --timeout TEXT               The maximum amount of time (in milliseconds)
-                                   that you want to wait for the report download.
-                                   Default: "3600000"
-      --help                       Show this message and exit.
+
+      --developer_token TEXT          The developer token that is used to access
+                                      the BingAds API. Default: "012345679ABCDEF"
+      --oauth2_client_id TEXT         The Oauth client id obtained from the
+                                      BingAds developer center. Default:
+                                      "abc1234-1234-1234-abc-abcd1234"
+      --oauth2_client_secret TEXT     The Oauth client secret obtained from the
+                                      BingAds developer center. Default:
+                                      "ABCDefgh!1234567890"
+      --oauth2_refresh_token TEXT     The Oauth refresh token returned from the
+                                      adwords-downloader-refresh-oauth2-token
+                                      script. Default: "ABCDefgh!1234567890ABCDefg
+                                      h!1234567890ABCDefgh!1234567890ABCDefgh!1234
+                                      567890ABCDefgh!1234567890ABCDefgh!1234567890
+                                      ABCDefgh!1234567890"
+      --data_dir TEXT                 The directory where result data is written
+                                      to. Default: "/tmp/bingads/"
+      --data_file TEXT                The name of the file the result is written
+                                      to. Default: "ad_performance.csv.gz"
+      --first_date TEXT               The first day from which on data will be
+                                      downloaded. Default: "2015-01-01"
+      --environment TEXT              The deployment environment. Default:
+                                      "production"
+      --timeout INTEGER               The maximum amount of time (in milliseconds)
+                                      that you want to wait for the report
+                                      download. Default: "3600000"
+      --total_attempts_for_single_file INTEGER
+                                      The attempts to download a single file in
+                                      case of HTTP errors or timeouts. Default:
+                                      "5"
+      --retry_timeout_interval INTEGER
+                                      number of seconds to wait before trying
+                                      again to download a single day. Default:
+                                      "10"
+      --help                          Show this message and exit.
+

bingads_downloader/cli.py

@@ -1,17 +1,17 @@
 """Command line interface for adwords downloader"""
 
+import sys
+
 import click
 from bingads_downloader import downloader,config
 from functools import partial
 
 
 def config_option(config_function):
     """Helper decorator that turns an option function into a cli option"""
-
     return lambda function: \
         click.option('--' + config_function.__name__,
-                     help=f'{config_function.__doc__}. Default: "{config_function()}"') \
-            (function)
+                     help=f'{config_function.__doc__}. Default: "{config_function()}"')(function)
 
 
 def apply_options(kwargs):
@@ -20,6 +20,17 @@ def apply_options(kwargs):
         if value: setattr(config, key, partial(lambda v: v, value))
 
 
+def show_version():
+    """Shows the package version in logs, if possible"""
+    try:
+        import pkg_resources
+        version = pkg_resources.require("bingads-performance-downloader")[0].version
+        print('Bing ads performance downloader version {}'.format(version))
+    except:
+        print('Warning: cannot determine module version')
+        print(sys.exc_info())
+
+
 @click.command()
 @config_option(config.developer_token)
 @config_option(config.oauth2_client_id)
@@ -30,6 +41,7 @@ def refresh_oauth2_token(**kwargs):
     When options are not specified, then the defaults from config.py are used.
     """
     apply_options(kwargs)
+    show_version()
     downloader.refresh_oauth_token()
 
 
@@ -43,10 +55,13 @@ def refresh_oauth2_token(**kwargs):
 @config_option(config.first_date)
 @config_option(config.environment)
 @config_option(config.timeout)
+@config_option(config.total_attempts_for_single_file)
+@config_option(config.retry_timeout_interval)
 def download_data(**kwargs):
     """
     Downloads data.
     When options are not specified, then the defaults from config.py are used.
     """
     apply_options(kwargs)
+    show_version()
     downloader.download_data()

bingads_downloader/config.py

@@ -45,3 +45,13 @@ def oauth2_refresh_token() -> str:
 def timeout() -> int:
     """The maximum amount of time (in milliseconds) that you want to wait for the report download"""
     return 3600000
+
+
+def total_attempts_for_single_file() -> int:
+    """The attempts to download a single file in case of HTTP errors or timeouts"""
+    return 5
+
+
+def retry_timeout_interval() -> int:
+    """number of seconds to wait before trying again to download a single day"""
+    return 10

bingads_downloader/downloader.py

@@ -2,6 +2,7 @@
 import errno
 import sys
 import tempfile
+import urllib
 import webbrowser
 from pathlib import Path
 
@@ -59,6 +60,7 @@ def download_ad_performance_data(api_client: BingReportClient):
     first_date = datetime.datetime.strptime(config.first_date(), '%Y-%m-%d')
     last_date = datetime.datetime.now() - datetime.timedelta(days=1)
     current_date = last_date
+    remaining_attempts = config.total_attempts_for_single_file
     while current_date >= first_date:
         print(current_date)
         relative_filepath = Path('{date:%Y/%m/%d}/bing/'.format(
@@ -70,9 +72,23 @@ def download_ad_performance_data(api_client: BingReportClient):
             with tempfile.TemporaryDirectory() as tmp_dir:
                 tmp_filepath = Path(tmp_dir, relative_filepath)
                 tmp_filepath.parent.mkdir(exist_ok=True, parents=True)
-                submit_and_download(report_request, api_client, str(filepath))
-
-        current_date += datetime.timedelta(days=-1)
+                try:
+                    start_time = time.time()
+                    submit_and_download(report_request, api_client, str(filepath))
+                    print('Successfully downloaded data for {date:%Y-%m-%d} in {elapsed:.1f} seconds'
+                          .format(date=current_date, elapsed=time.time() - start_time))
+                    # date is decreased only if the download above does not fail
+                    current_date -= datetime.timedelta(days=1)
+                    remaining_attempts = config.total_attempts_for_single_file
+                except urllib.error.URLError as url_error:
+                    if remaining_attempts == 0:
+                        print('Too many failed attempts while downloading this day, quitting', file=sys.stderr)
+                        raise
+                    print('ERROR WHILE DOWNLOADING REPORT, RETRYING in {} seconds, attempt {}#...'
+                          .format(config.retry_timeout_interval, remaining_attempts), file=sys.stderr)
+                    print(url_error, file=sys.stderr)
+                    time.sleep(config.retry_timeout_interval)
+                    remaining_attempts -= 1
 
 
 def build_ad_performance_request_for_single_day(api_client: BingReportClient,

setup.py

@@ -22,5 +22,5 @@
             'refresh-bingsads-api-oauth2-token=bingads_downloader.cli:refresh_oauth2_token'
         ]
     },
-    python_requires='>=3.3'
+    python_requires='>=3.6'
 )