Enable caching by default for API requests and Library Manager

Author: ivankravets

HISTORY.rst

@@ -44,6 +44,7 @@ PlatformIO 3.0
 * Added global ``lib_extra_dirs`` option to ``[platformio]`` section for
   `Project Configuration File "platformio.ini" <http://docs.platformio.org/en/stable/projectconf.html>`__
   (`issue #842 <https://github.com/platformio/platformio-core/issues/842>`_)
+* Enabled caching by default for API requests and Library Manager (see `enable_cache <http://docs.platformio.org/en/latest/userguide/cmd_settings.html#enable-cache>`__ setting)
 * Changed a default exit combination for Device Monitor from ``Ctrl+]`` to ``Ctrl+C``
 * Improved detecting of ARM mbed media disk for uploading
 * Improved Project Generator for CLion IDE when source folder contains nested items

docs/userguide/cmd_settings.rst

@@ -97,6 +97,16 @@ Check for the platform updates interval.
 
 Enable SSL for PlatformIO Services
 
+.. _enable_cache:
+
+``enable_cache``
+^^^^^^^^^^^^^^^^
+
+:Default:   Yes
+:Values:    Yes/No
+
+Enable caching for API requests and Library Manager
+
 .. _setting_force_verbose:
 
 ``force_verbose``
@@ -144,17 +154,21 @@ Examples
 
 1. List all settings and theirs current values
 
-.. code-block:: bash
+.. code::
+
+    > platformio settings get
 
-    $ platformio settings get
     Name                            Value [Default]   Description
     ------------------------------------------------------------------------------------------
-    auto_update_libraries           Yes               Automatically update libraries (Yes/No)
-    auto_update_platforms           Yes               Automatically update platforms (Yes/No)
+    auto_update_libraries           No                Automatically update libraries (Yes/No)
+    auto_update_platforms           No                Automatically update platforms (Yes/No)
     check_libraries_interval        7                 Check for the library updates interval (days)
     check_platformio_interval       3                 Check for the new PlatformIO interval (days)
     check_platforms_interval        7                 Check for the platform updates interval (days)
+    enable_cache                    Yes               Enable caching for API requests and Library Manager
+    enable_ssl                      No                Enable SSL for PlatformIO Services
     enable_telemetry                Yes               Telemetry service (Yes/No)
+    force_verbose                   No                Force verbose output when processing environments
 
 
 2. Show specified setting
@@ -223,8 +237,12 @@ Examples
 
     Name                            Value [Default]   Description
     ------------------------------------------------------------------------------------------
-    auto_update_libraries           Yes               Automatically update libraries (Yes/No)
-    auto_update_platforms           Yes               Automatically update platforms (Yes/No)
+    auto_update_libraries           No                Automatically update libraries (Yes/No)
+    auto_update_platforms           No                Automatically update platforms (Yes/No)
     check_libraries_interval        7                 Check for the library updates interval (days)
     check_platformio_interval       3                 Check for the new PlatformIO interval (days)
     check_platforms_interval        7                 Check for the platform updates interval (days)
+    enable_cache                    Yes               Enable caching for API requests and Library Manager
+    enable_ssl                      No                Enable SSL for PlatformIO Services
+    enable_telemetry                Yes               Telemetry service (Yes/No)
+    force_verbose                   No                Force verbose output when processing environments

platformio/app.py

@@ -56,6 +56,10 @@
         "description": "Enable SSL for PlatformIO Services",
         "value": False
     },
+    "enable_cache": {
+        "description": "Enable caching for API requests and Library Manager",
+        "value": True
+    },
     "enable_telemetry": {
         "description":
         ("Telemetry service <http://docs.platformio.org/en/stable/"
@@ -122,16 +126,20 @@ def _unlock_state_file(self):
             self._lockfile.release()
 
 
-class LocalCache(object):
+class ContentCache(object):
 
     def __init__(self, cache_dir=None):
+        self.cache_dir = None
+        self.db_path = None
+        if not get_setting("enable_cache"):
+            return
         self.cache_dir = cache_dir or join(util.get_home_dir(), ".cache")
         if not self.cache_dir:
             os.makedirs(self.cache_dir)
         self.db_path = join(self.cache_dir, "db.data")
 
     def __enter__(self):
-        if not isfile(self.db_path):
+        if not self.db_path or not isfile(self.db_path):
             return self
         newlines = []
         found = False
@@ -169,24 +177,26 @@ def key_from_args(*args):
         return h.hexdigest()
 
     def get(self, key):
+        if not self.cache_dir:
+            return None
         cache_path = self.get_cache_path(key)
         if not isfile(cache_path):
             return None
-        with open(cache_path) as fp:
+        with open(cache_path, "rb") as fp:
             data = fp.read()
             if data[0] in ("{", "["):
                 return json.loads(data)
             return data
 
     def set(self, key, data, valid):
-        if not data:
+        if not self.cache_dir or not data:
             return
         tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400}
         assert valid.endswith(tuple(tdmap.keys()))
         cache_path = self.get_cache_path(key)
         if not isdir(dirname(cache_path)):
             os.makedirs(dirname(cache_path))
-        with open(cache_path, "w") as fp:
+        with open(cache_path, "wb") as fp:
             if isinstance(data, dict) or isinstance(data, list):
                 json.dump(data, fp)
             else:
@@ -197,7 +207,7 @@ def set(self, key, data, valid):
         return True
 
     def clean(self):
-        if isdir(self.cache_dir):
+        if self.cache_dir and isdir(self.cache_dir):
             util.rmtree_(self.cache_dir)
 
 

platformio/maintenance.py

@@ -43,8 +43,8 @@ def in_silence(ctx=None):
 
 
 def clean_cache():
-    with app.LocalCache() as lc:
-        lc.clean()
+    with app.ContentCache() as cc:
+        cc.clean()
 
 
 def on_platformio_start(ctx, force, caller):

platformio/managers/lib.py

@@ -170,8 +170,11 @@ def _cmp_dates(datestr1, datestr2):
 
     def get_latest_repo_version(self, name, requirements):
         item = self.max_satisfying_repo_version(
-            util.get_api_result("/lib/versions/%d" % self._get_pkg_id_by_name(
-                name, requirements)), requirements)
+            util.get_api_result(
+                "/lib/versions/%d" % self._get_pkg_id_by_name(name,
+                                                              requirements),
+                cache_valid="1h"),
+            requirements)
         return item['version'] if item else None
 
     def _get_pkg_id_by_name(self,

platformio/managers/package.py

@@ -14,15 +14,15 @@
 
 import json
 import os
-from os.path import basename, dirname, isdir, isfile, islink, join
-from shutil import copytree
+import shutil
+from os.path import basename, dirname, getsize, isdir, isfile, islink, join
 from tempfile import mkdtemp
 
 import click
 import requests
 import semantic_version
 
-from platformio import exception, telemetry, util
+from platformio import app, exception, telemetry, util
 from platformio.downloader import FileDownloader
 from platformio.unpacker import FileUnpacker
 from platformio.vcsclient import VCSClientFactory
@@ -190,7 +190,7 @@ def _install_from_url(self, name, url, requirements=None, sha1=None):
                     self.unpack(url, tmp_dir)
                 else:
                     util.rmtree_(tmp_dir)
-                    copytree(url, tmp_dir)
+                    shutil.copytree(url, tmp_dir)
             elif url.startswith(("http://", "https://")):
                 dlpath = self.download(url, tmp_dir, sha1)
                 assert isfile(dlpath)
@@ -262,6 +262,9 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
 
     _INSTALLED_CACHE = {}
 
+    FILE_CACHE_VALID = "1m"  # 1 month
+    FILE_CACHE_MAX_SIZE = 1024 * 1024
+
     def __init__(self, package_dir, repositories=None):
         self.repositories = repositories
         self.package_dir = package_dir
@@ -273,13 +276,32 @@ def __init__(self, package_dir, repositories=None):
     def manifest_name(self):
         raise NotImplementedError()
 
-    @staticmethod
-    def download(url, dest_dir, sha1=None):
+    def download(self, url, dest_dir, sha1=None):
+        cache_key_fname = app.ContentCache.key_from_args(url, "fname")
+        cache_key_data = app.ContentCache.key_from_args(url, "data")
+        if self.FILE_CACHE_VALID:
+            with app.ContentCache() as cc:
+                fname = cc.get(cache_key_fname)
+                cache_path = cc.get_cache_path(cache_key_data)
+                if fname and isfile(cache_path):
+                    dst_path = join(dest_dir, fname)
+                    shutil.copy(cache_path, dst_path)
+                    return dst_path
+
         fd = FileDownloader(url, dest_dir)
         fd.start()
         if sha1:
             fd.verify(sha1)
-        return fd.get_filepath()
+        dst_path = fd.get_filepath()
+        if not self.FILE_CACHE_VALID or getsize(
+                dst_path) > BasePkgManager.FILE_CACHE_MAX_SIZE:
+            return dst_path
+
+        with app.ContentCache() as cc:
+            cc.set(cache_key_fname, basename(dst_path), self.FILE_CACHE_VALID)
+            cc.set(cache_key_data, "DUMMY", self.FILE_CACHE_VALID)
+            shutil.copy(dst_path, cc.get_cache_path(cache_key_data))
+        return dst_path
 
     @staticmethod
     def unpack(source_path, dest_dir):
@@ -580,6 +602,8 @@ def update(  # pylint: disable=too-many-return-statements
 
 class PackageManager(BasePkgManager):
 
+    FILE_CACHE_VALID = None  # disable package caching
+
     @property
     def manifest_name(self):
         return "package.json"

platformio/managers/platform.py

@@ -27,6 +27,8 @@
 
 class PlatformManager(BasePkgManager):
 
+    FILE_CACHE_VALID = None  # disable platform caching
+
     def __init__(self, package_dir=None, repositories=None):
         if not repositories:
             repositories = [

platformio/util.py

@@ -482,22 +482,22 @@ def _get_api_result(
 
 
 def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
-    from platformio.app import LocalCache
+    from platformio.app import ContentCache
     total = 0
     max_retries = 5
-    cache_key = (LocalCache.key_from_args(url, params, data, auth)
+    cache_key = (ContentCache.key_from_args(url, params, data, auth)
                  if cache_valid else None)
     while total < max_retries:
         try:
-            with LocalCache() as lc:
+            with ContentCache() as cc:
                 if cache_key:
-                    result = lc.get(cache_key)
+                    result = cc.get(cache_key)
                     if result is not None:
                         return result
             result = _get_api_result(url, params, data)
             if cache_valid:
-                with LocalCache() as lc:
-                    lc.set(cache_key, result, cache_valid)
+                with ContentCache() as cc:
+                    cc.set(cache_key, result, cache_valid)
             return result
         except (requests.exceptions.ConnectionError,
                 requests.exceptions.Timeout) as e: