Implement a "max_results" parameter for the data method, to allow for retrieval of more than 1000 data points.

This implementation is subject to an existing bug in the pagination link header in the API that will break when and if that bug is fixed.

Author: lcmcninch

Adafruit_IO/client.py

@@ -22,13 +22,18 @@
 import json
 import platform
 import pkg_resources
+import re
+from urllib.parse import urlparse
+from urllib.parse import parse_qs
 # import logging
 
 import requests
 
 from .errors import RequestError, ThrottlingError
 from .model import Data, Feed, Group
 
+API_PAGE_LIMIT = 1000
+
 # set outgoing version, pulled from setup.py
 version = pkg_resources.require("Adafruit_IO")[0].version
 default_headers = {
@@ -60,6 +65,9 @@ def __init__(self, username, key, proxies=None, base_url='https://io.adafruit.co
         # constructing the path.
         self.base_url = base_url.rstrip('/')
 
+        # Store the last response of a get or post
+        self._last_response = None
+
     @staticmethod
     def to_red(data):
         """Hex color feed to red channel.
@@ -116,6 +124,7 @@ def _get(self, path, params=None):
                                 headers=self._headers({'X-AIO-Key': self.key}),
                                 proxies=self.proxies,
                                 params=params)
+        self._last_response = response
         self._handle_error(response)
         return response.json()
 
@@ -125,6 +134,7 @@ def _post(self, path, data):
                                                         'Content-Type': 'application/json'}),
                                  proxies=self.proxies,
                                  data=json.dumps(data))
+        self._last_response = response
         self._handle_error(response)
         return response.json()
 
@@ -133,6 +143,7 @@ def _delete(self, path):
                                    headers=self._headers({'X-AIO-Key': self.key,
                                                           'Content-Type': 'application/json'}),
                                    proxies=self.proxies)
+        self._last_response = response
         self._handle_error(response)
 
     # Data functionality.
@@ -232,17 +243,52 @@ def receive_previous(self, feed):
         path = "feeds/{0}/data/previous".format(feed)
         return Data.from_dict(self._get(path))
 
-    def data(self, feed, data_id=None):
+    def data(self, feed, data_id=None, max_results=API_PAGE_LIMIT):
         """Retrieve data from a feed. If data_id is not specified then all the data
         for the feed will be returned in an array.
         :param string feed: Name/Key/ID of Adafruit IO feed.
         :param string data_id: ID of the piece of data to delete.
+        :param int max_results: The maximum number of results to return. To
+            return all data, set to None.
         """
-        if data_id is None:
-            path = "feeds/{0}/data".format(feed)
-            return list(map(Data.from_dict, self._get(path)))
-        path = "feeds/{0}/data/{1}".format(feed, data_id)
-        return Data.from_dict(self._get(path))
+        if data_id:
+            path = "feeds/{0}/data/{1}".format(feed, data_id)
+            return Data.from_dict(self._get(path))
+
+        params = {'limit': max_results} if max_results else None
+        data = []
+        path = "feeds/{0}/data".format(feed)
+        while True:
+            data.extend(list(map(Data.from_dict, self._get(path,
+                                                           params=params))))
+            nlink = self.get_next_link()
+            if not nlink:
+                break
+            # Parse the link for the query parameters
+            params = parse_qs(urlparse(nlink).query)
+            if max_results:
+                if len(data) >= max_results:
+                    break
+                params['limit'] = max_results - len(data)
+        return data
+
+    def get_next_link(self):
+        """Parse the `next` page URL in the pagination Link header.
+
+        This is necessary because of a bug in the API's implementation of the
+        link header. If that bug is fixed, the link would be accesible by
+        response.links['next']['url'] and this method would be broken.
+
+        :return: The url for the next page of data
+        :rtype: str
+        """
+        if not self._last_response:
+            return
+        link_header = self._last_response.headers['link']
+        res = re.search('rel="next", <(.+?)>', link_header)
+        if not res:
+            return
+        return res.groups()[0]
 
     def create_data(self, feed, data):
         """Create a new row of data in the specified feed.

docs/data.rst

@@ -33,6 +33,19 @@ You can get all of the data for a feed by using the ``data(feed)`` method. The r
     for d in data:
         print('Data value: {0}'.format(d.value))
 
+By default, the maximum number of data points returned is 1000. This limit can be changed by using the max_results parameter.
+
+.. code-block:: python
+
+    # Get less than the default number of data points
+    data = aio.data('Test', max_results=100)
+
+    # Get more than the default number of data points
+    data = aio.data('Test', max_results=2000)
+
+    # Get all of the points
+    data = aio.data('Test', max_results=None)
+
 You can also get a specific value by ID by using the ``feeds(feed, data_id)`` method. This will return a single piece of feed data with the provided data ID if it exists in the feed. The returned object will be an instance of the Data class.
 
 

tests/test_client.py

@@ -22,8 +22,8 @@ class TestClient(base.IOTestCase):
 
     # If your IP isn't put on the list of non-throttled IPs, uncomment the
     # function below to waste time between tests to prevent throttling.
-    #def tearDown(self):
-    time.sleep(30.0)
+    def tearDown(self):
+        time.sleep(30.0)
 
     # Helper Methods
     def get_client(self):
@@ -48,7 +48,7 @@ def ensure_group_deleted(self, client, group):
 
     def empty_feed(self, client, feed):
         # Remove all the data from a specified feed (but don't delete the feed).
-        data = client.data(feed)
+        data = client.data(feed, max_results=None)
         for d in data:
             client.delete(feed, d.id)
 
@@ -138,7 +138,7 @@ def test_create_data(self):
         data = Data(value=42)
         result = aio.create_data('testfeed', data)
         self.assertEqual(int(result.value), 42)
-    
+
     def test_location_data(self):
         """receive_location
         """