add pagination wrapper

Author: jvmvik

serpapi/pagination.py

@@ -0,0 +1,29 @@
+
+# Paginate response in SearpApi
+class Pagination:
+ def __init__(self, client, start = 0, end = 1000000000, page_size = 10):
+ self.client = client
+ self.start = start
+ self.end = end
+ self.page_size = page_size
+ 
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ # execute search
+ self.client.params_dict['start'] = self.start
+ result = self.client.get_dict()
+ 
+ # quit if no next page
+ if not 'next' in result['serpapi_pagination']:
+ raise StopIteration
+
+ # increment page
+ self.start += self.page_size
+
+ # ends
+ if self.start > self.end:
+ raise StopIteration
+
+ return result

serpapi/serp_api_client.py

@@ -1,5 +1,6 @@
 import requests
 import json
+from serpapi.pagination import Pagination
 
 GOOGLE_ENGINE = 'google'
 BING_ENGINE = 'bing'
@@ -165,4 +166,9 @@ def get_location(self, q, limit = 5):
  self.params_dict["limit"] = limit
  buffer = self.get_results('/locations.json')
  return json.loads(buffer)
-
+ 
+ def pagination(self, start = 0, end = 1000000000):
+ """Return:
+ Generator to iterate the search results pagination
+ """
+ return Pagination(self, start, end)

tests/test_example.py

@@ -58,6 +58,9 @@ def test_async(self):
  print("execute async search: q = " + company)
  search.params_dict["q"] = company
  data = search.get_dict()
+ if data is not None:
+ print("oops data is empty for: " + company)
+ continue
  print("add search to the queue where id: " + data['search_metadata']['id'])
  # add search to the search_queue
  search_queue.put(data)

tests/test_google_search.py

@@ -10,6 +10,18 @@ class TestSearchApi(unittest.TestCase):
 def setUp(self):
 GoogleSearch.SERP_API_KEY = os.getenv("API_KEY", "demo")
 
+@unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided")
+def test_paginate(self):
+search = GoogleSearch({"q": "Coffee", "location": "Austin,Texas"})
+pages = search.pagination(0, 20)
+print("display generated")
+urls = []
+for result in pages:
+urls.append(result['serpapi_pagination']['next'])
+self.assertEqual(len(urls), 2)
+self.assertTrue("start=10" in urls[0])
+self.assertTrue("start=20" in urls[1])
+
 @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided")
 def test_get_json(self):
 search = GoogleSearch({"q": "Coffee", "location": "Austin,Texas"})