Add support for retrying failed requests with exponential backoff.

This feature can be enabled by setting the `retries` configuration for the `Google_Task_Runner` class: ~~~php $client = new Google_Client(); $client->setClassConfig('Google_Task_Runner', 'retries', 5); ~~~ Once `retries` is configured, the following error types will be retried by default: * Errors with a `500` or a `503` HTTP status code. * Errors with a `rateLimitExceeded` or a `userRateLimitExceeded` reason type. * cURL errors: * `CURLE_COULDNT_RESOLVE_HOST` * `CURLE_COULDNT_CONNECT` * `CURLE_OPERATION_TIMEOUTED` * `CURLE_SSL_CONNECT_ERROR` * `CURLE_GOT_NOTHING` These error types can be configured using the `retry_map` configurations for the `Google_IO_Exception` and `Google_Service_Exception` classes. The retry map is an associative array which maps errors types to the number of times each error can be retried (-1 can be used to retry an error indefinitely). For example: ~~~php $client = new Google_Client(); $client->setClassConfig( 'Google_Service_Exception', 'retry_map', array( // Don't retry 500 errors '500' => 0, // Retry 503 errors once '503' => 1, // Retry rate limit errors until we hit `retries` 'rateLimitExceeded' => -1, 'userRateLimitExceeded' => -1 ) ); ~~~ For convenience, `Google_Config` provides three constants for configuring retry limits: `Google_Config::TASK_RETRY_NEVER`, `Google_Config::TASK_RETRY_ONCE`, and `Google_Config::TASK_RETRY_ALWAYS`.

Author: whatthejeff

src/Google/Config.php

@@ -25,6 +25,9 @@ class Google_Config
  const GZIP_UPLOADS_ENABLED = true;
  const GZIP_UPLOADS_DISABLED = false;
  const USE_AUTO_IO_SELECTION = "auto";
+ const TASK_RETRY_NEVER = 0;
+ const TASK_RETRY_ONCE = 1;
+ const TASK_RETRY_ALWAYS = -1;
  protected $configuration;
 
  /**
@@ -101,6 +104,36 @@ public function __construct($ini_file_location = null)
  'federated_signon_certs_url' =>
  'https://www.googleapis.com/oauth2/v1/certs',
  ),
+ 'Google_Task_Runner' => array(
+ // Delays are specified in seconds
+ 'initial_delay' => 1,
+ 'max_delay' => 60,
+ // Base number for exponential backoff
+ 'factor' => 2,
+ // A random number between -jitter and jitter will be added to the
+ // factor on each iteration to allow for better distribution of
+ // retries.
+ 'jitter' => .5,
+ // Maximum number of retries allowed
+ 'retries' => 0
+ ),
+ 'Google_Service_Exception' => array(
+ 'retry_map' => array(
+ '500' => self::TASK_RETRY_ALWAYS,
+ '503' => self::TASK_RETRY_ALWAYS,
+ 'rateLimitExceeded' => self::TASK_RETRY_ALWAYS,
+ 'userRateLimitExceeded' => self::TASK_RETRY_ALWAYS
+ )
+ ),
+ 'Google_IO_Exception' => array(
+ 'retry_map' => array(
+ CURLE_COULDNT_RESOLVE_HOST => self::TASK_RETRY_ALWAYS,
+ CURLE_COULDNT_CONNECT => self::TASK_RETRY_ALWAYS,
+ CURLE_OPERATION_TIMEOUTED => self::TASK_RETRY_ALWAYS,
+ CURLE_SSL_CONNECT_ERROR => self::TASK_RETRY_ALWAYS,
+ CURLE_GOT_NOTHING => self::TASK_RETRY_ALWAYS
+ )
+ ),
  // Set a default directory for the file cache.
  'Google_Cache_File' => array(
  'directory' => sys_get_temp_dir() . '/Google_Client'

src/Google/Http/REST.php

@@ -26,7 +26,8 @@
 class Google_Http_REST
 {
  /**
- * Executes a Google_Http_Request
+ * Executes a Google_Http_Request and (if applicable) automatically retries
+ * when errors occur.
  *
  * @param Google_Client $client
  * @param Google_Http_Request $req
@@ -35,6 +36,27 @@ class Google_Http_REST
  * invalid or malformed post body, invalid url)
  */
  public static function execute(Google_Client $client, Google_Http_Request $req)
+ {
+ $runner = new Google_Task_Runner(
+ $client,
+ sprintf('%s %s', $req->getRequestMethod(), $req->getUrl()),
+ array(get_class(), 'doExecute'),
+ array($client, $req)
+ );
+
+ return $runner->run();
+ }
+
+ /**
+ * Executes a Google_Http_Request
+ *
+ * @param Google_Client $client
+ * @param Google_Http_Request $req
+ * @return array decoded result
+ * @throws Google_Service_Exception on server side error (ie: not authenticated,
+ * invalid or malformed post body, invalid url)
+ */
+ public static function doExecute(Google_Client $client, Google_Http_Request $req)
  {
  $httpRequest = $client->getIo()->makeRequest($req);
  $httpRequest->setExpectedClass($req->getExpectedClass());
@@ -74,13 +96,19 @@ public static function decodeHttpResponse($response, Google_Client $client = nul
  $errors = $decoded['error']['errors'];
  }
 
+ $map = null;
  if ($client) {
  $client->getLogger()->error(
  $err,
  array('code' => $code, 'errors' => $errors)
  );
+
+ $map = $client->getClassConfig(
+ 'Google_Service_Exception',
+ 'retry_map'
+ );
  }
- throw new Google_Service_Exception($err, $code, null, $errors);
+ throw new Google_Service_Exception($err, $code, null, $errors, $map);
  }
 
  // Only attempt to decode the response, if the response code wasn't (204) 'no content'

src/Google/IO/Curl.php

@@ -90,9 +90,11 @@ public function executeRequest(Google_Http_Request $request)
  $response = curl_exec($curl);
  if ($response === false) {
  $error = curl_error($curl);
+ $code = curl_errno($curl);
+ $map = $this->client->getClassConfig('Google_IO_Exception', 'retry_map');
 
  $this->client->getLogger()->error('cURL ' . $error);
- throw new Google_IO_Exception($error);
+ throw new Google_IO_Exception($error, $code, null, $map);
  }
  $headerSize = curl_getinfo($curl, CURLINFO_HEADER_SIZE);
 

src/Google/IO/Exception.php

@@ -17,6 +17,51 @@
 
 require_once realpath(dirname(__FILE__) . '/../../../autoload.php');
 
-class Google_IO_Exception extends Google_Exception
+class Google_IO_Exception extends Google_Exception implements Google_Task_Retryable
 {
+ /**
+ * @var array $retryMap Map of errors with retry counts.
+ */
+ private $retryMap = array();
+
+ /**
+ * Creates a new IO exception with an optional retry map.
+ *
+ * @param string $message
+ * @param int $code
+ * @param Exception|null $previous
+ * @param array|null $retryMap Map of errors with retry counts.
+ */
+ public function __construct(
+ $message,
+ $code = 0,
+ Exception $previous = null,
+ array $retryMap = null
+ ) {
+ if (version_compare(PHP_VERSION, '5.3.0') >= 0) {
+ parent::__construct($message, $code, $previous);
+ } else {
+ parent::__construct($message, $code);
+ }
+
+ if (is_array($retryMap)) {
+ $this->retryMap = $retryMap;
+ }
+ }
+
+ /**
+ * Gets the number of times the associated task can be retried.
+ *
+ * NOTE: -1 is returned if the task can be retried indefinitely
+ *
+ * @return integer
+ */
+ public function allowedRetries()
+ {
+ if (isset($this->retryMap[$this->code])) {
+ return $this->retryMap[$this->code];
+ }
+
+ return 0;
+ }
 }

src/Google/Service/Exception.php

@@ -1,28 +1,51 @@
 <?php
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 require_once realpath(dirname(__FILE__) . '/../../../autoload.php');
 
-class Google_Service_Exception extends Google_Exception
+class Google_Service_Exception extends Google_Exception implements Google_Task_Retryable
 {
  /**
  * Optional list of errors returned in a JSON body of an HTTP error response.
  */
  protected $errors = array();
 
  /**
- * Override default constructor to add ability to set $errors.
+ * @var array $retryMap Map of errors with retry counts.
+ */
+ private $retryMap = array();
+
+ /**
+ * Override default constructor to add the ability to set $errors and a retry
+ * map.
  *
  * @param string $message
  * @param int $code
  * @param Exception|null $previous
  * @param [{string, string}] errors List of errors returned in an HTTP
  * response. Defaults to [].
+ * @param array|null $retryMap Map of errors with retry counts.
  */
  public function __construct(
  $message,
  $code = 0,
  Exception $previous = null,
- $errors = array()
+ $errors = array(),
+ array $retryMap = null
  ) {
  if (version_compare(PHP_VERSION, '5.3.0') >= 0) {
  parent::__construct($message, $code, $previous);
@@ -31,6 +54,10 @@ public function __construct(
  }
 
  $this->errors = $errors;
+
+ if (is_array($retryMap)) {
+ $this->retryMap = $retryMap;
+ }
  }
 
  /**
@@ -50,4 +77,27 @@ public function getErrors()
  {
  return $this->errors;
  }
+
+ /**
+ * Gets the number of times the associated task can be retried.
+ *
+ * NOTE: -1 is returned if the task can be retried indefinitely
+ *
+ * @return integer
+ */
+ public function allowedRetries()
+ {
+ if (isset($this->retryMap[$this->code])) {
+ return $this->retryMap[$this->code];
+ }
+
+ $errors = $this->getErrors();
+
+ if (!empty($errors) && isset($errors[0]['reason']) &&
+ isset($this->retryMap[$errors[0]['reason']])) {
+ return $this->retryMap[$errors[0]['reason']];
+ }
+
+ return 0;
+ }
 }

src/Google/Task/Exception.php

@@ -0,0 +1,22 @@
+<?php
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+require_once realpath(dirname(__FILE__) . '/../../../autoload.php');
+
+class Google_Task_Exception extends Google_Exception
+{
+}

src/Google/Task/Retryable.php

@@ -0,0 +1,34 @@
+<?php
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+require_once realpath(dirname(__FILE__) . '/../../../autoload.php');
+
+/**
+ * Interface for checking how many times a given task can be retried following
+ * a failure.
+ */
+interface Google_Task_Retryable
+{
+ /**
+ * Gets the number of times the associated task can be retried.
+ *
+ * NOTE: -1 is returned if the task can be retried indefinitely
+ *
+ * @return integer
+ */
+ public function allowedRetries();
+}