Author: chabotc
Date: Thu Oct 9 13:58:33 2008
New Revision: 703257
URL: http://svn.apache.org/viewvc?rev=703257&view=rev
Log:
SHINDIG-542 by Bruno rovagnati, implement curl_multi fetcher for fetching
multiple http requests simultanious, plus use that in the preloading code (and
hopefully soon for the new message bundle handling too thats still pending)
Modified:
incubator/shindig/trunk/php/src/common/RemoteContent.php
incubator/shindig/trunk/php/src/common/RemoteContentFetcher.php
incubator/shindig/trunk/php/src/common/RemoteContentRequest.php
incubator/shindig/trunk/php/src/common/sample/BasicRemoteContent.php
incubator/shindig/trunk/php/src/common/sample/BasicRemoteContentFetcher.php
incubator/shindig/trunk/php/src/gadgets/SigningFetcher.php
incubator/shindig/trunk/php/src/gadgets/servlet/GadgetRenderingServlet.php
Modified: incubator/shindig/trunk/php/src/common/RemoteContent.php
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/php/src/common/RemoteContent.php?rev=703257&r1=703256&r2=703257&view=diff
==============================================================================
--- incubator/shindig/trunk/php/src/common/RemoteContent.php (original)
+++ incubator/shindig/trunk/php/src/common/RemoteContent.php Thu Oct 9
13:58:33 2008
@@ -31,4 +31,6 @@
abstract class RemoteContent {
abstract public function fetch($request, $context);
+
+ abstract public function multiFetch(Array $requests, Array $contexts);
}
Modified: incubator/shindig/trunk/php/src/common/RemoteContentFetcher.php
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/php/src/common/RemoteContentFetcher.php?rev=703257&r1=703256&r2=703257&view=diff
==============================================================================
--- incubator/shindig/trunk/php/src/common/RemoteContentFetcher.php (original)
+++ incubator/shindig/trunk/php/src/common/RemoteContentFetcher.php Thu Oct 9
13:58:33 2008
@@ -29,6 +29,8 @@
abstract public function fetchRequest($request);
+ abstract public function multiFetchRequest(Array $requests);
+
public function getNextFetcher()
{
return $this->fetcher;
Modified: incubator/shindig/trunk/php/src/common/RemoteContentRequest.php
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/php/src/common/RemoteContentRequest.php?rev=703257&r1=703256&r2=703257&view=diff
==============================================================================
--- incubator/shindig/trunk/php/src/common/RemoteContentRequest.php (original)
+++ incubator/shindig/trunk/php/src/common/RemoteContentRequest.php Thu Oct 9
13:58:33 2008
@@ -21,6 +21,8 @@
class RemoteContentRequest {
// these are used for making the request
private $uri = '';
+ // to get real url after signed requests
+ private $notSignedUri = '';
private $method = '';
private $headers = array();
private $postBody = false;
@@ -39,6 +41,7 @@
public function __construct($uri, $headers = false, $postBody = false)
{
$this->uri = $uri;
+ $this->notSignedUri = $uri;
$this->headers = $headers;
$this->postBody = $postBody;
$this->created = time();
@@ -279,6 +282,11 @@
return $this->uri;
}
+ public function getNotSignedUrl()
+ {
+ return $this->notSignedUri;
+ }
+
public function getMethod()
{
return $this->method;
@@ -363,6 +371,11 @@
{
$this->uri = $uri;
}
+
+ public function setNotSignedUri($uri)
+ {
+ $this->notSignedUri = $uri;
+ }
}
/**
Modified: incubator/shindig/trunk/php/src/common/sample/BasicRemoteContent.php
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/php/src/common/sample/BasicRemoteContent.php?rev=703257&r1=703256&r2=703257&view=diff
==============================================================================
--- incubator/shindig/trunk/php/src/common/sample/BasicRemoteContent.php
(original)
+++ incubator/shindig/trunk/php/src/common/sample/BasicRemoteContent.php Thu
Oct 9 13:58:33 2008
@@ -33,11 +33,46 @@
$ret = $cachedRequest;
} else {
$ret = $remoteContentFetcher->fetchRequest($request);
- // only cache requests that returned a 200 OK
- if ($request->getHttpCode() == '200') {
+ // only cache requests that returned a 200 OK and is
not a POST
+ if ($request->getHttpCode() == '200' && !
$request->isPost()) {
$cache->set($request->toHash(), $request);
}
}
return $ret;
}
+
+ public function multiFetch(Array $requests, Array $contexts)
+ {
+ $cache = Config::get('data_cache');
+ $cache = new $cache();
+ $remoteContentFetcher = new BasicRemoteContentFetcher();
+
+ $rets = array();
+ $requestsToProc = array();
+
+ foreach ($requests as $request) {
+ list(, $context) = each($contexts);
+ if (! ($request instanceof RemoteContentRequest)) {
+ throw new RemoteContentException("Invalid
request type in remoteContent");
+ }
+ // determine which requests we can load from cache, and
which we have to actually fetch
+ if (! $context->getIgnoreCache() && !
$request->isPost() && ($cachedRequest = $cache->get($request->toHash(),
$context->getRefreshInterval())) !== false) {
+ $rets[] = $cachedRequest;
+ } else {
+ $requestsToProc[] = $request;
+ }
+ }
+
+ $newRets =
$remoteContentFetcher->multiFetchRequest($requestsToProc);
+
+ foreach ($newRets as $request) {
+ // only cache requests that returned a 200 OK and is
not a POST
+ if ($request->getHttpCode() == '200' && !
$request->isPost()) {
+ $cache->set($request->toHash(), $request);
+ }
+ $rets[] = $request;
+ }
+
+ return $rets;
+ }
}
\ No newline at end of file
Modified:
incubator/shindig/trunk/php/src/common/sample/BasicRemoteContentFetcher.php
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/php/src/common/sample/BasicRemoteContentFetcher.php?rev=703257&r1=703256&r2=703257&view=diff
==============================================================================
--- incubator/shindig/trunk/php/src/common/sample/BasicRemoteContentFetcher.php
(original)
+++ incubator/shindig/trunk/php/src/common/sample/BasicRemoteContentFetcher.php
Thu Oct 9 13:58:33 2008
@@ -82,4 +82,91 @@
unset($request->handle);
return $request;
}
+
+ public function multiFetchRequest(Array $requests)
+ {
+ $mh = curl_multi_init();
+ foreach ($requests as $request) {
+ $request->handle = curl_init();
+ curl_setopt($request->handle, CURLOPT_URL,
$request->getUrl());
+ curl_setopt($request->handle, CURLOPT_FOLLOWLOCATION,
1);
+ curl_setopt($request->handle, CURLOPT_RETURNTRANSFER,
1);
+ curl_setopt($request->handle, CURLOPT_AUTOREFERER, 1);
+ curl_setopt($request->handle, CURLOPT_MAXREDIRS, 10);
+ curl_setopt($request->handle, CURLOPT_CONNECTTIMEOUT,
2);
+ curl_setopt($request->handle, CURLOPT_TIMEOUT, 2);
+ curl_setopt($request->handle, CURLOPT_HEADER, 1);
+ curl_setopt($request->handle, CURLOPT_SSL_VERIFYPEER,
0);
+ // Set this so the multihandler will return data
+ curl_setopt($request->handle, CURLOPT_RETURNTRANSFER,
1);
+
+ $proxy = Config::get('proxy');
+ if (! empty($proxy)) {
+ curl_setopt($request->handle, CURLOPT_PROXY,
$proxy);
+ }
+ if ($request->hasHeaders()) {
+ $headers = explode("\n",
$request->getHeaders());
+ $outHeaders = array();
+ foreach ($headers as $header) {
+ if (strpos($header, ':')) {
+ $key = trim(substr($header, 0,
strpos($header, ':')));
+ $val = trim(substr($header,
strpos($header, ':') + 1));
+ if (strcasecmp($key,
"Transfer-Encoding") != 0 && strcasecmp($key, "Cache-Control") != 0 &&
strcasecmp($key, "Expires") != 0 && strcasecmp($key, "Content-Length") != 0) {
+ $outHeaders[] = "$key:
$val";
+ }
+ }
+ }
+ $outHeaders[] = "User-Agent: Shindig PHP";
+ curl_setopt($request->handle,
CURLOPT_HTTPHEADER, $outHeaders);
+ }
+ if ($request->isPost()) {
+ curl_setopt($request->handle, CURLOPT_POST, 1);
+ curl_setopt($request->handle,
CURLOPT_POSTFIELDS, $request->getPostBody());
+ }
+ curl_multi_add_handle($mh, $request->handle);
+ }
+
+ $running = null;
+ //execute the handles
+ do {
+ curl_multi_exec($mh, $running);
+ } while ($running > 0);
+
+ //Ideally this should be 0 after curl_multi_info_read() is
call, meaning all
+ //calls have bee processed
+ $msg_queue = null;
+ $responses = curl_multi_info_read($mh, $msg_queue);
+
+ foreach ($requests as $request) {
+ // Execute the request
+ $content = curl_multi_getcontent($request->handle);
+ $header = '';
+ $body = '';
+ // on redirects and such we get multiple headers back
from curl it seems, we really only want the last one
+ while (substr($content, 0, strlen('HTTP')) == 'HTTP' &&
strpos($content, "\r\n\r\n") !== false) {
+ $header = substr($content, 0, strpos($content,
"\r\n\r\n"));
+ $content = $body = substr($content,
strlen($header) + 4);
+ }
+ $httpCode = curl_getinfo($request->handle,
CURLINFO_HTTP_CODE);
+ $contentType = curl_getinfo($request->handle,
CURLINFO_CONTENT_TYPE);
+ if (! $httpCode) {
+ $httpCode = '404';
+ }
+ $request->setHttpCode($httpCode);
+ $request->setContentType($contentType);
+ $request->setResponseHeaders($header);
+ $request->setResponseContent($body);
+ $request->setResponseSize(strlen($content));
+ }
+
+ // close the handles
+ foreach ($requests as $request) {
+ curl_close($request->handle);
+ unset($request->handle);
+ }
+ curl_multi_close($mh);
+ unset($mh);
+
+ return $requests;
+ }
}
Modified: incubator/shindig/trunk/php/src/gadgets/SigningFetcher.php
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/php/src/gadgets/SigningFetcher.php?rev=703257&r1=703256&r2=703257&view=diff
==============================================================================
--- incubator/shindig/trunk/php/src/gadgets/SigningFetcher.php (original)
+++ incubator/shindig/trunk/php/src/gadgets/SigningFetcher.php Thu Oct 9
13:58:33 2008
@@ -102,7 +102,12 @@
return $this->getNextFetcher()->fetchRequest($signed);
}
- private function signRequest($url, $method)
+ public function multiFetchRequest(Array $requests)
+ {
+ return $this->getNextFetcher()->multiFetchRequest($requests);
+ }
+
+ public function signRequest($url, $method)
{
try {
// Parse the request into parameters for OAuth signing,
stripping out
@@ -238,5 +243,4 @@
// make a last sanity check on the key of the data by using a
regular expression
return ereg(SigningFetcher::$ALLOWED_PARAM_NAME,
$canonParamName);
}
-
}
Modified:
incubator/shindig/trunk/php/src/gadgets/servlet/GadgetRenderingServlet.php
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/php/src/gadgets/servlet/GadgetRenderingServlet.php?rev=703257&r1=703256&r2=703257&view=diff
==============================================================================
--- incubator/shindig/trunk/php/src/gadgets/servlet/GadgetRenderingServlet.php
(original)
+++ incubator/shindig/trunk/php/src/gadgets/servlet/GadgetRenderingServlet.php
Thu Oct 9 13:58:33 2008
@@ -361,7 +361,7 @@
*/
private function appendPreloads(Gadget $gadget, GadgetContext $context)
{
- $resp = '';
+ $resp = Array();
$gadgetSigner = Config::get('security_token_signer');
$gadgetSigner = new $gadgetSigner();
$token = '';
@@ -371,6 +371,8 @@
$token = '';
// no token given, safe to ignore
}
+ $unsignedRequests = $unsignedContexts = Array();
+ $signedRequests = Array();
foreach ($gadget->getPreloads() as $preload) {
try {
if (($preload->getAuth() == Auth::$NONE ||
$token != null) && (count($preload->getViews()) == 0 ||
in_array($context->getView(), $preload->getViews()))) {
@@ -380,13 +382,17 @@
$request->getOptions()->viewerSigned =
$preload->isSignViewer();
switch
(strtoupper(trim($preload->getAuth()))) {
case "NONE":
- $brc = new
BasicRemoteContent();
- $response =
$brc->fetch($request, $context);
+ //
Unify all unsigned requests to one single multi request
+ $unsignedRequests[] =
$request;
+ $unsignedContexts[] =
$context;
break;
case "SIGNED":
+ //
Unify all signed requests to one single multi request
$signingFetcherFactory
= new SigningFetcherFactory(Config::get("private_key_file"));
$fetcher =
$signingFetcherFactory->getSigningFetcher(new BasicRemoteContentFetcher(),
$token);
- $response =
$fetcher->fetch($preload->getHref(), $request->getMethod());
+ $req =
$fetcher->signRequest($preload->getHref(), $request->getMethod());
+
$req->setNotSignedUri($preload->getHref());
+ $signedRequests[] =
$req;
break;
default:
@ob_end_clean();
@@ -394,7 +400,17 @@
echo "<html><body><h1>"
. "500 - Internal Server Error" . "</h1></body></html>";
die();
}
- $resp[$preload->getHref()] = array(
+ }
+ } catch (Exception $e) {
+ throw new Exception($e);
+ }
+ }
+ if (count($unsignedRequests)) {
+ try {
+ $brc = new BasicRemoteContent();
+ $responses =
$brc->multiFetch($unsignedRequests, $unsignedContexts);
+ foreach ($responses as $response) {
+ $resp[$response->getUrl()] = array(
'body' =>
$response->getResponseContent(),
'rc' =>
$response->getHttpCode());
}
@@ -402,8 +418,20 @@
throw new Exception($e);
}
}
+ if (count($signedRequests)) {
+ try {
+ $fetcher =
$signingFetcherFactory->getSigningFetcher(new BasicRemoteContentFetcher(),
$token);
+ $responses =
$fetcher->multiFetchRequest($signedRequests);
+ foreach ($responses as $response) {
+ $resp[$response->getNotSignedUrl()] =
array(
+ 'body' =>
$response->getResponseContent(),
+ 'rc' =>
$response->getHttpCode());
+ }
+ } catch (Exception $e) {
+ throw new Exception($e);
+ }
+ }
$resp = count($resp) ? json_encode($resp) : "{}";
return "gadgets.io.preloaded_ = " . $resp . ";\n";
}
-
-}
\ No newline at end of file
+}
