Cscott has uploaded a new change for review. https://gerrit.wikimedia.org/r/214351
Change subject: WIP: Use Parsoid v2 API. ...................................................................... WIP: Use Parsoid v2 API. Update the ParsoidVirtualRESTService and the RestbaseVirtualRESTService to use Parsoid's v2 API, instead of the deprecated v1 API. Since Visual Editor still issues requests using the v1 API, convert v1 API requests into v2 API requests when needed for a smooth transition. The next step will be to convert Visual Editor to issue v2 API requests, and then the v1->v2 conversion code added here can be removed. TO DO: x Test Parsoid v1->v2 conversion [done] . Test Parsoid v1->Restbase conversion . Update VE to use Parsoid v2 API . Test Parsoid v2 (no conversion) . Test Parsoid v2->Restbase conversion Change-Id: I07ac60cdec7a52ef93187d40099325a069e3239a --- M includes/libs/virtualrest/ParsoidVirtualRESTService.php M includes/libs/virtualrest/RestbaseVirtualRESTService.php 2 files changed, 348 insertions(+), 127 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/51/214351/1 diff --git a/includes/libs/virtualrest/ParsoidVirtualRESTService.php b/includes/libs/virtualrest/ParsoidVirtualRESTService.php index 32a27f7..7c5670f 100644 --- a/includes/libs/virtualrest/ParsoidVirtualRESTService.php +++ b/includes/libs/virtualrest/ParsoidVirtualRESTService.php @@ -25,17 +25,21 @@ class ParsoidVirtualRESTService extends VirtualRESTService { /** * Example requests: - * GET /local/v1/page/$title/html/$oldid - * * $oldid is optional - * POST /local/v1/transform/html/to/wikitext/$title/$oldid + * GET /v2/local/html/$title/{$revision} + * * $revision is optional + * POST /v2/local/wt/{$title}/{$revision} * * body: array( 'html' => ... ) - * * $title and $oldid are optional - * POST /local/v1/transform/wikitext/to/html/$title + * * $title and $revision are optional + * POST /v2/local/html/{$title}/{$revision} * * body: array( 'wikitext' => ... ) or array( 'wikitext' => ..., 'body' => true/false ) * * $title is optional + * * $revision is optional + * There are also deprecated "v1" requests; see onParsoid1Request + * for details. * @param array $params Key/value map * - url : Parsoid server URL - * - prefix : Parsoid prefix for this wiki + * - prefix : Parsoid prefix for this wiki (v1 requests only) + * - domain : Wiki domain to use (v2 requests only) * - timeout : Parsoid timeout (optional) * - forwardCookies : Cookies to forward to Parsoid, or false. (optional) * - HTTPProxy : Parsoid HTTP proxy (optional) @@ -46,7 +50,27 @@ $params['url'] = $params['URL']; unset( $params['URL'] ); } - parent::__construct( $params ); + // set up defaults and merge them with the given params + $mparams = array_merge( array( + 'url' => 'http://localhost:8000/', + 'prefix' => 'localhost', + 'domain' => 'localhost', + 'forwardCookies' => false, + 'HTTPProxy' => null, + ), $params ); + // Ensure the correct url format. + $mparams['url'] = preg_replace( + '#/?$#', + '/', + $mparams['url'] + ); + // Ensure the correct domain format. + $mparams['domain'] = preg_replace( + '/^(https?:\/\/)?([^\/:]+?)(\/|:\d+\/?)?$/', + '$2', + $mparams['domain'] + ); + parent::__construct( $mparams ); } public function onRequests( array $reqs, Closure $idGeneratorFunc ) { @@ -55,70 +79,155 @@ $parts = explode( '/', $req['url'] ); list( + $version, // 'v1' or 'v2' $targetWiki, // 'local' - $version, // 'v1' - $reqType // 'page' or 'transform' + $format, // 'html' or 'wt' + // $title (optional) + // $revision (optional) ) = $parts; + if ( $version !== 'v2' ) { + $result[$key] = $this->onParsoid1Request( $req, $idGeneratorFunc ); + continue; + } if ( $targetWiki !== 'local' ) { + throw new Exception( "Only 'local' target wiki is currently supported" ); - } elseif ( $version !== 'v1' ) { - throw new Exception( "Only version 1 exists" ); - } elseif ( $reqType !== 'page' && $reqType !== 'transform' ) { - throw new Exception( "Request type must be either 'page' or 'transform'" ); } - - $req['url'] = $this->params['url'] . '/' . urlencode( $this->params['prefix'] ) . '/'; - - if ( $reqType === 'page' ) { - $title = $parts[3]; - if ( $parts[4] !== 'html' ) { - throw new Exception( "Only 'html' output format is currently supported" ); - } - if ( isset( $parts[5] ) ) { - $req['url'] .= $title . '?oldid=' . $parts[5]; - } else { - $req['url'] .= $title; - } - } elseif ( $reqType === 'transform' ) { - if ( $parts[4] !== 'to' ) { - throw new Exception( "Part index 4 is not 'to'" ); - } - - if ( isset( $parts[6] ) ) { - $req['url'] .= $parts[6]; - } - - if ( $parts[3] === 'html' & $parts[5] === 'wikitext' ) { - if ( !isset( $req['body']['html'] ) ) { - throw new Exception( "You must set an 'html' body key for this request" ); - } - if ( isset( $parts[7] ) ) { - $req['body']['oldid'] = $parts[7]; - } - } elseif ( $parts[3] == 'wikitext' && $parts[5] == 'html' ) { - if ( !isset( $req['body']['wikitext'] ) ) { - throw new Exception( "You must set a 'wikitext' body key for this request" ); - } - $req['body']['wt'] = $req['body']['wikitext']; - unset( $req['body']['wikitext'] ); - } else { - throw new Exception( "Transformation unsupported" ); - } + if ( $format !== 'html' && $format !== 'wt' ) { + throw new Exception( "Request format must be either 'html' or 'wt'" ); } - - if ( isset( $this->params['HTTPProxy'] ) && $this->params['HTTPProxy'] ) { + // replace /local/ with the current domain + $req['url'] = preg_replace( '#^v2/local/#', 'v2/' . $this->params['domain'] . '/', $req['url'] ); + // and prefix it with the service URL + $req['url'] = $this->params['url'] . $req['url']; + // set the appropriate proxy, timeout and headers + if ( $this->params['HTTPProxy'] ) { $req['proxy'] = $this->params['HTTPProxy']; } - if ( isset( $this->params['timeout'] ) ) { + if ( $this->params['timeout'] != null ) { $req['reqTimeout'] = $this->params['timeout']; } - - // Forward cookies - if ( isset( $this->params['forwardCookies'] ) ) { + if ( $this->params['forwardCookies'] ) { $req['headers']['Cookie'] = $this->params['forwardCookies']; } + $result[$key] = $req; + } + return $result; + } + /** + * Remap a Parsoid v1 request to a Parsoid v2 path + * + * Example requests: + * GET /local/v1/page/$title/html/$oldid + * * $oldid is optional + * POST /local/v1/transform/html/to/wikitext/$title/$oldid + * * body: array( 'html' => ... ) + * * $title and $oldid are optional + * POST /local/v1/transform/wikitext/to/html/$title + * * body: array( 'wikitext' => ... ) or array( 'wikitext' => ..., 'body' => true/false ) + * * $title is optional + * + * NOTE: the POST APIs aren't "real" Parsoid v1 APIs, they are just what + * Visual Editor "pretends" the V1 API is like. A previous version of + * ParsoidVirtualRESTService translated these to the "real" Parsoid v1 + * API. We now translate these to the "real" Parsoid v2 API. + */ + public function onParsoid1Request( array $req, Closure $idGeneratorFunc ) { + + $parts = explode( '/', $req['url'] ); + list( + $targetWiki, // 'local' + $version, // 'v1' + $reqType // 'page' or 'transform' + ) = $parts; + if ( $targetWiki !== 'local' ) { + throw new Exception( "Only 'local' target wiki is currently supported" ); + } elseif ( $reqType !== 'page' && $reqType !== 'transform' ) { + throw new Exception( "Request type must be either 'page' or 'transform'" ); + } + $req['url'] = $this->params['url'] . 'v2/' . $this->params['domain'] . '/'; + if ( $reqType === 'page' ) { + $title = $parts[3]; + if ( $parts[4] !== 'html' ) { + throw new Exception( "Only 'html' output format is currently supported" ); + } + $req['url'] .= 'html/' . $title; + if ( isset( $parts[5] ) ) { + $req['url'] .= '/' . $parts[5]; + } elseif ( isset( $req['query']['oldid'] ) && $req['query']['oldid'] ) { + $req['url'] .= '/' . $req['query']['oldid']; + unset( $req['query']['oldid'] ); + } + } elseif ( $reqType === 'transform' ) { + // from / to transform + if ( $parts[5] === 'wikitext' ) { + // but in parsoid v2 api, "wikitext" is called "wt" + $req['url'] .= 'wt'; + // and the response is JSON, not wikitext (text/plain) + $req['FIXUPv1'] = true; + } else { + $req['url'] .= $parts[5]; + } + // the title + if ( isset( $parts[6] ) ) { + $req['url'] .= '/' . $parts[6]; + } + // revision id + if ( isset( $parts[7] ) ) { + $req['url'] .= '/' . $parts[7]; + } elseif ( isset( $req['body']['oldid'] ) && $req['body']['oldid'] ) { + $req['url'] .= '/' . $req['body']['oldid']; + unset( $req['body']['oldid'] ); + } + if ( $parts[4] !== 'to' ) { + throw new Exception( "Part index 4 is not 'to'" ); + } + if ( $parts[3] === 'html' && $parts[5] === 'wikitext' ) { + if ( !isset( $req['body']['html'] ) ) { + throw new Exception( "You must set an 'html' body key for this request" ); + } + } elseif ( $parts[3] == 'wikitext' && $parts[5] == 'html' ) { + if ( !isset( $req['body']['wikitext'] ) ) { + throw new Exception( "You must set a 'wikitext' body key for this request" ); + } + } else { + throw new Exception( "Transformation unsupported" ); + } + } + // set the appropriate proxy, timeout and headers + if ( $this->params['HTTPProxy'] ) { + $req['proxy'] = $this->params['HTTPProxy']; + } + if ( $this->params['timeout'] != null ) { + $req['reqTimeout'] = $this->params['timeout']; + } + if ( $this->params['forwardCookies'] ) { + $req['headers']['Cookie'] = $this->params['forwardCookies']; + } + + return $req; + + } + + public function onResponses( array $reqs, Closure $idGeneratorFunc ) { + // Look for requests with 'FIXUPv1' set (these are v1 html2wt + // requests) and tweak the response. + $result = array(); + foreach ( $reqs as $key => $req ) { + if ( $req['FIXUPv1'] && $req['response']['code'] === 200 ) { + // Decode the JSON response and fake a v1 API response. + $body = FormatJson::parse( $req['response']['body'], FormatJson::FORCE_ASSOC ); + if ( $body->isGood() ) { + $resp = $body->getValue()['wikitext']; + $req['response']['headers']['content-type'] = $resp['headers']['content-type']; + $req['response']['body'] = $resp['body']; + } else { + // JSON parsing didn't succeed, something is wrong. + $req['response']['code'] = 500; + } + } $result[$key] = $req; } return $result; diff --git a/includes/libs/virtualrest/RestbaseVirtualRESTService.php b/includes/libs/virtualrest/RestbaseVirtualRESTService.php index 8fe5b92..621d2ab 100644 --- a/includes/libs/virtualrest/RestbaseVirtualRESTService.php +++ b/includes/libs/virtualrest/RestbaseVirtualRESTService.php @@ -48,14 +48,20 @@ public function __construct( array $params ) { // set up defaults and merge them with the given params $mparams = array_merge( array( - 'url' => 'http://localhost:7231', + 'url' => 'http://localhost:7231/', 'domain' => 'localhost', 'timeout' => 100, 'forwardCookies' => false, 'HTTPProxy' => null, 'parsoidCompat' => false ), $params ); - // ensure the correct domain format + // Ensure the correct url format (trailing slash). + $mparams['url'] = preg_replace( + '#/?$#', + '/' + $mparams['url'] + ); + // Ensure the correct domain format. $mparams['domain'] = preg_replace( '/^(https?:\/\/)?([^\/:]+?)(\/|:\d+\/?)?$/', '$2', @@ -73,7 +79,7 @@ $result = array(); foreach ( $reqs as $key => $req ) { // replace /local/ with the current domain - $req['url'] = preg_replace( '/^\/local\//', '/' . $this->params['domain'] . '/', $req['url'] ); + $req['url'] = preg_replace( '#^local/#', $this->params['domain'] . '/', $req['url'] ); // and prefix it with the service URL $req['url'] = $this->params['url'] . $req['url']; // set the appropriate proxy, timeout and headers @@ -94,84 +100,190 @@ } /** - * Remaps Parsoid requests to Restbase paths + * Remaps Parsoid v1/v2 requests to Restbase paths */ public function onParsoidRequests( array $reqs, Closure $idGeneratorFunc ) { $result = array(); foreach ( $reqs as $key => $req ) { $parts = explode( '/', $req['url'] ); - list( - $targetWiki, // 'local' - $version, // 'v1' - $reqType // 'page' or 'transform' - ) = $parts; - if ( $targetWiki !== 'local' ) { - throw new Exception( "Only 'local' target wiki is currently supported" ); - } elseif ( $reqType !== 'page' && $reqType !== 'transform' ) { - throw new Exception( "Request type must be either 'page' or 'transform'" ); + if ( $parts[0] === 'v2' ) { + $result[$key] = onParsoid2Request( $req, $idGeneratorFunc ); + } elseif ( $parts[1] === 'v1' ) { + $result[$key] = onParsoid1Request( $req, $idGeneratorFunc ); + } else { + throw new Exception( "Only v1 and v2 are supported." ); } - $req['url'] = $this->params['url'] . '/' . $this->params['domain'] . '/v1/' . $reqType . '/'; - if ( $reqType === 'page' ) { - $title = $parts[3]; - if ( $parts[4] !== 'html' ) { - throw new Exception( "Only 'html' output format is currently supported" ); - } - $req['url'] .= 'html/' . $title; - if ( isset( $parts[5] ) ) { - $req['url'] .= '/' . $parts[5]; - } elseif ( isset( $req['query']['oldid'] ) && $req['query']['oldid'] ) { - $req['url'] .= '/' . $req['query']['oldid']; - unset( $req['query']['oldid'] ); - } - } elseif ( $reqType === 'transform' ) { - // from / to transform - $req['url'] .= $parts[3] . '/to/' . $parts[5]; - // the title - if ( isset( $parts[6] ) ) { - $req['url'] .= '/' . $parts[6]; - } - // revision id - if ( isset( $parts[7] ) ) { - $req['url'] .= '/' . $parts[7]; - } elseif ( isset( $req['body']['oldid'] ) && $req['body']['oldid'] ) { - $req['url'] .= '/' . $req['body']['oldid']; - unset( $req['body']['oldid'] ); - } - if ( $parts[4] !== 'to' ) { - throw new Exception( "Part index 4 is not 'to'" ); - } - if ( $parts[3] === 'html' & $parts[5] === 'wikitext' ) { - if ( !isset( $req['body']['html'] ) ) { - throw new Exception( "You must set an 'html' body key for this request" ); - } - } elseif ( $parts[3] == 'wikitext' && $parts[5] == 'html' ) { - if ( !isset( $req['body']['wikitext'] ) ) { - throw new Exception( "You must set a 'wikitext' body key for this request" ); - } - if ( isset( $req['body']['body'] ) ) { - $req['body']['bodyOnly'] = $req['body']['body']; - unset( $req['body']['body'] ); - } - } else { - throw new Exception( "Transformation unsupported" ); - } - } - // set the appropriate proxy, timeout and headers - if ( $this->params['HTTPProxy'] ) { - $req['proxy'] = $this->params['HTTPProxy']; - } - if ( $this->params['timeout'] != null ) { - $req['reqTimeout'] = $this->params['timeout']; - } - if ( $this->params['forwardCookies'] ) { - $req['headers']['Cookie'] = $this->params['forwardCookies']; - } - $result[$key] = $req; } return $result; } + /** + * Remap a Parsoid v1 request to a Restbase path + * + * Example requests: + * GET /local/v1/page/$title/html/$oldid + * * $oldid is optional + * POST /local/v1/transform/html/to/wikitext/$title/$oldid + * * body: array( 'html' => ... ) + * * $title and $oldid are optional + * POST /local/v1/transform/wikitext/to/html/$title + * * body: array( 'wikitext' => ... ) or array( 'wikitext' => ..., 'body' => true/false ) + * * $title is optional + * + * NOTE: the POST APIs aren't "real" Parsoid v1 APIs, they are just what + * Visual Editor "pretends" the V1 API is like. (See + * ParsoidVirtualRESTService.) + */ + public function onParsoid1Request( array $req, Closure $idGeneratorFunc ) { + $parts = explode( '/', $req['url'] ); + list( + $targetWiki, // 'local' + $version, // 'v1' + $reqType // 'page' or 'transform' + ) = $parts; + if ( $targetWiki !== 'local' ) { + throw new Exception( "Only 'local' target wiki is currently supported" ); + } elseif ( $version !== 'v1' ) { + throw new Exception( "Version mismatch: should not happen." ); + } elseif ( $reqType !== 'page' && $reqType !== 'transform' ) { + throw new Exception( "Request type must be either 'page' or 'transform'" ); + } + $req['url'] = $this->params['url'] . '/' . $this->params['domain'] . '/v1/' . $reqType . '/'; + if ( $reqType === 'page' ) { + $title = $parts[3]; + if ( $parts[4] !== 'html' ) { + throw new Exception( "Only 'html' output format is currently supported" ); + } + $req['url'] .= 'html/' . $title; + if ( isset( $parts[5] ) ) { + $req['url'] .= '/' . $parts[5]; + } elseif ( isset( $req['query']['oldid'] ) && $req['query']['oldid'] ) { + $req['url'] .= '/' . $req['query']['oldid']; + unset( $req['query']['oldid'] ); + } + } elseif ( $reqType === 'transform' ) { + // from / to transform + $req['url'] .= $parts[3] . '/to/' . $parts[5]; + // the title + if ( isset( $parts[6] ) ) { + $req['url'] .= '/' . $parts[6]; + } + // revision id + if ( isset( $parts[7] ) ) { + $req['url'] .= '/' . $parts[7]; + } elseif ( isset( $req['body']['oldid'] ) && $req['body']['oldid'] ) { + $req['url'] .= '/' . $req['body']['oldid']; + unset( $req['body']['oldid'] ); + } + if ( $parts[4] !== 'to' ) { + throw new Exception( "Part index 4 is not 'to'" ); + } + if ( $parts[3] === 'html' & $parts[5] === 'wikitext' ) { + if ( !isset( $req['body']['html'] ) ) { + throw new Exception( "You must set an 'html' body key for this request" ); + } + } elseif ( $parts[3] == 'wikitext' && $parts[5] == 'html' ) { + if ( !isset( $req['body']['wikitext'] ) ) { + throw new Exception( "You must set a 'wikitext' body key for this request" ); + } + if ( isset( $req['body']['body'] ) ) { + $req['body']['bodyOnly'] = $req['body']['body']; + unset( $req['body']['body'] ); + } + } else { + throw new Exception( "Transformation unsupported" ); + } + } + // set the appropriate proxy, timeout and headers + if ( $this->params['HTTPProxy'] ) { + $req['proxy'] = $this->params['HTTPProxy']; + } + if ( $this->params['timeout'] != null ) { + $req['reqTimeout'] = $this->params['timeout']; + } + if ( $this->params['forwardCookies'] ) { + $req['headers']['Cookie'] = $this->params['forwardCookies']; + } + + return $req; + + } + + /** + * Remap a Parsoid v2 request to a Restbase path + * Example requests: + * GET /v2/local/html/$title/{$revision} + * * $revision is optional + * POST /v2/local/wt/{$title}/{$revision} + * * body: array( 'html' => ... ) + * * $title and $revision are optional + * POST /v2/local/html/{$title}/{$revision} + * * body: array( 'wikitext' => ... ) or array( 'wikitext' => ..., 'body' => true/false ) + * * $title is optional + * * $revision is optional + */ + public function onParsoid2Request( array $req, Closure $idGeneratorFunc ) { + + $parts = explode( '/', $req['url'] ); + list( + $version, // 'v2' + $targetWiki, // 'local' + $format, // 'html' or 'wt' + // $title, // optional + // $revision, // optional + ) = $parts; + $title = isset( $parts[3] ) ? $parts[3] : null; + $revision = isset( $parts[4] ) ? $parts[4] : null; + if ( $targetWiki !== 'local' ) { + throw new Exception( "Only 'local' target wiki is currently supported" ); + } elseif ( $version !== 'v2' ) { + throw new Exception( "Version mismatch: should not happen." ); + } + $req['url'] = $this->params['url'] . '/' . $this->params['domain'] . '/v1/'; + if ( $req['method'] === 'GET' ) { + $req['url'] .= 'page/' . $title . '/' . $format; + if ( $revision !== null) { + $req['url'] .= '/' . $revision; + } + } else if ( $format === 'html' ) { + $req['url'] .= 'transform/wikitext/to/html'; + if ( $title !== null ) { + $req['url'] .= '/' . $title; + if ( $revision !== null ) { + $req['url'] .= '/' . $revision; + } + } + } else if ( $format === 'wt' ) { + $req['url'] .= 'transform/html/to/wikitext'; + if ( $title !== null ) { + $req['url'] .= '/' . $title; + if ( $revision !== null ) { + $req['url'] .= '/' . $revision; + } + } + if ( isset( $req['body']['body'] ) ) { + $req['body']['bodyOnly'] = $req['body']['body']; + unset( $req['body']['body'] ); + } + } else { + throw new Exception( "Request format must be either 'html' or 'wt'" ); + } + // set the appropriate proxy, timeout and headers + if ( $this->params['HTTPProxy'] ) { + $req['proxy'] = $this->params['HTTPProxy']; + } + if ( $this->params['timeout'] != null ) { + $req['reqTimeout'] = $this->params['timeout']; + } + if ( $this->params['forwardCookies'] ) { + $req['headers']['Cookie'] = $this->params['forwardCookies']; + } + + return $req; + + } + } -- To view, visit https://gerrit.wikimedia.org/r/214351 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I07ac60cdec7a52ef93187d40099325a069e3239a Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: master Gerrit-Owner: Cscott <canan...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits