Yaron Koren has uploaded a new change for review. https://gerrit.wikimedia.org/r/204774
Change subject: Data recreation now done via Ajax, using 2 new API actions ...................................................................... Data recreation now done via Ajax, using 2 new API actions Change-Id: Id3daf081f226807636833d36128e3665617b82b8 --- M Cargo.php A CargoRecreateDataAPI.php R CargoRecreateTablesAPI.php M i18n/en.json M specials/CargoRecreateData.php 5 files changed, 295 insertions(+), 108 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Cargo refs/changes/74/204774/1 diff --git a/Cargo.php b/Cargo.php index 73944e5..87dad7f 100644 --- a/Cargo.php +++ b/Cargo.php @@ -25,7 +25,6 @@ $cgScriptPath = $wgScriptPath . '/extensions/Cargo'; $wgJobClasses['cargoPopulateTable'] = 'CargoPopulateTableJob'; -$wgJobClasses['cargoRecreateTables'] = 'CargoRecreateTablesJob'; $wgHooks['ParserFirstCallInit'][] = 'cargoRegisterParserFunctions'; $wgHooks['MakeGlobalVariablesScript'][] = 'CargoHooks::setGlobalJSVariables'; @@ -52,6 +51,8 @@ // API modules $wgAPIModules['cargoquery'] = 'CargoQueryAPI'; +$wgAPIModules['cargorecreatetables'] = 'CargoRecreateTablesAPI'; +$wgAPIModules['cargorecreatedata'] = 'CargoRecreateDataAPI'; // Register classes and special pages. $wgAutoloadClasses['CargoHooks'] = $dir . '/Cargo.hooks.php'; @@ -68,7 +69,6 @@ $wgAutoloadClasses['CargoRecurringEvent'] = $dir . '/parserfunctions/CargoRecurringEvent.php'; $wgAutoloadClasses['CargoDisplayMap'] = $dir . '/parserfunctions/CargoDisplayMap.php'; $wgAutoloadClasses['CargoPopulateTableJob'] = $dir . '/CargoPopulateTableJob.php'; -$wgAutoloadClasses['CargoRecreateTablesJob'] = $dir . '/CargoRecreateTablesJob.php'; $wgAutoloadClasses['CargoRecreateDataAction'] = $dir . '/CargoRecreateDataAction.php'; $wgAutoloadClasses['CargoRecreateData'] = $dir . '/specials/CargoRecreateData.php'; $wgSpecialPages['CargoTables'] = 'CargoTables'; @@ -84,6 +84,8 @@ $wgSpecialPages['PageValues'] = 'CargoPageValues'; $wgAutoloadClasses['CargoPageValues'] = $dir . '/specials/CargoPageValues.php'; $wgAutoloadClasses['CargoQueryAPI'] = $dir . '/CargoQueryAPI.php'; +$wgAutoloadClasses['CargoRecreateTablesAPI'] = $dir . '/CargoRecreateTablesAPI.php'; +$wgAutoloadClasses['CargoRecreateDataAPI'] = $dir . '/CargoRecreateDataAPI.php'; // Display formats $wgAutoloadClasses['CargoDisplayFormat'] = $dir . '/formats/CargoDisplayFormat.php'; diff --git a/CargoRecreateDataAPI.php b/CargoRecreateDataAPI.php new file mode 100644 index 0000000..417bf2f --- /dev/null +++ b/CargoRecreateDataAPI.php @@ -0,0 +1,90 @@ +<?php +/** + * Adds and handles the 'cargorecreatedata' action to the MediaWiki API. + * + * @ingroup Cargo + * @author Yaron Koren + */ + +class CargoRecreateDataAPI extends ApiBase { + + public function __construct( $query, $moduleName ) { + parent::__construct( $query, $moduleName ); + } + + public function execute() { + global $wgUser; + + if ( !$wgUser->isAllowed( 'recreatecargodata' ) || $wgUser->isBlocked() ) { + $this->dieUsageMsg( array( 'badaccess-groups' ) ); + } + + $params = $this->extractRequestParams(); + $templateStr = $params['template']; + $tableStr = $params['table']; + + if ( $templateStr == '' ) { + $this->dieUsage( 'The template must be specified', 'param_substr' ); + } + + if ( $tableStr == '' ) { + $this->dieUsage( 'The table must be specified', 'param_substr' ); + } + + // Create the jobs. + $jobParams = array( + 'dbTableName' => $tableStr, + 'replaceOldRows' => $params['replaceOldRows'] + ); + $jobs = array(); + $templateTitle = Title::makeTitleSafe( NS_TEMPLATE, $templateStr ); + $titlesWithThisTemplate = $templateTitle->getTemplateLinksTo( array( + 'LIMIT' => 500, 'OFFSET' => $params['offset'] ) ); + foreach ( $titlesWithThisTemplate as $titleWithThisTemplate ) { + $jobs[] = new CargoPopulateTableJob( $titleWithThisTemplate, $jobParams ); + } + JobQueueGroup::singleton()->push( $jobs ); + + // Set top-level elements. + $result = $this->getResult(); + $result->addValue( null, 'success', true ); + } + + protected function getAllowedParams() { + return array( + 'template' => array( + ApiBase::PARAM_TYPE => 'string', + ), + 'table' => array( + ApiBase::PARAM_TYPE => 'string', + ), + 'offset' => array( + ApiBase::PARAM_TYPE => 'integer', + ApiBase::PARAM_DFLT => 0, + ), + 'replaceOldRows' => array( + ApiBase::PARAM_TYPE => 'boolean', + ), + ); + } + + protected function getParamDescription() { + return array( + 'template' => 'The template whose data to use', + 'table' => 'The Cargo database table to repopulate', + 'replaceOldRows' => 'Whether to replace old rows for each page while repopulating the table', + ); + } + + protected function getDescription() { + return 'An API module to recreate data for the Cargo extension ' + . '(http://www.mediawiki.org/Extension:Cargo)'; + } + + protected function getExamples() { + return array( + 'api.php?action=cargorecreatedata&template=City&table=Cities' + ); + } + +} diff --git a/CargoRecreateTablesJob.php b/CargoRecreateTablesAPI.php similarity index 84% rename from CargoRecreateTablesJob.php rename to CargoRecreateTablesAPI.php index 8093e0b..1f48937 100644 --- a/CargoRecreateTablesJob.php +++ b/CargoRecreateTablesAPI.php @@ -1,41 +1,62 @@ <?php /** - * Background job to recreate the database table(s) for one template using the - * data from the call(s) to that template in one page. + * Adds and handles the 'cargorecreatetables' action to the MediaWiki API. * - * @author Yaron Koren * @ingroup Cargo + * @author Yaron Koren */ -class CargoRecreateTablesJob extends Job { +class CargoRecreateTablesAPI extends ApiBase { - /** - * - * @param Title $title - * @param array|bool $params - */ - function __construct( $title, $params = false ) { - parent::__construct( 'cargoRecreateTables', $title, $params ); + public function __construct( $query, $moduleName ) { + parent::__construct( $query, $moduleName ); } - /** - * Run a CargoRecreateTables job. - * - * @return boolean success - */ - function run() { - wfProfileIn( __METHOD__ ); + public function execute() { + global $wgUser; - if ( is_null( $this->title ) ) { - $this->error = "cargoRecreateTables: Invalid title"; - wfProfileOut( __METHOD__ ); - return false; + if ( !$wgUser->isAllowed( 'recreatecargodata' ) || $wgUser->isBlocked() ) { + $this->dieUsageMsg( array( 'badaccess-groups' ) ); } - $templatePageID = $this->title->getArticleID(); + $params = $this->extractRequestParams(); + $templateStr = $params['template']; + if ( $templateStr == '' ) { + $this->dieUsage( 'The template must be specified', 'param_substr' ); + } + + $templateTitle = Title::makeTitleSafe( NS_TEMPLATE, $templateStr ); + $templatePageID = $templateTitle->getArticleID(); $success = self::recreateDBTablesForTemplate( $templatePageID ); - wfProfileOut( __METHOD__ ); - return $success; + + // Set top-level elements. + $result = $this->getResult(); + $result->addValue( null, 'success', true ); + } + + protected function getAllowedParams() { + return array( + 'template' => array( + ApiBase::PARAM_TYPE => 'string', + ), + ); + } + + protected function getParamDescription() { + return array( + 'template' => 'The template whose declared Cargo table(s) should be recreated', + ); + } + + protected function getDescription() { + return 'An API module to recreate tables for the Cargo extension ' + . '(http://www.mediawiki.org/Extension:Cargo)'; + } + + protected function getExamples() { + return array( + 'api.php?action=cargorecreatetables&template=City' + ); } /** diff --git a/i18n/en.json b/i18n/en.json index b7c0fc8..d7b5c29 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -12,7 +12,8 @@ "cargo-createdatatable": "Create data table", "cargo-recreatedata-desc": "Recreate Cargo data for this template?", "cargo-recreatedata-createdata": "Create Cargo data for this template?", - "cargo-recreatedata-success": "The data will be recreated.", + "cargo-recreatedata-tablecreated": "Recreated Cargo table \"$1\".", + "cargo-recreatedata-success": "The data is being recreated.", "viewdata": "View data", "cargotables": "Cargo tables", "cargo-cargotables-tablelist": "The following tables are defined:", diff --git a/specials/CargoRecreateData.php b/specials/CargoRecreateData.php index cba854f..ed1ce11 100644 --- a/specials/CargoRecreateData.php +++ b/specials/CargoRecreateData.php @@ -20,6 +20,8 @@ } function execute( $query = null ) { + global $cgScriptPath; + $out = $this->getOutput(); $this->setHeaders(); @@ -35,85 +37,16 @@ return true; } - $formSubmitted = $this->getRequest()->getText( 'submitted' ) == 'yes'; - if ( $formSubmitted ) { - // Recreate the data! - $this->recreateData(); - // Redirect to the main template page - we need to - // add "action=purge" to the URL so that the new - // "View table" link will show up on the page. - $out->redirect( $this->mTemplateTitle->getFullURL( array( 'action' => 'purge' ) ) ); - return true; - } + $templateData = array(); + $dbr = wfGetDB( DB_SLAVE ); - // Simple form. - // Add in a little bit of JS to make sure that the button - // isn't accidentally pressed twice. - $text = '<form method="post" onSubmit="submitButton.disabled = true; return true;">'; - $msg = $tableExists ? 'cargo-recreatedata-desc' : 'cargo-recreatedata-createdata'; - $text .= Html::element( 'p', null, $this->msg( $msg )->parse() ); - $text .= Html::hidden( 'action', 'recreatedata' ) . "\n"; - $text .= Html::hidden( 'submitted', 'yes' ) . "\n"; - - $text .= Html::input( 'submitButton', $this->msg( 'ok' )->parse(), 'submit' ); - $text .= "\n</form>"; - - $out->addHTML( $text ); - - return true; - } - - function callPopulateTableJobsForTemplate( $templateTitle, $jobParams ) { - // We need to break this up into batches, to avoid running out - // of memory for large page sets. - // @TODO For *really* large page sets, it might make sense - // to create a job for each batch. - $offset = 0; - do { - $jobs = array(); - $titlesWithThisTemplate = $templateTitle->getTemplateLinksTo( array( - 'LIMIT' => 500, 'OFFSET' => $offset ) ); - foreach ( $titlesWithThisTemplate as $titleWithThisTemplate ) { - $jobs[] = new CargoPopulateTableJob( $titleWithThisTemplate, $jobParams ); - } - $offset += 500; - JobQueueGroup::singleton()->push( $jobs ); - } while ( count( $titlesWithThisTemplate ) >= 500 ); - } - - /** - * Calls jobs to drop and recreate the table(s) for this template. - */ - function recreateData() { - // If this template calls #cargo_declare (as opposed to - // #cargo_attach), drop and re-generate the Cargo DB table - // for it.` - if ( $this->mIsDeclared ) { - // Call this directly, instead of as a job; so that - // the re-creation of the table(s) always happens - // before any of the rows are added. - // Hopefully this will not ever cause the session to - // time out. - - //$job = new CargoRecreateTablesJob( $this->mTemplateTitle ); - //JobQueueGroup::singleton()->push( $job ); - CargoRecreateTablesJob::recreateDBTablesForTemplate( $this->mTemplateTitle->getArticleID() ); - } - - // Now create a job, CargoPopulateTable, for each page - // that calls this template. - $jobParams = array( - 'dbTableName' => $this->mTableName, - 'replaceOldRows' => !$this->mIsDeclared + $templateData[] = array( + 'name' => $this->mTemplateTitle->getText(), + 'numPages' => $this->getNumPagesThatCallTemplate( $dbr, $this->mTemplateTitle ) ); - $this->callPopulateTableJobsForTemplate( $this->mTemplateTitle, $jobParams ); - - // If this template calls #cargo_declare, see if any templates - // have attached themselves to this table, and if so, call - // this job for their pages as well. if ( $this->mIsDeclared ) { - $dbr = wfGetDB( DB_SLAVE ); + // Get all attached templates. $res = $dbr->select( 'page_props', array( 'pp_page' @@ -126,13 +59,153 @@ while ( $row = $dbr->fetchRow( $res ) ) { $templateID = $row['pp_page']; $attachedTemplateTitle = Title::newFromID( $templateID ); - $jobParams = array( - 'dbTableName' => $this->mTableName, - 'replaceOldRows' => false + $numPages = $this->getNumPagesThatCallTemplate( $dbr, $attachedTemplateTitle ); + $attachedTemplateName = $attachedTemplateTitle->getText(); + $templateData[] = array( + 'name' => $attachedTemplateName, + 'numPages' => $numPages ); - - $this->callPopulateTableJobsForTemplate( $attachedTemplateTitle, $jobParams ); + $pagesPerAttachedTemplate[$attachedTemplateName] = $numPages; } } + + $templateDataJS = json_encode( $templateData ); + $recreateTableDoneMsg = wfMessage( 'cargo-recreatedata-tablecreated', $this->mTableName )->text(); + $recreateDataDoneMsg = wfMessage( 'cargo-recreatedata-success' )->text(); + + $jsText = <<<END +<script type="text/javascript"> +var cargoScriptPath = "$cgScriptPath"; +var tableName = "{$this->mTableName}"; +var templateData = $templateDataJS; +var recreateTableDoneMsg = '$recreateTableDoneMsg'; +var recreateDataDoneMsg = '$recreateDataDoneMsg'; +var numTotalPages = 0; +var numTotalPagesHandled = 0; + +for ( var i = 0; i < templateData.length; i++ ) { + numTotalPages += parseInt( templateData[i]['numPages'] ); +} + + +function cargoReplaceRecreateDataForm() { + $("#recreateDataCanvas").html( "<div id=\"recreateTableProgress\"></div>" ); + $("#recreateDataCanvas").append( "<div id=\"recreateDataProgress\"></div>" ); +} + +/** + * Recursive function that uses Ajax to populate a Cargo DB table with the + * data for one or more templates. + */ +function cargoCreateJobs( templateNum, numPagesHandled, replaceOldRows ) { + var curTemplate = templateData[templateNum]; + var templateName = curTemplate['name']; + var numPages = curTemplate['numPages']; + if ( numTotalPages > 1000 ) { + var remainingPixels = 100 * numTotalPagesHandled / numTotalPages; + var progressImage = "<progress value=\"" + remainingPixels + "\" max=\"100\"></progress>"; + } else { + var progressImage = "<img src=\"" + cargoScriptPath + "/skins/loading.gif\" />"; } + $("#recreateDataProgress").html( "<p>" + progressImage + "</p>" ); + var queryStringData = { + //action: "cargorecreatedata", + table: tableName, + template: templateName, + offset: numPagesHandled + }; + if ( replaceOldRows ) { + queryStringData['replaceOldRows'] = true; + } + $.get( + "/w/api.php", + queryStringData + ) + .done(function( msg ) { + newNumPagesHandled = Math.min( numPagesHandled + 500, numPages ); + numTotalPagesHandled += newNumPagesHandled - numPagesHandled; + if ( newNumPagesHandled < numPages ) { + cargoCreateJobs( templateNum, newNumPagesHandled, replaceOldRows ); + } else { + if ( templateNum + 1 < templateData.length ) { + cargoCreateJobs( templateNum + 1, 0, replaceOldRows ); + } else { + // We're done. + $("#recreateDataProgress").html( "<p>" + recreateDataDoneMsg + "</p>" ); + } + } + }); +} + +END; + + if ( $this->mIsDeclared ) { + $jsText .= <<<END +$( "#cargoSubmit" ).click( function() { + cargoReplaceRecreateDataForm(); + + var templateName = templateData[0]['name']; + $("#recreateTableProgress").html( "<img src=\"" + cargoScriptPath + "/skins/loading.gif\" />" ); + $.get( + "/w/api.php", + { /*action: "cargorecreatetables",*/ template: templateName } + ) + .done(function( msg ) { + $("#recreateTableProgress").html( "<p>" + recreateTableDoneMsg + "</p>" ); + cargoCreateJobs( 0, 0, false ); + }); +}); +</script> + +END; + } else { + $jsText .= <<<END +$( "#cargoSubmit" ).click( function() { + cargoReplaceRecreateDataForm(); + cargoCreateJobs( 0, 0, true ); +}); +</script> + +END; + } + $out->addScript( $jsText ); + + $formSubmitted = $this->getRequest()->getText( 'submitted' ) == 'yes'; + if ( $formSubmitted ) { + // Recreate the data! + $this->recreateData(); + // Redirect to the main template page - we need to + // add "action=purge" to the URL so that the new + // "View table" link will show up on the page. + $out->redirect( $this->mTemplateTitle->getFullURL( array( 'action' => 'purge' ) ) ); + return true; + } + + // Simple form. + $text = '<div id="recreateDataCanvas">' . "\n"; + $msg = $tableExists ? 'cargo-recreatedata-desc' : 'cargo-recreatedata-createdata'; + $text .= Html::element( 'p', null, $this->msg( $msg )->parse() ); + $text .= Html::element( 'button', array( 'id' => 'cargoSubmit' ), $this->msg( 'ok' )->parse() ); + $text .= "\n</div>"; + + $out->addHTML( $text ); + + return true; + } + + function getNumPagesThatCallTemplate( $dbr, $templateTitle ) { + $res = $dbr->select( + array( 'page', 'templatelinks' ), + 'COUNT(*)', + array( + "tl_from=page_id", + "tl_namespace" => $templateTitle->getNamespace(), + "tl_title" => $templateTitle->getDBkey() ), + __METHOD__, + array() + ); + $row = $dbr->fetchRow( $res ); + return $row[0]; + } + } -- To view, visit https://gerrit.wikimedia.org/r/204774 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Id3daf081f226807636833d36128e3665617b82b8 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/Cargo Gerrit-Branch: master Gerrit-Owner: Yaron Koren <yaro...@gmail.com> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits