Dan-nl has uploaded a new change for review. https://gerrit.wikimedia.org/r/127839
Change subject: preview without upload ...................................................................... preview without upload the toolset was having an issue retrieving large media files; e.g., > 5mb, during “Step 3: Batch preview” of the batch upload process; it would return a blank page or an error and the background jobs would not run properly or not run at all. i re-factored “Step 3: Batch preview” so that it no longer retrieves mediafiles from the remote server, but instead presents the user with a template preview of the first 3 items. once the user accepts the preview results, the background jobs do the work of retrieving the mediafiles. Bug: 63864 Change-Id: I91fa14730b9614cef4b912b218a692fb19271ce9 --- M i18n/en.json M includes/Forms/PreviewForm.php M includes/Handlers/Forms/MetadataMappingHandler.php M includes/Handlers/UploadHandler.php M includes/Models/MediawikiTemplate.php M resources/css/ext.gwtoolset.css 6 files changed, 296 insertions(+), 104 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/GWToolset refs/changes/39/127839/1 diff --git a/i18n/en.json b/i18n/en.json index 35d9891..d2b8f7f 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -170,7 +170,7 @@ "gwtoolset-template-field": "Template field", "gwtoolset-step-3-instructions-heading": "Step 3: Batch preview", "gwtoolset-step-3-instructions-1": "Below are the results of uploading the {{PLURAL:$1|first record|first $1 records}} from the metadata file you selected and mapping {{PLURAL:$1|it|them}} to the MediaWiki template you selected in \"{{int:gwtoolset-step-2-heading}}\".", - "gwtoolset-step-3-instructions-2": "Review these pages and if the results meet your expectations, and there are additional records waiting to be uploaded, continue the batch upload process by clicking on the \"{{int:gwtoolset-process-batch}}\" button below.", + "gwtoolset-step-3-instructions-2": "Review the templates, and if the results meet your expectations, continue with “Step 4: Batch Upload”, by clicking on the \"{{int:gwtoolset-process-batch}}\" button below.", "gwtoolset-step-3-instructions-3": "If you are not happy with the results, go back to \"{{int:gwtoolset-step-2-heading}}\" and adjust the mapping as necessary.\n\nIf you need to make adjustments to the metadata file itself, go ahead and do so and re-upload it by beginning the process again with \"{{int:gwtoolset-step-1-heading}}\".", "gwtoolset-title-bad": "The title, which was created based on the metadata and the MediaWiki template mapping, is not valid.\n\nTry another field from the metadata for title and title-identifier, or if possible, change the metadata where needed. See [https://commons.wikimedia.org/wiki/Commons:File_naming File naming] for more information.\n\n<strong>Invalid title:</strong> $1.", "gwtoolset-batchjob-metadata-created": "Metadata batch job created. Your metadata file will be analyzed shortly and each item will be uploaded to the wiki in a background process. You can check the page \"$1\" to see when they have been uploaded.", diff --git a/includes/Forms/PreviewForm.php b/includes/Forms/PreviewForm.php index 9962817..78cba01 100644 --- a/includes/Forms/PreviewForm.php +++ b/includes/Forms/PreviewForm.php @@ -12,6 +12,7 @@ Html, IContextSource, Linker, + ParserOptions, Title; class PreviewForm { @@ -26,8 +27,9 @@ * * @param {array} $expected_post_fields * - * @param {array} $mediafile_titles - * a collection of MediaWiki Title objects + * @param {array} $metadata_items + * each item is either a Title object or an array containing + * categories, a Title object, and the wikitext for the item * * @return {string} * an html form that is filtered @@ -36,7 +38,7 @@ IContextSource $Context, array $user_options, array $expected_post_fields, - array $mediafile_titles + array $metadata_items ) { $process_button = (int)$user_options['gwtoolset-record-count'] > (int)Config::$preview_throttle @@ -69,7 +71,6 @@ wfMessage( 'gwtoolset-step-3-instructions-heading' )->escaped() ) . - Html::rawElement( 'p', array(), @@ -77,14 +78,6 @@ ->numParams( (int)Config::$preview_throttle ) ->escaped() ) . - - Html::rawElement( - 'h3', - array(), - wfMessage( 'gwtoolset-results' )->escaped() - ) . - - self::getTitlesAsList( $mediafile_titles ) . Html::openElement( 'form', @@ -119,8 +112,7 @@ array( 'type' => 'hidden', 'name' => 'gwtoolset-record-begin', - // this difference between record-begin and record-current is intentional - 'value' => (int)$user_options['gwtoolset-record-current'] + 'value' => 1 ) ) . @@ -132,17 +124,19 @@ wfMessage( 'gwtoolset-step-3-instructions-2' )->parse() ) . + wfMessage( 'gwtoolset-step-3-instructions-3' )->parse() . + + Html::rawElement( 'ul', array(), $step1_link . $step2_link ) . + Html::rawElement( 'p', array(), $process_button ) . - wfMessage( 'gwtoolset-step-3-instructions-3' )->parse() . - Html::closeElement( 'form' ) . - Html::rawElement( 'ul', array(), $step1_link . $step2_link ); + self::getMetadataAsWikitext( $metadata_items, $Context ); } /** @@ -205,16 +199,16 @@ * Title(s), which are the result of processing the metadata file * with the mapping information given in step 2 : Metadata Mapping * - * @param {array} $mediafile_titles + * @param {array} $metadata_items * a collection of MediaWiki Title objects * * @return {string} * the string contains a Title link assumed to be filtered by Title */ - public static function getTitlesAsList( array $mediafile_titles ) { + public static function getMetadataAsTitleList( array $metadata_items ) { $result = Html::openElement( 'ul' ); - foreach ( $mediafile_titles as $Title ) { + foreach ( $metadata_items as $Title ) { if ( $Title instanceof Title ) { $result .= Html::rawElement( 'li', @@ -229,4 +223,73 @@ return $result; } + + /** + * a decorator method that creates an HTML preview of the metadata items + * after they have been mapped into the chosen MediaWiki template. + * No mediafile is shown in order to avoid issues with downloading + * large mediafiles. + * + * @param {array} $metadata_items + * each item is an array containing + * $item['categories'] {array} + * $item['Title'] {Title} + * $item['wikitext'] {string} + * + * @param {IContextSource} $Context + * + * @return {string} + */ + public static function getMetadataAsWikitext( + array $metadata_items, + IContextSource $Context + ) { + $result = null; + global $wgParser; + $Skin = $Context->getSkin(); + $Output = $Context->getOutput(); + + $parser_options = ParserOptions::newFromContext( $Context ); + $parser_options->setEditSection( false ); + $parser_options->setIsPreview( true ); + + foreach ( $metadata_items as $item ) { + $parser_options->setTargetLanguage( + $item['Title']->getPageLanguage() + ); + + $parser_out = $wgParser->parse( + $item['wikitext'], $item['Title'], + $parser_options + ); + + $lang = $item['Title']->getPageViewLanguage(); + $Output->setCategoryLinks( $item['categories'] ); + + $result .= + Html::openElement( + 'div', + array( + 'class' => 'mw-content-' . $lang->getDir(), + 'dir' => $lang->getDir(), + 'lang' => $lang->getHtmlCode(), + ) + ) . + + Html::rawElement( + 'h2', + array( 'class' => 'preview-title' ), + $item['Title'] + ) . + + $parser_out->getText() . + $Skin->getCategories() . + Html::closeElement( 'div' ); + } + + // set the page caterogies to nothing + $Output->setCategoryLinks( array() ); + + return $result; + } } diff --git a/includes/Handlers/Forms/MetadataMappingHandler.php b/includes/Handlers/Forms/MetadataMappingHandler.php index 3c48f9d..732b03e 100644 --- a/includes/Handlers/Forms/MetadataMappingHandler.php +++ b/includes/Handlers/Forms/MetadataMappingHandler.php @@ -298,9 +298,9 @@ * {array} $options['metadata-mapped-to-mediawiki-template'] * {string} $options['metadata-raw'] * - * @return {null|Title|bool} + * @return {bool|array|null|Title} */ - public function processMatchingElement( array &$user_options, array $options ) { + public function processMatchingElement( array $user_options, array $options ) { $result = null; $this->_MediawikiTemplate->metadata_raw = $options['metadata-raw']; @@ -317,6 +317,8 @@ $options, $this->_whitelisted_post ); + } else if ( $user_options['preview'] ) { + $result = $this->_UploadHandler->getPreview( $user_options ); } else { $result = $this->_UploadHandler->saveMediafileAsContent( $user_options ); } @@ -520,13 +522,13 @@ if ( $user_options['preview'] === true ) { $user_options['gwtoolset-mediafile-throttle'] = (int)Config::$preview_throttle; - $mediafile_titles = $this->processMetadata( $user_options ); + $metadata_items = $this->processMetadata( $user_options ); $result = PreviewForm::getForm( $this->SpecialPage->getContext(), $user_options, $this->_expected_post_fields, - $mediafile_titles + $metadata_items ); } else { $user_options['save-as-batch-job'] = true; diff --git a/includes/Handlers/UploadHandler.php b/includes/Handlers/UploadHandler.php index 7ebeb78..e4c65aa 100644 --- a/includes/Handlers/UploadHandler.php +++ b/includes/Handlers/UploadHandler.php @@ -20,6 +20,7 @@ MimeMagic, MWException, MWHttpRequest, + Status, Title, UploadBase, UploadFromUrl, @@ -36,6 +37,16 @@ * @var {GWToolset\Helpers\GWTFileBackend} */ protected $_GWTFileBackend; + + /** + * @var {array} + */ + protected $_global_categories; + + /** + * @var {array} + */ + protected $_item_specific_categories; /** * @var {GWToolset\Modles\Mapping} @@ -142,23 +153,26 @@ * that are applied to all of the media files being uploaded. * * @return {null|string} - * the resulting wiki text is filtered + * sanitized */ protected function addGlobalCategories() { - $result = - PHP_EOL . PHP_EOL . PHP_EOL . PHP_EOL . - '<!-- Categories -->' . PHP_EOL; + $result = null; - if ( !empty( $this->user_options['categories'] ) ) { - $categories = explode( Config::$category_separator, $this->user_options['categories'] ); + $this->setGlobalCategories(); + $categories = $this->_global_categories; - foreach ( $categories as $category ) { - $result .= - '[[' . - Utils::getNamespaceName( NS_CATEGORY ) . - Utils::stripIllegalCategoryChars( Utils::sanitizeString( $category ) ) . - ']]' . PHP_EOL; - } + if ( empty( $categories ) ) { + return $result; + } + + $result = PHP_EOL . PHP_EOL . '<!-- Global Categories -->' . PHP_EOL; + + foreach ( $categories as $category ) { + $result .= + '[[' . + Utils::getNamespaceName( NS_CATEGORY ) . + Utils::stripIllegalCategoryChars( Utils::sanitizeString( $category ) ) . + ']]' . PHP_EOL; } return $result; @@ -176,42 +190,25 @@ * or only a category-metadata value. * * @return {null|string} - * the resulting wiki text is sanitized + * sanitized */ protected function addItemSpecificCategories() { $result = null; - if ( !empty( $this->user_options['gwtoolset-category-metadata'] ) ) { - $category_count = count( $this->user_options['gwtoolset-category-metadata'] ); + $this->setItemSpecificCategories(); + $categories = $this->_item_specific_categories; - for ( $i = 0; $i < $category_count; $i += 1 ) { - $phrase = null; - $metadata_values = array(); + if ( empty( $categories ) ) { + return $result; + } - if ( !empty( $this->user_options['gwtoolset-category-phrase'][$i] ) ) { - $phrase = - Utils::sanitizeString( - $this->user_options['gwtoolset-category-phrase'][$i] - ) . - ' '; - } + $result = PHP_EOL . PHP_EOL . '<!-- Item Specific Categories -->' . PHP_EOL; - if ( !empty( $this->user_options['gwtoolset-category-metadata'][$i] ) ) { - $metadata_values = - $this->_Metadata->getFieldValuesAsArray( - $this->user_options['gwtoolset-category-metadata'][$i] - ); - } - - foreach( $metadata_values as $metadata_value ) { - $result .= - '[[' . - Utils::getNamespaceName( NS_CATEGORY ) . - Utils::stripIllegalCategoryChars( $phrase ) . - Utils::stripIllegalCategoryChars( $metadata_value ) . - ']]' . PHP_EOL; - } - } + foreach( $categories as $category ) { + $result .= '[[' . + Utils::getNamespaceName( NS_CATEGORY ) . + $category . + ']]' . PHP_EOL; } return $result; @@ -243,9 +240,11 @@ * $url = 'http://images.memorix.nl/gam/thumb/150x150/115165d2-1267-7db5-4abb-54d273c47a81.jpg'; * * @param {string} $url + * * @throws {GWTException} + * * @return {array} - * the values in the array are not filtered + * the values in the array are not filtered * $result['content-type'] * $result['extension'] * $result['url'] @@ -289,6 +288,27 @@ throw new GWTException( array( 'gwtoolset-mapping-media-file-url-extension-bad' => array( $url ) ) ); + } + + return $result; + } + + /** + * @return {array} + */ + protected function getCategoriesForPreview() { + $result = array(); + + $categories = array_merge( + $this->_global_categories, + $this->_item_specific_categories + ); + + // $Output->setCategoryLinks requires an array with the category name + // as the key and a sortkey as the value; not sure what the are valid + // sortkey values, but 0 seems to work well + foreach( $categories as $category ) { + $result[$category] = 0; } return $result; @@ -348,6 +368,36 @@ } /** + * @return {array} + */ + protected function getUploadParams() { + $result = array(); + + $result['gwtoolset-url-to-the-media-file'] = $this->_MediawikiTemplate->mediawiki_template_array['gwtoolset-url-to-the-media-file']; + $evaluated_url = $this->evaluateMediafileUrl( $result['gwtoolset-url-to-the-media-file'] ); + $result['gwtoolset-url-to-the-media-file'] = $evaluated_url['url']; + $result['evaluated-media-file-extension'] = $evaluated_url['extension']; + + $result['title'] = $this->_MediawikiTemplate->getTitle( $result ); + $result['ignorewarnings'] = true; + $result['watch'] = true; + + $result['comment'] = + wfMessage( 'gwtoolset-create-mediafile' ) + ->params( + wfMessage( 'gwtoolset-create-prefix' )->text(), + $this->_User->getName() + ) + ->text() . + PHP_EOL . + trim( $this->user_options['comment'] ); + + $result['text'] = $this->getWikiText(); + + return $result; + } + + /** * creates the wiki text for the media file page. * concatenates several pieces of information in order to create the wiki * text for the mediafile wiki text @@ -362,6 +412,7 @@ PHP_EOL . PHP_EOL . PHP_EOL . PHP_EOL . $this->_MediawikiTemplate->getGWToolsetTemplateAsWikiText() . $this->addMetadata() . + PHP_EOL . PHP_EOL . $this->addGlobalCategories() . $this->addItemSpecificCategories(); } @@ -427,6 +478,24 @@ } /** + * @param {array} $user_options + * @return {array} + */ + public function getPreview( array $user_options ) { + $this->validateUserOptions( $user_options ); + $this->user_options = $user_options; + + $upload_params = $this->getUploadParams(); + $this->validateUploadParams( $upload_params ); + + return array( + 'categories' => $this->getCategoriesForPreview(), + 'Title' => $this->getTitle( $upload_params['title'] ), + 'wikitext' => $upload_params['text'] + ); + } + + /** * @todo does ContentHandler filter $options['text']? * @todo does WikiPage filter $options['comment']? * @@ -434,52 +503,29 @@ * @throws {GWTException} * @return {null|Title} */ - public function saveMediafileAsContent( array &$user_options ) { - $Title = null; - $Status = null; - $options = array(); + public function saveMediafileAsContent( array $user_options ) { + $Status = Status::newGood(); $this->validateUserOptions( $user_options ); $this->user_options = $user_options; - $options['gwtoolset-url-to-the-media-file'] = - $this->_MediawikiTemplate->mediawiki_template_array['gwtoolset-url-to-the-media-file']; + $upload_params = $this->getUploadParams(); + $this->validateUploadParams( $upload_params ); - $evaluated_url = $this->evaluateMediafileUrl( $options['gwtoolset-url-to-the-media-file'] ); - $options['gwtoolset-url-to-the-media-file'] = $evaluated_url['url']; - $options['evaluated-media-file-extension'] = $evaluated_url['extension']; - - $options['title'] = $this->_MediawikiTemplate->getTitle( $options ); - $options['ignorewarnings'] = true; - $options['watch'] = true; - $options['comment'] = - wfMessage( 'gwtoolset-create-mediafile' ) - ->params( - wfMessage( 'gwtoolset-create-prefix' )->text(), - $this->_User->getName() - ) - ->text() . - PHP_EOL . - trim( $this->user_options['comment'] ); - - $options['text'] = $this->getWikiText(); - - WikiChecks::increaseHTTPTimeout(); - $this->validatePageOptions( $options ); - $Title = $this->getTitle( $options['title'] ); + $Title = $this->getTitle( $upload_params['title'] ); if ( !$Title->isKnown() ) { - $Status = $this->uploadMediaFileViaUploadFromUrl( $options, $Title ); + $Status = $this->uploadMediaFileViaUploadFromUrl( $upload_params, $Title ); } else { if ( $this->user_options['gwtoolset-reupload-media'] === true ) { // this will re-upload the mediafile, but will not change the page contents - $Status = $this->uploadMediaFileViaUploadFromUrl( $options, $Title ); + $Status = $this->uploadMediaFileViaUploadFromUrl( $upload_params, $Title ); } - if ( $Status === null || $Status->isOk() ) { - $Content = ContentHandler::makeContent( $options['text'], $Title ); + if ( $Status->isOk() ) { + $Content = ContentHandler::makeContent( $upload_params['text'], $Title ); $Page = new WikiPage( $Title ); - $Status = $Page->doEditContent( $Content, $options['comment'], 0, false, $this->_User ); + $Status = $Page->doEditContent( $Content, $upload_params['comment'], 0, false, $this->_User ); } } @@ -491,7 +537,7 @@ $this->_MediawikiTemplate->mediawiki_template_array['gwtoolset-url-to-the-media-file'] ) . PHP_EOL . 'evaluated URL: ' . - Utils::sanitizeUrl( $options['gwtoolset-url-to-the-media-file'] ) . PHP_EOL; + Utils::sanitizeUrl( $upload_params['gwtoolset-url-to-the-media-file'] ) . PHP_EOL; throw new GWTException( $msg ); } @@ -552,6 +598,80 @@ } /** + * processes user_options['categories'] + * + * creates sanitized categories, which have been + * stripped of illegal category characters + */ + protected function setGlobalCategories() { + $categories = array(); + $this->_global_categories = array(); + + if ( !empty( $this->user_options['categories'] ) ) { + $categories = explode( + Config::$category_separator, + $this->user_options['categories'] + ); + } + + foreach( $categories as $key => $item ) { + $this->_global_categories[$key] = + Utils::stripIllegalCategoryChars( + Utils::sanitizeString( $item ) + ); + } + } + + /** + * processes user_options['gwtoolset-category-metadata'] + * and user_options['gwtoolset-category-phrase'] + * + * creates sanitized categories, which have been + * stripped of illegal category characters + */ + protected function setItemSpecificCategories() { + $this->_item_specific_categories = array(); + + if ( !empty( $this->user_options['gwtoolset-category-metadata'] ) ) { + $category_count = count( $this->user_options['gwtoolset-category-metadata'] ); + + for ( $i = 0; $i < $category_count; $i += 1 ) { + $phrase = null; + $metadata_values = array(); + + if ( !empty( $this->user_options['gwtoolset-category-phrase'][$i] ) ) { + $phrase = $this->user_options['gwtoolset-category-phrase'][$i]; + } + + if ( !empty( $this->user_options['gwtoolset-category-metadata'][$i] ) ) { + $metadata_values = + $this->_Metadata->getFieldValuesAsArray( + $this->user_options['gwtoolset-category-metadata'][$i] + ); + } + + foreach( $metadata_values as $metadata_value ) { + if ( !empty( $phrase ) ) { + $this->_item_specific_categories[] = + Utils::stripIllegalCategoryChars( + Utils::sanitizeString( $phrase ) + ) . + ' ' . + Utils::stripIllegalCategoryChars( + Utils::sanitizeString( $metadata_value ) + ); + } else { + $this->_item_specific_categories[] = + Utils::stripIllegalCategoryChars( + Utils::sanitizeString( $metadata_value ) + ); + } + } + } + } + } + + /** * @todo does UploadFromUrl filter $options['gwtoolset-url-to-the-media-file'] * @todo does UploadFromUrl filter $options['comment'] * @todo does UploadFromUrl filter $options['text'] @@ -564,6 +684,7 @@ protected function uploadMediaFileViaUploadFromUrl( array &$options, Title $Title ) { // Initialize this object and the upload object $Upload = new UploadFromUrl(); + WikiChecks::increaseHTTPTimeout(); $Upload->initialize( $Title->getBaseText(), @@ -624,7 +745,7 @@ * @param {array} $options * @throws {MWException} */ - protected function validatePageOptions( array &$options ) { + protected function validateUploadParams( array &$options ) { if ( !isset( $options['ignorewarnings'] ) ) { throw new MWException( wfMessage( 'gwtoolset-developer-issue' ) diff --git a/includes/Models/MediawikiTemplate.php b/includes/Models/MediawikiTemplate.php index 1a13a0b..36396ba 100644 --- a/includes/Models/MediawikiTemplate.php +++ b/includes/Models/MediawikiTemplate.php @@ -139,7 +139,7 @@ return '<!-- GWToolset Template -->' . PHP_EOL . '{{Uploaded with GWToolset' . PHP_EOL . - ' | gwtoolset-title-|identifier = ' . + ' | gwtoolset-title-identifier = ' . Utils::sanitizeString( $this->mediawiki_template_array['gwtoolset-title-identifier'] ) . PHP_EOL . diff --git a/resources/css/ext.gwtoolset.css b/resources/css/ext.gwtoolset.css index 430106f..5a4ae01 100644 --- a/resources/css/ext.gwtoolset.css +++ b/resources/css/ext.gwtoolset.css @@ -71,3 +71,9 @@ padding: 7px 0 7px 18px; overflow: auto; } + +div#content .preview-title { + font-weight: bold; + border: none; + margin-top: 2em; +} -- To view, visit https://gerrit.wikimedia.org/r/127839 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I91fa14730b9614cef4b912b218a692fb19271ce9 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/GWToolset Gerrit-Branch: master Gerrit-Owner: Dan-nl <d_ent...@yahoo.com> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits