EBernhardson has uploaded a new change for review.
https://gerrit.wikimedia.org/r/174861
Change subject: [WIP] Archive and takeover namespaces with Flow
......................................................................
[WIP] Archive and takeover namespaces with Flow
Change-Id: Ie785403748cc22bc28ff52a172bfaafad8aaf18b
---
M autoload.php
M i18n/en.json
M i18n/qqq.json
A includes/Import/Plain/ImportHeader.php
A includes/Import/Plain/ObjectRevision.php
A includes/Import/Wikitext/Converter.php
A includes/Import/Wikitext/ImportSource.php
A includes/Utils/NamespaceIterator.php
A maintenance/convertNamespace.php
9 files changed, 395 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Flow
refs/changes/61/174861/1
diff --git a/autoload.php b/autoload.php
index fa96909..36e372a 100644
--- a/autoload.php
+++ b/autoload.php
@@ -160,6 +160,7 @@
$wgAutoloadClasses['Flow\\Formatter\\TopicListFormatter'] = __DIR__ .
'/includes/Formatter/TopicListFormatter.php';
$wgAutoloadClasses['Flow\\Formatter\\TopicListQuery'] = __DIR__ .
'/includes/Formatter/TopicListQuery.php';
$wgAutoloadClasses['Flow\\Formatter\\TopicRow'] = __DIR__ .
'/includes/Formatter/TopicRow.php';
+$wgAutoloadClasses['Flow\\Import\\Basic\\ObjectRevision'] = __DIR__ .
'/includes/Import/Plain/ObjectRevision.php';
$wgAutoloadClasses['Flow\\Import\\FileImportSourceStore'] = __DIR__ .
'/includes/Import/ImportSourceStore.php';
$wgAutoloadClasses['Flow\\Import\\HistoricalUIDGenerator'] = __DIR__ .
'/includes/Import/Importer.php';
$wgAutoloadClasses['Flow\\Import\\IImportHeader'] = __DIR__ .
'/includes/Import/ImportSource.php';
@@ -194,8 +195,11 @@
$wgAutoloadClasses['Flow\\Import\\LiquidThreadsApi\\TopicIterator'] = __DIR__
. '/includes/Import/LiquidThreadsApi/Iterators.php';
$wgAutoloadClasses['Flow\\Import\\NullImportSourceStore'] = __DIR__ .
'/includes/Import/ImportSourceStore.php';
$wgAutoloadClasses['Flow\\Import\\PageImportState'] = __DIR__ .
'/includes/Import/Importer.php';
+$wgAutoloadClasses['Flow\\Import\\Plain\\ImportHeader'] = __DIR__ .
'/includes/Import/Plain/ImportHeader.php';
$wgAutoloadClasses['Flow\\Import\\TalkpageImportOperation'] = __DIR__ .
'/includes/Import/Importer.php';
$wgAutoloadClasses['Flow\\Import\\TopicImportState'] = __DIR__ .
'/includes/Import/Importer.php';
+$wgAutoloadClasses['Flow\\Import\\Wikitext\\Converter'] = __DIR__ .
'/includes/Import/Wikitext/Converter.php';
+$wgAutoloadClasses['Flow\\Import\\Wikitext\\ImportSource'] = __DIR__ .
'/includes/Import/Wikitext/ImportSource.php';
$wgAutoloadClasses['Flow\\LinksTableUpdater'] = __DIR__ .
'/includes/LinksTableUpdater.php';
$wgAutoloadClasses['Flow\\Log\\Formatter'] = __DIR__ .
'/includes/Log/Formatter.php';
$wgAutoloadClasses['Flow\\Log\\Logger'] = __DIR__ . '/includes/Log/Logger.php';
@@ -323,6 +327,7 @@
$wgAutoloadClasses['Flow\\Tests\\UrlGeneratorTest'] = __DIR__ .
'/tests/phpunit/UrlGeneratorTest.php';
$wgAutoloadClasses['Flow\\Tests\\WatchedTopicItemTest'] = __DIR__ .
'/tests/phpunit/WatchedTopicItemsTest.php';
$wgAutoloadClasses['Flow\\UrlGenerator'] = __DIR__ .
'/includes/UrlGenerator.php';
+$wgAutoloadClasses['Flow\\Utils\\NamespaceIterator'] = __DIR__ .
'/includes/Utils/NamespaceIterator.php';
$wgAutoloadClasses['Flow\\Utils\\PagesWithPropertyIterator'] = __DIR__ .
'/includes/Utils/PagesWithPropertyIterator.php';
$wgAutoloadClasses['Flow\\View'] = __DIR__ . '/includes/View.php';
$wgAutoloadClasses['Flow\\WatchedTopicItems'] = __DIR__ .
'/includes/WatchedTopicItems.php';
diff --git a/i18n/en.json b/i18n/en.json
index 369a75c..d18cd38 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -457,5 +457,6 @@
"apihelp-query+flowinfo-description": "Get basic Flow information about
a page.",
"apihelp-query+flowinfo-example-1": "Fetch Flow information about
[[Talk:Sandbox]], [[Main Page]], and [[Talk:Flow]]",
"flow-edited": "Edited",
- "flow-edited-by": "Edited by $1"
+ "flow-edited-by": "Edited by $1",
+ "flow-importer-wt-converted-template": "Wikitext talk page converted to
Flow"
}
diff --git a/i18n/qqq.json b/i18n/qqq.json
index cfd195b..045f4e5 100644
--- a/i18n/qqq.json
+++ b/i18n/qqq.json
@@ -461,5 +461,6 @@
"apihelp-query+flowinfo-description":
"{{doc-apihelp-description|query+flowinfo}}",
"apihelp-query+flowinfo-example-1":
"{{doc-apihelp-example|query+flowinfo}}",
"flow-edited": "Message displayed below a post to indicate it has last
been edited by the original author\n{{Identical|Edited}}",
- "flow-edited-by": "Message displayed below a post to indicate it has
last been edited by a user other than the original author"
+ "flow-edited-by": "Message displayed below a post to indicate it has
last been edited by a user other than the original author",
+ "flow-importer-wt-converted-template": "Name of a wikitext template
that is added to the header of flow boards that were converted from wikitext"
}
diff --git a/includes/Import/Plain/ImportHeader.php
b/includes/Import/Plain/ImportHeader.php
new file mode 100644
index 0000000..45d7a07
--- /dev/null
+++ b/includes/Import/Plain/ImportHeader.php
@@ -0,0 +1,16 @@
+<?php
+
+namespace Flow\Import\Plain;
+
+use ArrayIterator;
+use Flow\Import\IImportHeader;
+
+class ImportHeader implements IImportHeader {
+ public function __construct( array $revisions ) {
+ $this->revisions = $revisions;
+ }
+
+ public function getRevisions() {
+ return new ArrayIterator( $this->revisions );
+ }
+}
diff --git a/includes/Import/Plain/ObjectRevision.php
b/includes/Import/Plain/ObjectRevision.php
new file mode 100644
index 0000000..b077326
--- /dev/null
+++ b/includes/Import/Plain/ObjectRevision.php
@@ -0,0 +1,29 @@
+<?php
+
+namespace Flow\Import\Basic;
+
+use Flow\Import\IObjectRevision;
+
+class ObjectRevision implements IObjectRevision {
+ protected $text;
+ protected $timestamp;
+ protected $author;
+
+ public function __construct( $text, $timestamp, $author ) {
+ $this->text = $text;
+ $this->timestamp = $timestamp;
+ $this->author = $author;
+ }
+
+ public function getText() {
+ return $this->text;
+ }
+
+ public function getTimestamp() {
+ return $this->timestamp;
+ }
+
+ public function getAuthor() {
+ return $this->author;
+ }
+}
diff --git a/includes/Import/Wikitext/Converter.php
b/includes/Import/Wikitext/Converter.php
new file mode 100644
index 0000000..6cb5696
--- /dev/null
+++ b/includes/Import/Wikitext/Converter.php
@@ -0,0 +1,158 @@
+<?php
+
+namespace Flow\Import\Wikitext;
+
+use DateTime;
+use DateTimeZone;
+use Flow\Exception\FlowException;
+use MovePage;
+use MWExceptionHandler;
+use Psr\Log\LoggerInterface;
+use Revision;
+use Title;
+use User;
+use WikiPage;
+use WikitextContent;
+
+/**
+ * Does not really convert. Archives wikitext pages out of the way and puts
+ * a new flow board in place. No flow revision is created, after conversion
+ * the namespace must be configured with flow-board as the default content
+ * model.
+ *
+ * It is plausible something with the EchoDiscussionParser could be worked up
+ * to do an import. We know it wont work for everything, but we don't know if
+ * it works for 90%, 99%, or 99.99% of topics.
+ */
+class Converter {
+ /**
+ * @var User
+ */
+ protected $user;
+
+ /**
+ * @var LoggerInterface
+ */
+ protected $logger;
+
+ public function __construct( User $user, LoggerInterface $logger ) {
+ $this->user = $user;
+ $this->logger = $logger;
+ }
+
+ /**
+ * @param Traversable<Title> $titles
+ */
+ public function convert( $titles ) {
+ /** @var Title $title */
+ foreach ( $titles as $title ) {
+ if ( $title->isSubpage() ) {
+ continue;
+ }
+ if ( $title->getContentModel() !== 'wikitext' ) {
+ continue;
+ }
+ // @todo check for lqt
+
+ $titleText = $title->getPrefixedText();
+ try {
+ $archiveTitle = $this->decideArchiveTitle(
$title );
+ $archiveTitleText =
$archiveTitle->getPrefixedText();
+ $this->logger->info( "Archiving page from
$titleText to $archiveTitleText" );
+ $this->movePage( $title, $archiveTitle );
+ $this->createArchiveRevision( $title,
$archiveTitle );
+ } catch ( \Exception $e ) {
+ MWExceptionHandler::logException( $e );
+ $this->logger->error( "Exception while
importing: $titleText" );
+ $this->logger->error( (string)$e );
+ }
+ }
+ }
+
+ protected function movePage( Title $from, Title $to ) {
+ $mp = new MovePage( $from, $to );
+ $valid = $mp->isValidMove();
+ if ( !$valid->isOK() ) {
+ throw new FlowException( "It is not valid to move
{$from->getPrefixedText()} to {$to->getPrefixedText()}" );
+ }
+
+ $status = $mp->move(
+ /* user */ $this->user,
+ /* reason */ "Conversion of wikitext talk to Flow from:
{$from->getPrefixedText()}",
+ /* create redirect */ false
+ );
+
+ if ( !$status->isGood() ) {
+ throw new FlowException( "Failed moving
{$from->getPrefixedText()} to {$to->getPrefixedText()}" );
+ }
+ }
+
+ protected function decideArchiveTitle( Title $source ) {
+ // @todo i18n. Would bes
+ $formats = array(
+ '%s/Archive %d',
+ '%s/Archive%d',
+ '%s/archive %d',
+ '%s/archive%d',
+ );
+
+ $format = false;
+ $n = 1;
+ $text = $source->getPrefixedText();
+ foreach ( $formats as $potential ) {
+ $title = Title::newFromText( sprintf( $potential,
$text, $n ) );
+ if ( $title && $title->exists() ) {
+ $format = $potential;
+ break;
+ }
+ }
+ if ( $format === false ) {
+ // assumes this creates a valid title
+ return Title::newFromText( sprintf( $formats[0], $text,
$n ) );
+ }
+ for ( ++$n; $n < 20; ++$n ) {
+ $title = Title::newFromText( sprintf( $format, $n ) );
+ if ( $title && !$title->exists() ) {
+ return $title;
+ }
+ }
+
+ throw new FlowException( "All titles 1 through 20 exist for
format: $format" );
+ }
+
+ protected function createArchiveRevision( Title $title, Title
$archiveTitle ) {
+ $page = WikiPage::factory( $archiveTitle );
+ $revision = $page->getRevision();
+ if ( $revision === null ) {
+ throw new FlowException( "Expected a revision at
{$archiveTitle->getPrefixedText()}." );
+ }
+
+ $content = $revision->getContent( Revision::RAW );
+ if ( !$content instanceof WikitextContent ) {
+ throw new FlowException( "Expected wikitext content at
{$archiveTitle->getPrefixedText()}." );
+ }
+ $status = $page->doEditContent(
+ $this->createArchiveRevisionContent( $content, $title ),
+ 'Wikitext talk to Flow conversion',
+ EDIT_FORCE_BOT | EDIT_SUPPRESS_RC,
+ false,
+ $this->user
+ );
+
+ if ( !$status->isGood() ) {
+ throw new FlowException( "Failed creating archive
revision at {$archiveTitle->getPrefixedText()}" );
+ }
+ }
+
+ protected function createArchiveRevisionContent( WikitextContent
$content, Title $title ) {
+ $now = new DateTime( "now", new DateTimeZone( "GMT" ) );
+ $arguments = implode( '|', array(
+ 'from=' . $title->getPrefixedText(),
+ 'date=' . $now->format( 'Y-m-d' ),
+ ) );
+
+ $newWikitext = $content->getNativeData() . "\n\n{{Archive of
wikitext talk page converted to Flow|$arguments}}";
+
+ return new WikitextContent( $newWikitext );
+ }
+}
diff --git a/includes/Import/Wikitext/ImportSource.php
b/includes/Import/Wikitext/ImportSource.php
new file mode 100644
index 0000000..8c1753e
--- /dev/null
+++ b/includes/Import/Wikitext/ImportSource.php
@@ -0,0 +1,78 @@
+<?php
+
+namespace Flow\Import\Wikitext;
+
+use DateTime;
+use DateTimeZone;
+use FlowHooks;
+use Flow\Import\Basic\ImportHeader;
+use Flow\Import\Basic\ObjectRevision;
+use ParserOptions;
+use Title;
+
+class ImportSource implements IImportSource {
+
+ public function __construct( Title $title ) {
+ if ( !$title->isLocal() ) {
+ throw new \MWException( "Invalid non-local title:
{$title->getPrefixedText()}" );
+ }
+ $this->title = $title;
+ }
+
+ public function getHeader() {
+ global $wgParser;
+
+ $rev = $this->getHeaderRevision();
+ if ( $rev === null ) {
+ return null;
+ }
+
+ return new ImportHeader( array(
+ $rev,
+ $this->generateNextRevision( $rev ),
+ ) );
+ }
+
+ /**
+ * @return IObjectRevision|null
+ */
+ protected function getHeaderRevision() {
+ $revision = Revision::newFromTitle( $this->title );
+ if ( !$revision ) {
+ return null;
+ }
+
+ $options = new ParserOptions;
+ $options->setTidy( true );
+ $options->setEditSection( false );
+ $content = $revision->getContent();
+
+ // If sections exist only take the content from the top of the
page
+ // to the first section.
+ $output = $parser->parse( $content, $this->title, $options );
+ $sections = $output->getSections();
+ if ( $sections ) {
+ $content = substr( $content, 0,
$sections[0]['byteoffset'] );
+ }
+
+ $templateName = wfMessage(
'flow-importer-wt-converted-template' )->inContentLanguage()->plain();
+
+ $now = new DateTime( "now", new DateTimeZone( "GMT" ) );
+ $arguments = implode( '|', array(
+ 'from=' . $this->title->getPrefixedText(),
+ 'date=' . $now->format( 'Y-m-d' ),
+ ) );
+ $content .= "\n\n{{{$templateName}|$arguments}}";
+
+ return new ObjectRevision(
+ $content,
+ wfTimestampNow(),
+
FlowHooks::getOccupationController()->getTalkpageManager()
+ );
+ }
+
+ public function getTopics() {
+ return new ArrayIterator( array() );
+ }
+}
+
diff --git a/includes/Utils/NamespaceIterator.php
b/includes/Utils/NamespaceIterator.php
new file mode 100644
index 0000000..f0fe0e3
--- /dev/null
+++ b/includes/Utils/NamespaceIterator.php
@@ -0,0 +1,56 @@
+<?php
+
+namespace Flow\Utils;
+
+use DatabaseBase;
+use EchoBatchRowIterator;
+use EchoCallbackIterator;
+use IteratorAggregate;
+use RecursiveIteratorIterator;
+use Title;
+
+/**
+ * Iterates over all titles that have the specified page property
+ */
+class NamespaceIterator implements IteratorAggregate {
+ /**
+ * @var DatabaseBase
+ */
+ protected $db;
+
+ /**
+ * @var int
+ */
+ protected $namespace;
+
+ /**
+ * @param DatabaseBase $db
+ * @param int $namespace
+ */
+ public function __construct( DatabaseBase $db, $namespace ) {
+ $this->db = $db;
+ $this->namespace = $namespace;
+ }
+
+ /**
+ * @return Iterator<Title>
+ */
+ public function getIterator() {
+ $it = new EchoBatchRowIterator(
+ $this->db,
+ /* tables */ array( 'page' ),
+ /* pk */ 'page_id',
+ /* rows per batch */ 500
+ );
+ $it->addConditions( array(
+ 'page_namespace' => $this->namespace,
+ ) );
+ $it->setFetchColumns( array( 'page_title' ) );
+ $it = new RecursiveIteratorIterator( $it );
+
+ $namespace = $this->namespace;
+ return new EchoCallbackIterator( $it, function( $row ) use (
$namespace ) {
+ return Title::makeTitle( $namespace, $row->page_title );
+ } );
+ }
+}
diff --git a/maintenance/convertNamespace.php b/maintenance/convertNamespace.php
new file mode 100644
index 0000000..f2b99a1
--- /dev/null
+++ b/maintenance/convertNamespace.php
@@ -0,0 +1,49 @@
+<?php
+
+use Flow\Utils\NamespaceIterator;
+use Psr\Log\NullLogger;
+
+require_once ( getenv( 'MW_INSTALL_PATH' ) !== false
+ ? getenv( 'MW_INSTALL_PATH' ) . '/maintenance/Maintenance.php'
+ : dirname( __FILE__ ) . '/../../../maintenance/Maintenance.php' );
+
+/**
+ * Converts a single namespace from wikitext talk pages to flow talk pages.
Does not
+ * modify liquid threads pages it comes across, use convertLqt.php for that.
Does not
+ * modify sub-pages.
+ */
+class ConvertNamespace extends Maintenance {
+ public function __construct() {
+ parent::__construct();
+ $this->mDescription = "Converts LiquidThreads data to Flow
data";
+ $this->addArg( 'namespace', 'Name of the namespace to convert'
);
+ $this->addOption( 'verbose', 'Report on import progress to
stdout' );
+ }
+
+ public function execute() {
+ global $wgLang;
+
+ $logger = $this->getOption( 'verbose' )
+ ? new MaintenanceDebugLogger( $this )
+ : new NullLogger;
+
+ $provided = $this->getArg( 0 );
+ $namespace = $wgLang->getNsIndex( $provided );
+ if ( !$namespace ) {
+ $this->error( "Invalid namespace provided: $provided" );
+ return;
+ }
+ $namespaceName = $wgLang->getNsText( $namespace );
+
+ $logger->info( "Starting conversion of $namespaceName
namespace" );
+
+ $user =
FlowHooks::getOccupationController()->getTalkpageManager();
+ $converter = new Flow\Import\Wikitext\Converter( $user, $logger
);
+ $it = new NamespaceIterator( wfGetDB( DB_SLAVE ), $namespace );
+ $converter->convert( $it );
+ }
+}
+
+$maintClass = "ConvertNamespace";
+require_once ( RUN_MAINTENANCE_IF_MAIN );
+
--
To view, visit https://gerrit.wikimedia.org/r/174861
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie785403748cc22bc28ff52a172bfaafad8aaf18b
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Flow
Gerrit-Branch: master
Gerrit-Owner: EBernhardson <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
