Mattflaschen has uploaded a new change for review.
https://gerrit.wikimedia.org/r/202979
Change subject: WIP: LQT maintenance refactoring
......................................................................
WIP: LQT maintenance refactoring
Change-Id: Idb4c95d189c6de3df3a0c5103a619c7844296b20
---
M includes/Import/Converter.php
M includes/Import/LiquidThreadsApi/Objects.php
M maintenance/convertLqt.php
D maintenance/convertLqtPage.php
A maintenance/convertLqtPageFromRemoteApiForTesting.php
A maintenance/convertLqtPageOnLocalWiki.php
M maintenance/convertNamespaceFromWikitext.php
M tests/phpunit/Import/ConverterTest.php
8 files changed, 190 insertions(+), 141 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Flow
refs/changes/79/202979/1
diff --git a/includes/Import/Converter.php b/includes/Import/Converter.php
index b24bf2d..f689816 100644
--- a/includes/Import/Converter.php
+++ b/includes/Import/Converter.php
@@ -34,7 +34,7 @@
* @var DatabaseBase Slave database of the current wiki. Required
* to lookup past page moves.
*/
- protected $dbr;
+ protected $dbw;
/**
* @var Importer Service capable of turning an IImportSource into
@@ -61,7 +61,7 @@
protected $strategy;
/**
- * @param DatabaseBase $dbr Slave wiki database to read from
+ * @param DatabaseBase $dbw Master wiki database to read from
* @param Importer $importer
* @param LoggerInterface $logger
* @param User $user Administrative user for moves and edits related
@@ -70,7 +70,7 @@
* @throws ImportException When $user does not have an Id
*/
public function __construct(
- DatabaseBase $dbr,
+ DatabaseBase $dbw,
Importer $importer,
LoggerInterface $logger,
User $user,
@@ -79,7 +79,7 @@
if ( !$user->getId() ) {
throw new ImportException( 'User must have id' );
}
- $this->dbr = $dbr;
+ $this->dbw = $dbw;
$this->importer = $importer;
$this->logger = $logger;
$this->user = $user;
@@ -186,7 +186,7 @@
* @return Title|null
*/
protected function getPageMovedFrom( Title $title ) {
- $row = $this->dbr->selectRow(
+ $row = $this->dbw->selectRow(
array( 'logging', 'page' ),
array( 'log_namespace', 'log_title', 'log_user' ),
array(
diff --git a/includes/Import/LiquidThreadsApi/Objects.php
b/includes/Import/LiquidThreadsApi/Objects.php
index 7bd7dc9..9517bd4 100644
--- a/includes/Import/LiquidThreadsApi/Objects.php
+++ b/includes/Import/LiquidThreadsApi/Objects.php
@@ -317,7 +317,7 @@
class MovedImportPost extends ImportPost {
public function getRevisions() {
$factory = function( $data, $parent ) {
- return new MovedImportRevision( $data, $parent );
+ return new MovedImportRevision( $data, $parent,
$this->importSource->getScriptUser() );
};
$pageData = $this->importSource->getPageData( $this->pageId );
return new RevisionIterator( $pageData, $this, $factory );
diff --git a/maintenance/convertLqt.php b/maintenance/convertLqt.php
index a4b7521..cef22d1 100644
--- a/maintenance/convertLqt.php
+++ b/maintenance/convertLqt.php
@@ -22,38 +22,34 @@
parent::__construct();
$this->mDescription = "Converts LiquidThreads data to Flow
data";
$this->addOption( 'logfile', 'File to read and store
associations between imported items and their sources. This is required for the
import to be idempotent.', true, true );
- $this->addOption( 'verbose', 'Report on import progress to
stdout' );
$this->addOption( 'debug', 'Include debug information with
progress report' );
$this->addOption( 'startId', 'Page id to start importing at
(inclusive)' );
$this->addOption( 'stopId', 'Page id to stop importing at
(exclusive)' );
}
public function execute() {
- if ( $this->getOption( 'verbose' ) ) {
- $logger = new MaintenanceDebugLogger( $this );
- if ( $this->getOption( 'debug' ) ) {
- $logger->setMaximumLevel( LogLevel::DEBUG );
- } else {
- $logger->setMaximumLevel( LogLevel::INFO );
- }
+ $logger = new MaintenanceDebugLogger( $this );
+ if ( $this->getOption( 'debug' ) ) {
+ $logger->setMaximumLevel( LogLevel::DEBUG );
} else {
- $logger = new NullLogger;
+ $logger->setMaximumLevel( LogLevel::INFO );
}
+
$importer = Flow\Container::get( 'importer' );
$talkpageManagerUser =
FlowHooks::getOccupationController()->getTalkpageManager();
+ $dbw = wfGetDB( DB_MASTER );
$strategy = new ConversionStrategy(
- wfGetDB( DB_MASTER ),
+ $dbw
new FileImportSourceStore( $this->getOption( 'logfile'
) ),
new LocalApiBackend( $talkpageManagerUser ),
Container::get( 'url_generator' ),
$talkpageManagerUser,
- Container::get( 'controller.notifications' )
+ Container::get( 'controller.notification' )
);
- $dbr = wfGetDB( DB_SLAVE );
$converter = new \Flow\Import\Converter(
- $dbr,
+ $dbw,
$importer,
$logger,
$talkpageManagerUser,
@@ -64,6 +60,7 @@
$stopId = $this->getOption( 'stopId' );
$logger->info( "Starting full wiki LQT conversion" );
+ $dbr = wfGetDB( DB_SLAVE );
$titles = new PagesWithPropertyIterator( $dbr,
'use-liquid-threads', $startId, $stopId );
$converter->convert( $titles );
}
diff --git a/maintenance/convertLqtPage.php b/maintenance/convertLqtPage.php
deleted file mode 100644
index a6bcf02..0000000
--- a/maintenance/convertLqtPage.php
+++ /dev/null
@@ -1,113 +0,0 @@
-<?php
-
-use Flow\Import\FileImportSourceStore;
-use Flow\Import\NullImportSourceStore;
-use Flow\Import\LiquidThreadsApi\LocalApiBackend;
-use Flow\Import\LiquidThreadsApi\RemoteApiBackend;
-use Flow\Import\LiquidThreadsApi\ImportSource as LiquidThreadsApiImportSource;
-use Flow\Import\Postprocessor\LqtRedirector;
-use Flow\Import\Postprocessor\LqtNotifications;
-use Psr\Log\LogLevel;
-
-require_once ( getenv( 'MW_INSTALL_PATH' ) !== false
- ? getenv( 'MW_INSTALL_PATH' ) . '/maintenance/Maintenance.php'
- : dirname( __FILE__ ) . '/../../../maintenance/Maintenance.php' );
-
-class ConvertLqtPage extends Maintenance {
- public function __construct() {
- parent::__construct();
- $this->mDescription = "Converts LiquidThreads data to Flow
data";
- $this->addArg( 'dstpage', 'Page name of the local page to
import to', true );
- $this->addOption( 'srcpage', 'Page name of the remote page to
import from. If not specified defaults to dstpage', false, true );
- $this->addOption( 'remoteapi', 'Remote API URL to read from',
false, true );
- $this->addOption( 'cacheremoteapidir', 'Cache remote api calls
to the specified directory', false, true );
- $this->addOption( 'logfile', 'File to read and store
associations between imported items and their sources', false, true );
- $this->addOption( 'verbose', 'Report on import progress to
stdout' );
- $this->addOption( 'debug', 'Include debug information to
progress report' );
- $this->addOption( 'allowunknownusernames', 'Allow import of
usernames that do not exist on this wiki. DO NOT USE IN PRODUCTION. This
simplifies testing imports of production data to a test wiki' );
- $this->addOption( 'redirect', 'Add redirects from LQT posts to
their Flow equivalents and update watchlists' );
- $this->addOption( 'convertnotifications', 'Convert LQT
notifications into Echo notifications' );
- }
-
- public function execute() {
- $dstPageName = $srcPageName = $this->getArg( 0 );
-
- if ( $this->hasOption( 'srcpage' ) ) {
- $srcPageName = $this->getOption( 'srcpage' );
- }
-
- if ( $this->hasOption( 'remoteapi' ) ) {
- if ( $this->hasOption( 'cacheremoteapidir' ) ) {
- $cacheDir = $this->getOption(
'cacheremoteapidir' );
- if ( !is_dir( $cacheDir ) ) {
- if ( !mkdir( $cacheDir ) ) {
- throw new
Flow\Exception\FlowException( 'Provided dir for caching remote api calls is not
creatable.' );
- }
- }
- if ( !is_writable( $cacheDir ) ) {
- throw new Flow\Exception\FlowException(
'Provided dir for caching remote api calls is not writable.' );
- }
- } else {
- $cacheDir = null;
- }
- $api = new RemoteApiBackend( $this->getOption(
'remoteapi' ), $cacheDir );
- } else {
- $api = new LocalApiBackend;
- }
-
- $importer = Flow\Container::get( 'importer' );
- if ( $this->getOption( 'allowunknownusernames' ) ) {
- $importer->setAllowUnknownUsernames( true );
- }
- $source = new LiquidThreadsApiImportSource(
- $api,
- $srcPageName,
-
\FlowHooks::getOccupationController()->getTalkpageManager()
- );
- $title = Title::newFromText( $dstPageName );
-
- if ( $this->hasOption( 'logfile' ) ) {
- $filename = $this->getOption( 'logfile' );
- $sourceStore = new FileImportSourceStore( $filename );
- } else {
- $sourceStore = new NullImportSourceStore;
- }
-
- if ( $this->hasOption( 'redirect' ) ) {
- if ( $this->hasOption( 'remoteapi' ) ) {
- $this->error( 'Cannot use remoteapi and
redirect together', true );
- }
-
- $urlGenerator = Flow\Container::get( 'url_generator' );
- $user = Flow\Container::get( 'occupation_controller'
)->getTalkpageManager();
- $redirector = new LqtRedirector( $urlGenerator, $user );
- $importer->addPostprocessor( $redirector );
- }
-
- if ( $this->hasOption( 'convertnotifications' ) ) {
- $importer->addPostprocessor( new LqtNotifications(
- Flow\Container::get( 'controller.notification'
),
- wfGetDB( DB_MASTER )
- ) );
- }
-
- if ( $this->getOption( 'verbose' ) ) {
- $logger = new MaintenanceDebugLogger( $this );
- if ( $this->getOption( 'debug' ) ) {
- $logger->setMaximumLevel( LogLevel::DEBUG );
- } else {
- $logger->setMaximumLevel( LogLevel::INFO );
- }
- $importer->setLogger( $logger );
- $api->setLogger( $logger );
- $logger->info( "Starting LQT import from $srcPageName
to $dstPageName" );
- }
-
- $importer->import( $source, $title, $sourceStore );
-
- $sourceStore->save();
- }
-}
-
-$maintClass = "ConvertLqtPage";
-require_once ( RUN_MAINTENANCE_IF_MAIN );
diff --git a/maintenance/convertLqtPageFromRemoteApiForTesting.php
b/maintenance/convertLqtPageFromRemoteApiForTesting.php
new file mode 100644
index 0000000..d98f4d3
--- /dev/null
+++ b/maintenance/convertLqtPageFromRemoteApiForTesting.php
@@ -0,0 +1,87 @@
+<?php
+
+use Flow\Import\FileImportSourceStore;
+use Flow\Import\LiquidThreadsApi\BaseConversionStrategy as
LiquidThreadsApiBaseConversionStrategy;
+use Flow\Import\LiquidThreadsApi\RemoteConversionStrategy as
LiquidThreadsApiRemoteConversionStrategy;
+use Flow\Import\LiquidThreadsApi\LocalApiBackend;
+use Flow\Import\LiquidThreadsApi\RemoteApiBackend;
+use Flow\Import\LiquidThreadsApi\ImportSource as LiquidThreadsApiImportSource;
+use Flow\Import\Postprocessor\LqtRedirector;
+use Flow\Import\Postprocessor\LqtNotifications;
+use Psr\Log\LogLevel;
+
+require_once ( getenv( 'MW_INSTALL_PATH' ) !== false
+ ? getenv( 'MW_INSTALL_PATH' ) . '/maintenance/Maintenance.php'
+ : dirname( __FILE__ ) . '/../../../maintenance/Maintenance.php' );
+
+/**
+ * This is *only* for use in testing, not production. The primary purpose is
to exercise
+ * the API (production also uses the API, but with FauxRequest) and Parsoid.
+ *
+ * This also does not test redirects or notification conversion.
+ */
+class ConvertLqtPageFromRemoteApiForTesting extends Maintenance {
+ public function __construct() {
+ parent::__construct();
+ $this->mDescription = "Converts LiquidThreads data to Flow
data. Destination page is determined by ConversionStrategy";
+ $this->addOption( 'dstpage', 'Page name of the destination page
on the current wiki. Defaults to same as source', false, true );
+ $this->addOption( 'srcpage', 'Page name of the source page to
import from.', true, true );
+ $this->addOption( 'remoteapi', 'Remote API URL to read from',
true, true );
+ $this->addOption( 'cacheremoteapidir', 'Cache remote api calls
to the specified directory', true, true );
+ $this->addOption( 'logfile', 'File to read and store
associations between imported items and their sources', true, true );
+ $this->addOption( 'debug', 'Include debug information to
progress report' );
+ }
+
+ public function execute() {
+ $cacheDir = $this->getOption( 'cacheremoteapidir' );
+ if ( !is_dir( $cacheDir ) ) {
+ if ( !mkdir( $cacheDir ) ) {
+ throw new Flow\Exception\FlowException(
'Provided dir for caching remote api calls is not creatable.' );
+ }
+ }
+ if ( !is_writable( $cacheDir ) ) {
+ throw new Flow\Exception\FlowException( 'Provided dir
for caching remote api calls is not writable.' );
+ }
+
+ $api = new RemoteApiBackend( $this->getOption( 'remoteapi' ),
$cacheDir );
+
+ $importer = Flow\Container::get( 'importer' );
+ $importer->setAllowUnknownUsernames( true );
+
+ $talkPageManagerUser =
\FlowHooks::getOccupationController()->getTalkpageManager();
+
+ $srcPageName = $this->getOption( 'srcpage' );
+ if ( $this->hasOption( 'dstpage' ) ) {
+ $dstPageName = $this->getOption( 'dstpage' );
+ } else {
+ $dstPageName = $srcPageName;
+ }
+
+ $dstTitle = Title::newFromText( $dstPageName );
+ $source = new LiquidThreadsApiImportSource(
+ $api,
+ $srcPageName,
+ $talkPageManagerUser
+ );
+
+ $logFilename = $this->getOption( 'logfile' );
+ $sourceStore = new FileImportSourceStore( $logFilename );
+
+ $logger = new MaintenanceDebugLogger( $this );
+ if ( $this->getOption( 'debug' ) ) {
+ $logger->setMaximumLevel( LogLevel::DEBUG );
+ } else {
+ $logger->setMaximumLevel( LogLevel::INFO );
+ }
+
+ $importer->setLogger( $logger );
+ $api->setLogger( $logger );
+
+ $logger->info( "Starting LQT conversion of page $srcPageName" );
+
+ $importer->import( $source, $dstTitle, $sourceStore );
+ }
+}
+
+$maintClass = "ConvertLqtPageFromRemoteApiForTesting";
+require_once ( RUN_MAINTENANCE_IF_MAIN );
diff --git a/maintenance/convertLqtPageOnLocalWiki.php
b/maintenance/convertLqtPageOnLocalWiki.php
new file mode 100644
index 0000000..e4a4fec
--- /dev/null
+++ b/maintenance/convertLqtPageOnLocalWiki.php
@@ -0,0 +1,81 @@
+<?php
+
+use Flow\Container;
+use Flow\Import\FileImportSourceStore;
+use Flow\Import\NullImportSourceStore;
+use Flow\Import\LiquidThreadsApi\ConversionStrategy as
LiquidThreadsApiConversionStrategy;
+use Flow\Import\LiquidThreadsApi\LocalApiBackend;
+use Flow\Import\LiquidThreadsApi\RemoteApiBackend;
+use Flow\Import\LiquidThreadsApi\ImportSource as LiquidThreadsApiImportSource;
+use Flow\Import\Postprocessor\LqtRedirector;
+use Flow\Import\Postprocessor\LqtNotifications;
+use Psr\Log\LogLevel;
+
+require_once ( getenv( 'MW_INSTALL_PATH' ) !== false
+ ? getenv( 'MW_INSTALL_PATH' ) . '/maintenance/Maintenance.php'
+ : dirname( __FILE__ ) . '/../../../maintenance/Maintenance.php' );
+
+/**
+ * This is intended for use both in testing and in production. It converts a
single LQT
+ * page on the current wiki to a Flow page on the current wiki, handling
archiving.
+ */
+class ConvertLqtPageOnLocalWiki extends Maintenance {
+ public function __construct() {
+ parent::__construct();
+ $this->mDescription = "Converts LiquidThreads data to Flow
data. Destination page is determined by ConversionStrategy";
+ $this->addOption( 'srcpage', 'Page name of the source page to
import from.', true, true );
+ $this->addOption( 'logfile', 'File to read and store
associations between imported items and their sources', true, true );
+ $this->addOption( 'debug', 'Include debug information to
progress report' );
+ }
+
+ public function execute() {
+ $talkPageManagerUser =
\FlowHooks::getOccupationController()->getTalkpageManager();
+
+ $api = new LocalApiBackend( $talkPageManagerUser );
+
+ $importer = Container::get( 'importer' );
+
+ $srcPageName = $this->getOption( 'srcpage' );
+
+ $logFilename = $this->getOption( 'logfile' );
+ $sourceStore = new FileImportSourceStore( $logFilename );
+
+ $dbw = wfGetDB( DB_MASTER );
+
+ $logger = new MaintenanceDebugLogger( $this );
+ if ( $this->getOption( 'debug' ) ) {
+ $logger->setMaximumLevel( LogLevel::DEBUG );
+ } else {
+ $logger->setMaximumLevel( LogLevel::INFO );
+ }
+
+ $strategy = new LiquidThreadsApiConversionStrategy(
+ $dbw,
+ $sourceStore,
+ $api,
+ Container::get( 'url_generator' ),
+ $talkPageManagerUser,
+ Container::get( 'controller.notification' )
+ );
+
+ $importer->setLogger( $logger );
+ $api->setLogger( $logger );
+
+ $converter = new \Flow\Import\Converter(
+ $dbw,
+ $importer,
+ $logger,
+ $talkPageManagerUser,
+ $strategy
+ );
+
+ $logger->info( "Starting LQT conversion of page $srcPageName" );
+
+ $converter->convert( new ArrayIterator( array(
+ \Title::newFromText( $srcPageName ),
+ ) ) );
+ }
+}
+
+$maintClass = "ConvertLqtPageOnLocalWiki";
+require_once ( RUN_MAINTENANCE_IF_MAIN );
diff --git a/maintenance/convertNamespaceFromWikitext.php
b/maintenance/convertNamespaceFromWikitext.php
index 297e05c..10bb95a 100644
--- a/maintenance/convertNamespaceFromWikitext.php
+++ b/maintenance/convertNamespaceFromWikitext.php
@@ -17,7 +17,6 @@
parent::__construct();
$this->mDescription = "Converts a single namespace of wikitext
talk pages to Flow";
$this->addArg( 'namespace', 'Name of the namespace to convert'
);
- $this->addOption( 'verbose', 'Report on import progress to
stdout' );
}
public function execute() {
@@ -31,13 +30,11 @@
}
// @todo send to prod logger?
- $logger = $this->getOption( 'verbose' )
- ? new MaintenanceDebugLogger( $this )
- : new NullLogger();
+ $logger = new MaintenanceDebugLogger( $this );
- $dbr = wfGetDB( DB_SLAVE );
+ $dbw = wfGetDB( DB_MASTER );
$converter = new \Flow\Import\Converter(
- $dbr,
+ $dbw,
Flow\Container::get( 'importer' ),
$logger,
FlowHooks::getOccupationController()->getTalkpageManager(),
@@ -51,6 +48,7 @@
$logger->info( "Starting conversion of $namespaceName
namespace" );
// Iterate over all existing pages of the namespace.
+ $dbr = wfGetDB( DB_SLAVE );
$it = new NamespaceIterator( $dbr, $namespace );
// NamespaceIterator is an IteratorAggregate. Get an Iterator
// so we can wrap that.
@@ -76,4 +74,3 @@
$maintClass = "ConvertNamespaceFromWikitext";
require_once ( RUN_MAINTENANCE_IF_MAIN );
-
diff --git a/tests/phpunit/Import/ConverterTest.php
b/tests/phpunit/Import/ConverterTest.php
index e5cbb07..005a980 100644
--- a/tests/phpunit/Import/ConverterTest.php
+++ b/tests/phpunit/Import/ConverterTest.php
@@ -80,14 +80,14 @@
}
protected function createConverter(
- DatabaseBase $dbr = null,
+ DatabaseBase $dbw = null,
Importer $importer = null,
LoggerInterface $logger = null,
User $user = null,
IConversionStrategy $strategy = null
) {
return new Converter(
- $dbr ?: wfGetDB( DB_SLAVE ),
+ $dbw ?: wfGetDB( DB_MASTER ),
$importer ?: $this->getMockBuilder(
'Flow\Import\Importer' )
->disableOriginalConstructor()
->getMock(),
--
To view, visit https://gerrit.wikimedia.org/r/202979
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Idb4c95d189c6de3df3a0c5103a619c7844296b20
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Flow
Gerrit-Branch: master
Gerrit-Owner: Mattflaschen <mflasc...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
