Aude has uploaded a new change for review.
https://gerrit.wikimedia.org/r/204786
Change subject: Split code out of pruneChanges script
......................................................................
Split code out of pruneChanges script
added some integration test for the ChangePruner.
as long as the pruning is directly interacting
with the database, performing queries, I don't
see a nice way to mock this and avoid using
database for this test.
Change-Id: I87065a723572639702b3a0f839d68fbeca14eac7
---
A repo/includes/ChangePruner.php
M repo/maintenance/pruneChanges.php
A repo/tests/phpunit/includes/ChangePrunerTest.php
3 files changed, 294 insertions(+), 111 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Wikibase
refs/changes/86/204786/1
diff --git a/repo/includes/ChangePruner.php b/repo/includes/ChangePruner.php
new file mode 100644
index 0000000..c4382f5
--- /dev/null
+++ b/repo/includes/ChangePruner.php
@@ -0,0 +1,157 @@
+<?php
+
+namespace Wikibase\Repo;
+
+ use Wikibase\Lib\Reporting\NullMessageReporter;
+
+class ChangePruner {
+
+ /**
+ * @var int
+ */
+ private $batchSize;
+
+ /**
+ * @var int the minimum number of seconds to keep changes for.
+ */
+ private $keepSeconds;
+
+ /**
+ * @var int the minimum number of seconds after dispatching to keep
changes for.
+ */
+ private $graceSeconds;
+
+ /**
+ * @var bool whether the dispatch time should be ignored
+ */
+ private $ignoreDispatch;
+
+ /**
+ * @var MessageReporter
+ */
+ private $messageReporter;
+
+ /**
+ * @param int $batchSize
+ * @param int $keepSeconds
+ * @param int $graceSeconds
+ * @param bool $ignoreDispatch
+ */
+ public function __construct( $batchSize, $keepSeconds, $graceSeconds,
$ignoreDispatch ) {
+ $this->batchSize = $batchSize;
+ $this->keepSeconds = $keepSeconds;
+ $this->graceSeconds = $graceSeconds;
+ $this->ignoreDispatch = $ignoreDispatch;
+
+ $this->messageReporter = new NullMessageReporter();
+ }
+
+ public function doPrune() {
+ while( true ) {
+ wfWaitForSlaves();
+
+ $until = $this->getCutoffTimestamp();
+
+ $this->messageReporter->reportMessage(
+ date( 'H:i:s' ) . " pruning entries older than "
+ . wfTimestamp( TS_ISO_8601, $until )
+ );
+
+ $affected = $this->pruneChanges( $until );
+
+ $this->messageReporter->reportMessage( date( 'H:i:s' )
. " $affected rows pruned." );
+
+ if ( $affected === 0 ) {
+ break;
+ }
+ }
+ }
+
+ /**
+ * Calculates the timestamp up to which changes can be pruned.
+ *
+ * @return int Timestamp up to which changes can be pruned (as Unix
period).
+ */
+ private function getCutoffTimestamp() {
+ $until = time() - $this->keepSeconds;
+
+ if ( !$this->ignoreDispatch ) {
+ $dbr = wfGetDB( DB_SLAVE );
+ $row = $dbr->selectRow(
+ array ( 'wb_changes_dispatch', 'wb_changes' ),
+ 'min(change_time) as timestamp',
+ array(
+ 'chd_disabled' => 0,
+ 'chd_seen = change_id'
+ ),
+ __METHOD__
+ );
+
+ if ( isset( $row->timestamp ) ) {
+ $dispatched = wfTimestamp( TS_UNIX,
$row->timestamp ) - $this->graceSeconds;
+
+ $until = min( $until, $dispatched );
+ }
+ }
+
+ return $this->limitCutoffTimestamp( $until );
+ }
+
+ /**
+ * Changes the cutoff timestamp to not affect more than $this->batchSize
+ * rows, if needed.
+ *
+ * @param int $until
+ *
+ * @return int
+ */
+ private function limitCutoffTimestamp( $until ) {
+ $dbr = wfGetDB( DB_SLAVE );
+ $changeTime = $dbr->selectField(
+ 'wb_changes',
+ 'change_time',
+ array( 'change_time < ' . $dbr->addQuotes( wfTimestamp(
TS_MW, $until ) ) ),
+ __METHOD__,
+ array(
+ 'OFFSET' => $this->batchSize,
+ 'ORDER BY' => 'change_time ASC',
+ )
+ );
+
+ return $changeTime ? intval( $changeTime ) : $until;
+ }
+
+ /**
+ * Prunes all changes older than $until from the changes table.
+ *
+ * @param int $until
+ *
+ * @return int the number of changes deleted.
+ */
+ private function pruneChanges( $until ) {
+ $dbw = wfGetDB( DB_MASTER );
+
+ $dbw->delete(
+ 'wb_changes',
+ array( 'change_time < ' . $dbw->addQuotes( wfTimestamp(
TS_MW, $until ) ) ),
+ __METHOD__
+ );
+
+ return $dbw->affectedRows();
+ }
+
+ /**
+ * @return MessageReporter
+ */
+ public function getMessageReporter() {
+ return $this->messageReporter;
+ }
+
+ /**
+ * @param MessageReporter $messageReporter
+ */
+ public function setMessageReporter( $messageReporter ) {
+ $this->messageReporter = $messageReporter;
+ }
+
+}
diff --git a/repo/maintenance/pruneChanges.php
b/repo/maintenance/pruneChanges.php
index 1894259..dead100 100644
--- a/repo/maintenance/pruneChanges.php
+++ b/repo/maintenance/pruneChanges.php
@@ -1,8 +1,12 @@
<?php
namespace Wikibase;
+
use Maintenance;
use Wikibase\Lib\PidLock;
+use Wikibase\Lib\Reporting\MessageReporter;
+use Wikibase\Lib\Reporting\ObservableMessageReporter;
+use Wikibase\Repo\ChangePruner;
/**
* Prune the Wikibase changes table to a maximum number of entries.
@@ -16,21 +20,6 @@
*
*/
class PruneChanges extends Maintenance {
-
- /**
- * @var int the minimum number of seconds to keep changes for.
- */
- private $keepSeconds = 0;
-
- /**
- * @var int the minimum number of seconds after dispatching to keep
changes for.
- */
- private $graceSeconds = 0;
-
- /**
- * @var bool whether the dispatch time should be ignored
- */
- private $ignoreDispatch = false;
public function __construct() {
parent::__construct();
@@ -67,121 +56,70 @@
exit( 5 );
}
- $this->ignoreDispatch = $this->getOption( 'ignore-dispatch',
false );
+ $changePruner = new ChangePruner(
+ $this->mBatchSize,
+ $this->getKeepSeconds(),
+ $this->getGraceSeconds(),
+ $this->getOption( 'ignore-dispatch', false )
+ );
- $this->keepSeconds = 0;
- $this->keepSeconds += intval( $this->getOption(
'number-of-days', 0 ) ) * 24 * 60 * 60;
- $this->keepSeconds += intval( $this->getOption( 'keep-days', 0
) ) * 24 * 60 * 60;
- $this->keepSeconds += intval( $this->getOption( 'keep-hours', 0
) ) * 60 * 60;
- $this->keepSeconds += intval( $this->getOption( 'keep-minutes',
0 ) ) * 60;
-
- if ( $this->keepSeconds === 0 ) {
- // one day
- $this->keepSeconds = 1 * 24 * 60 * 60;
- }
-
- $this->graceSeconds = 0;
- $this->graceSeconds += intval( $this->getOption(
'grace-minutes', 0 ) ) * 60;
-
- if ( $this->graceSeconds === 0 ) {
- // one hour
- $this->graceSeconds = 1 * 60 * 60;
- }
-
- $this->doPrune();
+ $changePruner->setMessageReporter( $this->newMessageReporter()
);
+ $changePruner->doPrune();
$pidLock->removeLock(); // delete lockfile on normal exit
}
- /**
- * Calculates the timestamp up to which changes can be pruned.
- *
- * @return int Timestamp up to which changes can be pruned (as Unix
period).
- */
- private function getCutoffTimestamp() {
- $until = time() - $this->keepSeconds;
+ private function getKeepSeconds() {
+ $keepSeconds = 0;
+ $keepSeconds += intval( $this->getOption( 'number-of-days', 0 )
) * 24 * 60 * 60;
+ $keepSeconds += intval( $this->getOption( 'keep-days', 0 ) ) *
24 * 60 * 60;
+ $keepSeconds += intval( $this->getOption( 'keep-hours', 0 ) ) *
60 * 60;
+ $keepSeconds += intval( $this->getOption( 'keep-minutes', 0 ) )
* 60;
- if ( !$this->ignoreDispatch ) {
- $dbr = wfGetDB( DB_SLAVE );
- $row = $dbr->selectRow(
- array ( 'wb_changes_dispatch', 'wb_changes' ),
- 'min(change_time) as timestamp',
- array(
- 'chd_disabled' => 0,
- 'chd_seen = change_id'
- ),
- __METHOD__
- );
-
- if ( isset( $row->timestamp ) ) {
- $dispatched = wfTimestamp( TS_UNIX,
$row->timestamp ) - $this->graceSeconds;
-
- $until = min( $until, $dispatched );
- }
+ if ( $keepSeconds === 0 ) {
+ // one day
+ $keepSeconds = 1 * 24 * 60 * 60;
}
- return $this->limitCutoffTimestamp( $until );
+ return $keepSeconds;
}
- /**
- * Changes the cutoff timestamp to not affect more than
$this->mBatchSize
- * rows, if needed.
- *
- * @param int $until
- *
- * @return int
- */
- private function limitCutoffTimestamp( $until ) {
- $dbr = wfGetDB( DB_SLAVE );
- $changeTime = $dbr->selectField(
- 'wb_changes',
- 'change_time',
- array( 'change_time < ' . $dbr->addQuotes( wfTimestamp(
TS_MW, $until ) ) ),
- __METHOD__,
- array(
- 'OFFSET' => $this->mBatchSize,
- 'ORDER BY' => 'change_time ASC',
- )
- );
+ private function getGraceSeconds() {
+ $graceSeconds = 0;
+ $graceSeconds += intval( $this->getOption( 'grace-minutes', 0 )
) * 60;
- return $changeTime ? intval( $changeTime ) : $until;
- }
-
- private function doPrune() {
- while( true ) {
- wfWaitForSlaves();
-
- $until = $this->getCutoffTimestamp();
-
- $this->output( date( 'H:i:s' ) . " pruning entries
older than "
- . wfTimestamp( TS_ISO_8601, $until ) . "\n" );
-
- $affected = $this->pruneChanges( $until );
- $this->output( date( 'H:i:s' ) . " $affected rows
pruned.\n" );
-
- if ( $affected === 0 ) {
- break;
- }
+ if ( $graceSeconds === 0 ) {
+ // one hour
+ $graceSeconds = 1 * 60 * 60;
}
+
+ return $graceSeconds;
}
/**
- * Prunes all changes older than $until from the changes table.
- *
- * @param int $until
- *
- * @return int the number of changes deleted.
+ * @return MessageReporter
*/
- private function pruneChanges( $until ) {
- $dbw = wfGetDB( DB_MASTER );
+ private function newMessageReporter() {
+ $reporter = new ObservableMessageReporter();
- $dbw->delete(
- 'wb_changes',
- array( 'change_time < ' . $dbw->addQuotes( wfTimestamp(
TS_MW, $until ) ) ),
- __METHOD__
+ $self = $this; // evil PHP 5.3 ;)
+ $reporter->registerReporterCallback(
+ function ( $message ) use ( $self ) {
+ $self->log( $message );
+ }
);
- return $dbw->affectedRows();
+ return $reporter;
+ }
+
+ /**
+ * Log a message unless we are quiet.
+ *
+ * @param string $message
+ */
+ public function log( $message ) {
+ $this->output( date( 'H:i:s' ) . ' ' . $message . "\n",
'pruneChanges::log' );
+ $this->cleanupChanneled();
}
}
diff --git a/repo/tests/phpunit/includes/ChangePrunerTest.php
b/repo/tests/phpunit/includes/ChangePrunerTest.php
new file mode 100644
index 0000000..8721589
--- /dev/null
+++ b/repo/tests/phpunit/includes/ChangePrunerTest.php
@@ -0,0 +1,88 @@
+<?php
+
+namespace Wikibase\Test;
+
+use Wikibase\ChangesTable;
+use Wikibase\Lib\Reporting\MessageReporter;
+use Wikibase\Lib\Reporting\ObservableMessageReporter;
+use Wikibase\Repo\ChangePruner;
+
+/**
+ * @covers Wikibase\Repo\ChangePruner
+ *
+ * @group Database
+ * @group Wikibase
+ * @group WikibaseRepo
+ *
+ * @licence GNU GPL v2+
+ * @author Katie Filbert < aude.w...@gmail.com >
+ */
+class ChangePrunerTest extends \MediaWikiTestCase {
+
+ private $messages = array();
+
+ public function testDoPrune() {
+ $pruner = new ChangePruner( 1, 1, 1, false );
+
+ $dbw = wfGetDB( DB_MASTER );
+ $dbw->delete( 'wb_changes', '*' );
+
+ $this->assertEquals( 0, $dbw->selectRowCount( 'wb_changes' ),
+ 'sanity check: wb_changes table is empty' );
+
+ $this->addTestChanges();
+ $this->assertEquals( 2, $dbw->selectRowCount( 'wb_changes' ),
+ 'sanity check: 2 changes added to wb_changes'
+ );
+
+ $pruner->setMessageReporter( $this->newMessageReporter() );
+ $pruner->doPrune();
+
+ $this->assertEquals( 6, count( $this->messages ), 'pruner has
reported 6 messages' );
+
+ $this->assertContains( 'pruning entries older than
2015-01-01T00:03:00Z', $this->messages[0] );
+ $this->assertContains( '1 rows pruned', $this->messages[1] );
+ $this->assertContains( '1 rows pruned', $this->messages[3] );
+ $this->assertContains( '0 rows pruned', $this->messages[5] );
+
+ $this->assertEquals( 0, $dbw->selectRowCount( 'wb_changes' ),
'wb_changes table is empty' );
+ }
+
+ private function addTestChanges() {
+ $changesTable = new ChangesTable( false );
+
+ $change = $changesTable->newRow( $this->getChangeRowData(
'20150101000005' ), true );
+ $change->save();
+
+ $change = $changesTable->newRow( $this->getChangeRowData(
'20150101000300' ), true );
+ $change->save();
+ }
+
+ private function getChangeRowData( $timestamp ) {
+ return array(
+ 'type' => 'wikibase-item~update',
+ 'time' => $timestamp,
+ 'user_id' => 0,
+ 'revision_id' => 9002,
+ 'object_id' => 'Q9000',
+ 'info' => array( 'diff' => array() )
+ );
+ }
+
+ /**
+ * @return MessageReporter
+ */
+ private function newMessageReporter() {
+ $reporter = new ObservableMessageReporter();
+
+ $self = $this; // evil PHP 5.3 ;)
+ $reporter->registerReporterCallback(
+ function ( $message ) use ( $self ) {
+ $this->messages[] = $message;
+ }
+ );
+
+ return $reporter;
+ }
+
+}
--
To view, visit https://gerrit.wikimedia.org/r/204786
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I87065a723572639702b3a0f839d68fbeca14eac7
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Wikibase
Gerrit-Branch: master
Gerrit-Owner: Aude <aude.w...@gmail.com>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
