jenkins-bot has submitted this change and it was merged.
Change subject: Add on option to dumpJson for outputting json snippets
......................................................................
Add on option to dumpJson for outputting json snippets
Change-Id: Ib621073e17d21e986b3534832460e3413e5018f6
---
M repo/includes/Dumpers/JsonDumpGenerator.php
M repo/maintenance/dumpJson.php
M repo/tests/phpunit/Dumpers/JsonDumpGeneratorTest.php
3 files changed, 66 insertions(+), 7 deletions(-)
Approvals:
WikidataJenkins: Verified
Thiemo Mättig (WMDE): Looks good to me, approved
jenkins-bot: Verified
diff --git a/repo/includes/Dumpers/JsonDumpGenerator.php
b/repo/includes/Dumpers/JsonDumpGenerator.php
index b5c53d2..0ed60a3 100644
--- a/repo/includes/Dumpers/JsonDumpGenerator.php
+++ b/repo/includes/Dumpers/JsonDumpGenerator.php
@@ -55,12 +55,17 @@
*/
protected $shardingFactor = 1;
- /*
+ /**
* @var int Number of the requested shard
*/
protected $shard = 0;
- /*
+ /**
+ * @var bool
+ */
+ private $useSnippets = false;
+
+ /**
* @var string|null
*/
protected $entityType = null;
@@ -102,7 +107,7 @@
*
* @param int $batchSize
*
- * @throws \InvalidArgumentException
+ * @throws InvalidArgumentException
*/
public function setBatchSize( $batchSize ) {
if ( !is_int( $batchSize ) || $batchSize < 1 ) {
@@ -110,6 +115,13 @@
}
$this->batchSize = $batchSize;
+ }
+
+ /**
+ * @param bool Whether to output valid json (false) or only comma
separated entities
+ */
+ public function setUseSnippets( $useSnippets ) {
+ $this->useSnippets = (bool)$useSnippets;
}
/**
@@ -194,8 +206,9 @@
*/
public function generateDump( EntityIdPager $idPager ) {
- $json = "[\n"; //TODO: make optional
- $this->writeToDump( $json );
+ if ( !$this->useSnippets ) {
+ $this->writeToDump( "[\n" );
+ }
$dumpCount = 0;
@@ -206,8 +219,10 @@
$this->progressReporter->reportMessage( 'Processed ' .
$dumpCount . ' entities.' );
};
- $json = "\n]\n"; //TODO: make optional
- $this->writeToDump( $json );
+ if ( !$this->useSnippets ) {
+ $this->writeToDump( "\n]" );
+ }
+ $this->writeToDump( "\n" );
}
/**
diff --git a/repo/maintenance/dumpJson.php b/repo/maintenance/dumpJson.php
index 0883b9a..19805bf 100644
--- a/repo/maintenance/dumpJson.php
+++ b/repo/maintenance/dumpJson.php
@@ -64,6 +64,7 @@
$this->addOption( 'output', "Output file (default is stdout).
Will be overwritten.", false, true );
$this->addOption( 'log', "Log file (default is stderr). Will be
appended.", false, true );
$this->addOption( 'quiet', "Disable progress reporting", false,
false );
+ $this->addOption( 'snippet', "Output a JSON snippet without
square brackets at the start and end. Allows output to be combined more
freely.", false, false );
}
public function initServices() {
@@ -148,6 +149,7 @@
$shardingFactor = (int)$this->getOption( 'sharding-factor', 1 );
$shard = (int)$this->getOption( 'shard', 0 );
$batchSize = (int)$this->getOption( 'batch-size', 100 );
+ $snippets = (bool)$this->getOption( 'snippet', false );
//TODO: Allow injection of an OutputStream for logging
$this->openLogFile( $this->getOption( 'log', 'php://stderr' ) );
@@ -190,6 +192,7 @@
$dumper->setShardingFilter( $shardingFactor, $shard );
$dumper->setEntityTypeFilter( $entityType );
$dumper->setBatchSize( $batchSize );
+ $dumper->setUseSnippets( $snippets );
$idStream = $this->makeIdStream( $entityType,
$exceptionReporter );
$dumper->generateDump( $idStream );
diff --git a/repo/tests/phpunit/Dumpers/JsonDumpGeneratorTest.php
b/repo/tests/phpunit/Dumpers/JsonDumpGeneratorTest.php
index bdee978..43d0872 100644
--- a/repo/tests/phpunit/Dumpers/JsonDumpGeneratorTest.php
+++ b/repo/tests/phpunit/Dumpers/JsonDumpGeneratorTest.php
@@ -410,4 +410,45 @@
ob_end_clean();
}
+ /**
+ * @dataProvider useSnippetsProvider
+ */
+ public function testSnippets(
+ array $ids,
+ $shardingFactor,
+ $shard
+ ) {
+ $dumper = $this->newDumpGenerator( $ids );
+ $dumper->setUseSnippets( true );
+
+ $pager = $this->makeIdPager( $ids );
+ $dumper->setShardingFilter( $shardingFactor, $shard );
+
+ // Generate the dump and grab the output
+ ob_start();
+ $dumper->generateDump( $pager );
+ $json = trim( ob_get_clean() );
+
+ $this->assertStringStartsWith( '{', $json, 'Snippet starts with
{' );
+ $this->assertStringEndsWith( '}', $json, 'Snippet ends with }'
);
+ }
+
+ public static function useSnippetsProvider() {
+ $ids = array();
+
+ for ( $i = 1; $i < 5; $i++ ) {
+ $ids[] = new ItemId( "Q$i" );
+ }
+
+ return array(
+ // Only one shard
+ array( $ids, 1, 0, ),
+
+ // Three shards
+ array( $ids, 3, 0, ),
+ array( $ids, 3, 1, ),
+ array( $ids, 3, 2, ),
+ );
+ }
+
}
--
To view, visit https://gerrit.wikimedia.org/r/139367
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ib621073e17d21e986b3534832460e3413e5018f6
Gerrit-PatchSet: 10
Gerrit-Project: mediawiki/extensions/Wikibase
Gerrit-Branch: master
Gerrit-Owner: Hoo man <h...@online.de>
Gerrit-Reviewer: Aude <aude.w...@gmail.com>
Gerrit-Reviewer: Daniel Kinzler <daniel.kinz...@wikimedia.de>
Gerrit-Reviewer: Hoo man <h...@online.de>
Gerrit-Reviewer: Jeroen De Dauw <jeroended...@gmail.com>
Gerrit-Reviewer: Thiemo Mättig (WMDE) <thiemo.maet...@wikimedia.de>
Gerrit-Reviewer: WikidataJenkins <wikidata-servi...@wikimedia.de>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
