jenkins-bot has submitted this change and it was merged. Change subject: Add on option to dumpJson for outputting json snippets ......................................................................
Add on option to dumpJson for outputting json snippets Change-Id: Ib621073e17d21e986b3534832460e3413e5018f6 --- M repo/includes/Dumpers/JsonDumpGenerator.php M repo/maintenance/dumpJson.php M repo/tests/phpunit/Dumpers/JsonDumpGeneratorTest.php 3 files changed, 66 insertions(+), 7 deletions(-) Approvals: WikidataJenkins: Verified Thiemo Mättig (WMDE): Looks good to me, approved jenkins-bot: Verified diff --git a/repo/includes/Dumpers/JsonDumpGenerator.php b/repo/includes/Dumpers/JsonDumpGenerator.php index b5c53d2..0ed60a3 100644 --- a/repo/includes/Dumpers/JsonDumpGenerator.php +++ b/repo/includes/Dumpers/JsonDumpGenerator.php @@ -55,12 +55,17 @@ */ protected $shardingFactor = 1; - /* + /** * @var int Number of the requested shard */ protected $shard = 0; - /* + /** + * @var bool + */ + private $useSnippets = false; + + /** * @var string|null */ protected $entityType = null; @@ -102,7 +107,7 @@ * * @param int $batchSize * - * @throws \InvalidArgumentException + * @throws InvalidArgumentException */ public function setBatchSize( $batchSize ) { if ( !is_int( $batchSize ) || $batchSize < 1 ) { @@ -110,6 +115,13 @@ } $this->batchSize = $batchSize; + } + + /** + * @param bool Whether to output valid json (false) or only comma separated entities + */ + public function setUseSnippets( $useSnippets ) { + $this->useSnippets = (bool)$useSnippets; } /** @@ -194,8 +206,9 @@ */ public function generateDump( EntityIdPager $idPager ) { - $json = "[\n"; //TODO: make optional - $this->writeToDump( $json ); + if ( !$this->useSnippets ) { + $this->writeToDump( "[\n" ); + } $dumpCount = 0; @@ -206,8 +219,10 @@ $this->progressReporter->reportMessage( 'Processed ' . $dumpCount . ' entities.' ); }; - $json = "\n]\n"; //TODO: make optional - $this->writeToDump( $json ); + if ( !$this->useSnippets ) { + $this->writeToDump( "\n]" ); + } + $this->writeToDump( "\n" ); } /** diff --git a/repo/maintenance/dumpJson.php b/repo/maintenance/dumpJson.php index 0883b9a..19805bf 100644 --- a/repo/maintenance/dumpJson.php +++ b/repo/maintenance/dumpJson.php @@ -64,6 +64,7 @@ $this->addOption( 'output', "Output file (default is stdout). Will be overwritten.", false, true ); $this->addOption( 'log', "Log file (default is stderr). Will be appended.", false, true ); $this->addOption( 'quiet', "Disable progress reporting", false, false ); + $this->addOption( 'snippet', "Output a JSON snippet without square brackets at the start and end. Allows output to be combined more freely.", false, false ); } public function initServices() { @@ -148,6 +149,7 @@ $shardingFactor = (int)$this->getOption( 'sharding-factor', 1 ); $shard = (int)$this->getOption( 'shard', 0 ); $batchSize = (int)$this->getOption( 'batch-size', 100 ); + $snippets = (bool)$this->getOption( 'snippet', false ); //TODO: Allow injection of an OutputStream for logging $this->openLogFile( $this->getOption( 'log', 'php://stderr' ) ); @@ -190,6 +192,7 @@ $dumper->setShardingFilter( $shardingFactor, $shard ); $dumper->setEntityTypeFilter( $entityType ); $dumper->setBatchSize( $batchSize ); + $dumper->setUseSnippets( $snippets ); $idStream = $this->makeIdStream( $entityType, $exceptionReporter ); $dumper->generateDump( $idStream ); diff --git a/repo/tests/phpunit/Dumpers/JsonDumpGeneratorTest.php b/repo/tests/phpunit/Dumpers/JsonDumpGeneratorTest.php index bdee978..43d0872 100644 --- a/repo/tests/phpunit/Dumpers/JsonDumpGeneratorTest.php +++ b/repo/tests/phpunit/Dumpers/JsonDumpGeneratorTest.php @@ -410,4 +410,45 @@ ob_end_clean(); } + /** + * @dataProvider useSnippetsProvider + */ + public function testSnippets( + array $ids, + $shardingFactor, + $shard + ) { + $dumper = $this->newDumpGenerator( $ids ); + $dumper->setUseSnippets( true ); + + $pager = $this->makeIdPager( $ids ); + $dumper->setShardingFilter( $shardingFactor, $shard ); + + // Generate the dump and grab the output + ob_start(); + $dumper->generateDump( $pager ); + $json = trim( ob_get_clean() ); + + $this->assertStringStartsWith( '{', $json, 'Snippet starts with {' ); + $this->assertStringEndsWith( '}', $json, 'Snippet ends with }' ); + } + + public static function useSnippetsProvider() { + $ids = array(); + + for ( $i = 1; $i < 5; $i++ ) { + $ids[] = new ItemId( "Q$i" ); + } + + return array( + // Only one shard + array( $ids, 1, 0, ), + + // Three shards + array( $ids, 3, 0, ), + array( $ids, 3, 1, ), + array( $ids, 3, 2, ), + ); + } + } -- To view, visit https://gerrit.wikimedia.org/r/139367 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ib621073e17d21e986b3534832460e3413e5018f6 Gerrit-PatchSet: 10 Gerrit-Project: mediawiki/extensions/Wikibase Gerrit-Branch: master Gerrit-Owner: Hoo man <h...@online.de> Gerrit-Reviewer: Aude <aude.w...@gmail.com> Gerrit-Reviewer: Daniel Kinzler <daniel.kinz...@wikimedia.de> Gerrit-Reviewer: Hoo man <h...@online.de> Gerrit-Reviewer: Jeroen De Dauw <jeroended...@gmail.com> Gerrit-Reviewer: Thiemo Mättig (WMDE) <thiemo.maet...@wikimedia.de> Gerrit-Reviewer: WikidataJenkins <wikidata-servi...@wikimedia.de> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits