[MediaWiki-commits] [Gerrit] (bug 52799) Introducing dumpJson. - change (mediawiki...Wikibase)
jenkins-bot has submitted this change and it was merged.
Change subject: (bug 52799) Introducing dumpJson.
..
(bug 52799) Introducing dumpJson.
This allows JSON dumps to be created for all entities.
This is a baseline implementation.
In later changes, this will be extended to allow for
entities to be filtered by type, ids to be loaded from file, etc.
Change-Id: I09eaa2a7b6fa9cb6a5ffc57b5f28c30e93d79f8b
---
M lib/WikibaseLib.classes.php
A lib/includes/Dumpers/JsonDumpGenerator.php
A lib/tests/phpunit/Dumpers/JsonDumpGeneratorTest.php
M repo/Wikibase.classes.php
M repo/includes/store/EntityPerPage.php
A repo/includes/store/sql/ConvertingResultWrapper.php
A repo/includes/store/sql/DatabaseRowEntityIdIterator.php
M repo/includes/store/sql/EntityPerPageTable.php
A repo/maintenance/dumpJson.php
A repo/tests/phpunit/includes/store/sql/DatabaseRowEntityIdIteratorTest.php
A repo/tests/phpunit/includes/store/sql/EntityPerPageTableTest.php
11 files changed, 783 insertions(+), 0 deletions(-)
Approvals:
Addshore: Looks good to me, approved
jenkins-bot: Verified
diff --git a/lib/WikibaseLib.classes.php b/lib/WikibaseLib.classes.php
index 09d3bec..c6120b8 100644
--- a/lib/WikibaseLib.classes.php
+++ b/lib/WikibaseLib.classes.php
@@ -84,6 +84,9 @@
'Wikibase\EntityDiffVisualizer' =>
'includes/EntityDiffVisualizer.php',
'Wikibase\EntityFactory' => 'includes/EntityFactory.php',
+ // includes/Dumpers
+ 'Wikibase\Dumpers\JsonDumpGenerator' =>
'includes/Dumpers/JsonDumpGenerator.php',
+
// includes/formatters
'Wikibase\Lib\DispatchingSnakFormatter' =>
'includes/formatters/DispatchingSnakFormatter.php',
'Wikibase\Lib\EntityIdFormatter' =>
'includes/formatters/EntityIdFormatter.php',
diff --git a/lib/includes/Dumpers/JsonDumpGenerator.php
b/lib/includes/Dumpers/JsonDumpGenerator.php
new file mode 100644
index 000..20f007f
--- /dev/null
+++ b/lib/includes/Dumpers/JsonDumpGenerator.php
@@ -0,0 +1,144 @@
+out = $out;
+ $this->entitySerializer = $entitySerializer;
+ $this->entityLookup = $lookup;
+ }
+
+ /**
+* Generates a JSON dump, writing to the file handle provided to the
constructor.
+*
+* @param Traversable $idStream an Iterator that returns EntityId
instances
+*/
+ public function generateDump( Traversable $idStream ) {
+
+ $json = "[\n"; //TODO: make optional
+ $this->writeToDump( $json );
+
+ $i = 0;
+
+ /* @var EntityId $id */
+ foreach ( $idStream as $id ) {
+ try {
+ if ( $i++ > 0 ) {
+ $this->writeToDump( ",\n" );
+ }
+
+ $entity = $this->entityLookup->getEntity( $id );
+ $data = $this->entitySerializer->getSerialized(
$entity );
+ $json = $this->encode( $data );
+ $this->writeToDump( $json );
+ } catch ( StorageException $ex ) {
+ $this->handleStorageException( $ex );
+ }
+ }
+
+ $json = "]\n"; //TODO: make optional
+ $this->writeToDump( $json );
+ }
+
+ /**
+* @param $ex
+*/
+ private function handleStorageException( $ex ) {
+ //TODO: optionally, log & ignore.
+ throw $ex;
+ }
+
+ /**
+* Encodes the given data as JSON
+*
+* @param $data
+*
+* @return string
+* @throws \MWException
+*/
+ public function encode( $data ) {
+ $json = json_encode( $data, $this->jsonFlags );
+
+ if ( $json === false ) {
+ // TODO: optionally catch & skip this
+ throw new MWException( 'Failed to encode data
structure.' );
+ }
+
+ return $json;
+ }
+
+ /**
+* Writers the given string to the output provided to the constructor.
+*
+* @param $json
+*/
+ private function writeToDump( $json ) {
+ //TODO: use output stream object
+ fwrite( $this->out, $json );
+ }
+
+ /**
+* Flags to use with json_encode as a bit field, see PHP's JSON_XXX
constants.
+*
+* @param int $jsonFlags
+*/
+ public function setJsonFlags( $jsonFlags ) {
+ $this->jsonFlags = $jsonFlags;
+ }
+
+ /**
+* Flags to use with json_encode as a bit field, see PHP's JSON_XXX
constants.
+*
+* @return int
+*/
+ public function getJsonFlags() {
+ return $thi
[MediaWiki-commits] [Gerrit] (bug 52799) Introducing dumpJson. - change (mediawiki...Wikibase)
Daniel Kinzler has uploaded a new change for review.
https://gerrit.wikimedia.org/r/83816
Change subject: (bug 52799) Introducing dumpJson.
..
(bug 52799) Introducing dumpJson.
This allows JSON dumps to be created for all entities.
This is a baseline implementation.
In later changes, this will be extended to allow for
entities to be filtered by type, ids to be loaded from file, etc.
Change-Id: I09eaa2a7b6fa9cb6a5ffc57b5f28c30e93d79f8b
---
M lib/WikibaseLib.classes.php
A lib/includes/Dumpers/JsonDumpGenerator.php
A lib/tests/phpunit/Dumpers/JsonDumpGeneratorTest.php
M repo/Wikibase.classes.php
M repo/includes/store/EntityPerPage.php
A repo/includes/store/sql/ConvertingResultWrapper.php
A repo/includes/store/sql/DatabaseRowEntityIdIterator.php
M repo/includes/store/sql/EntityPerPageTable.php
A repo/maintenance/dumpJson.php
A repo/tests/phpunit/includes/store/sql/DatabaseRowEntityIdIteratorTest.php
A repo/tests/phpunit/includes/store/sql/EntityPerPageTableTest.php
11 files changed, 782 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Wikibase
refs/changes/16/83816/1
diff --git a/lib/WikibaseLib.classes.php b/lib/WikibaseLib.classes.php
index f63ede2..af37853 100644
--- a/lib/WikibaseLib.classes.php
+++ b/lib/WikibaseLib.classes.php
@@ -85,6 +85,9 @@
'Wikibase\EntityDiffVisualizer' =>
'includes/EntityDiffVisualizer.php',
'Wikibase\EntityFactory' => 'includes/EntityFactory.php',
+ // includes/Dumpers
+ 'Wikibase\Dumpers\JsonDumpGenerator' =>
'includes/Dumpers/JsonDumpGenerator.php',
+
// includes/formatters
'Wikibase\Lib\EntityIdFormatter' =>
'includes/formatters/EntityIdFormatter.php',
'Wikibase\Lib\EntityIdLabelFormatter' =>
'includes/formatters/EntityIdLabelFormatter.php',
diff --git a/lib/includes/Dumpers/JsonDumpGenerator.php
b/lib/includes/Dumpers/JsonDumpGenerator.php
new file mode 100644
index 000..20f007f
--- /dev/null
+++ b/lib/includes/Dumpers/JsonDumpGenerator.php
@@ -0,0 +1,144 @@
+out = $out;
+ $this->entitySerializer = $entitySerializer;
+ $this->entityLookup = $lookup;
+ }
+
+ /**
+* Generates a JSON dump, writing to the file handle provided to the
constructor.
+*
+* @param Traversable $idStream an Iterator that returns EntityId
instances
+*/
+ public function generateDump( Traversable $idStream ) {
+
+ $json = "[\n"; //TODO: make optional
+ $this->writeToDump( $json );
+
+ $i = 0;
+
+ /* @var EntityId $id */
+ foreach ( $idStream as $id ) {
+ try {
+ if ( $i++ > 0 ) {
+ $this->writeToDump( ",\n" );
+ }
+
+ $entity = $this->entityLookup->getEntity( $id );
+ $data = $this->entitySerializer->getSerialized(
$entity );
+ $json = $this->encode( $data );
+ $this->writeToDump( $json );
+ } catch ( StorageException $ex ) {
+ $this->handleStorageException( $ex );
+ }
+ }
+
+ $json = "]\n"; //TODO: make optional
+ $this->writeToDump( $json );
+ }
+
+ /**
+* @param $ex
+*/
+ private function handleStorageException( $ex ) {
+ //TODO: optionally, log & ignore.
+ throw $ex;
+ }
+
+ /**
+* Encodes the given data as JSON
+*
+* @param $data
+*
+* @return string
+* @throws \MWException
+*/
+ public function encode( $data ) {
+ $json = json_encode( $data, $this->jsonFlags );
+
+ if ( $json === false ) {
+ // TODO: optionally catch & skip this
+ throw new MWException( 'Failed to encode data
structure.' );
+ }
+
+ return $json;
+ }
+
+ /**
+* Writers the given string to the output provided to the constructor.
+*
+* @param $json
+*/
+ private function writeToDump( $json ) {
+ //TODO: use output stream object
+ fwrite( $this->out, $json );
+ }
+
+ /**
+* Flags to use with json_encode as a bit field, see PHP's JSON_XXX
constants.
+*
+* @param int $jsonFlags
+*/
+ public function setJsonFlags( $jsonFlags ) {
+ $this->jsonFlags = $jsonFlags;
+ }
+
+ /**
+* Flags to use with json_encode as a bit field, see PHP's JSON_XXX
constants.
+*
+* @return int
+*/
+ publ
