jenkins-bot has submitted this change and it was merged.
Change subject: Convert legacy format on export
......................................................................
Convert legacy format on export
Bug: 66719
Change-Id: I30d1c82700577e30788348a7a6bc2c5274783bcc
---
M docs/options.wiki
M repo/Wikibase.hooks.php
M repo/config/Wikibase.default.php
M repo/includes/content/EntityHandler.php
M repo/includes/content/ItemHandler.php
M repo/includes/content/PropertyHandler.php
M repo/tests/phpunit/includes/content/EntityHandlerTest.php
M repo/tests/phpunit/includes/content/ItemHandlerTest.php
M repo/tests/phpunit/includes/content/PropertyHandlerTest.php
9 files changed, 191 insertions(+), 12 deletions(-)
Approvals:
WikidataJenkins: Verified
Thiemo Mättig (WMDE): Looks good to me, approved
jenkins-bot: Verified
diff --git a/docs/options.wiki b/docs/options.wiki
index 5b8c6bc..c287e6c 100644
--- a/docs/options.wiki
+++ b/docs/options.wiki
@@ -59,6 +59,7 @@
:Default: <code>array( 'length' => 250 )</code>
;usePropertyInfoTable: Whether to use the wb_property_info table for quick
lookup of meta-information about properties. True per default, can be set to
false in an environment where the necessary database update can't be deployed
right away. To set up the table manually, run
lib/includes/store/sql/wb_property_info.sql to create it, then use
repo/maintenance/rebuildPropertyInfo.php to populate the table.
;internalEntitySerializerClass: The class name of a serializer that is to be
used when serializing entities for storage. Defaults to null, causing the
default entity serializer from the SerializerFactory to be used. Should be set
to Wikibase\Lib\Serializers\LegacyInternalEntitySerializer for compatibility
with client wikis that run older code.
+;transformLegacyFormatOnExport: Whether entity revisions stored in a legacy
format should be converted on the fly while exporting. Enabled per default.
== Client Settings ==
diff --git a/repo/Wikibase.hooks.php b/repo/Wikibase.hooks.php
index 1ef5df3..5b74c48 100644
--- a/repo/Wikibase.hooks.php
+++ b/repo/Wikibase.hooks.php
@@ -1239,6 +1239,7 @@
$termIndex = $repo->getStore()->getTermIndex();
$errorLocalizer = $repo->getValidatorErrorLocalizer();
$siteLinkStore = $repo->getStore()->newSiteLinkCache();
+ $transformOnExport = $repo->getSettings()->getSetting(
'transformLegacyFormatOnExport' );
return new ItemHandler(
$entityPerPage,
@@ -1246,7 +1247,8 @@
$codec,
array( $validator ),
$errorLocalizer,
- $siteLinkStore
+ $siteLinkStore,
+ $transformOnExport
);
}
@@ -1258,6 +1260,7 @@
$termIndex = $repo->getStore()->getTermIndex();
$errorLocalizer = $repo->getValidatorErrorLocalizer();
$propertyInfoStore = $repo->getStore()->getPropertyInfoStore();
+ $transformOnExport = $repo->getSettings()->getSetting(
'transformLegacyFormatOnExport' );
return new PropertyHandler(
$entityPerPage,
@@ -1265,7 +1268,8 @@
$codec,
array( $validator ),
$errorLocalizer,
- $propertyInfoStore
+ $propertyInfoStore,
+ $transformOnExport
);
}
diff --git a/repo/config/Wikibase.default.php b/repo/config/Wikibase.default.php
index 1d60523..cd5bb4c 100644
--- a/repo/config/Wikibase.default.php
+++ b/repo/config/Wikibase.default.php
@@ -80,6 +80,8 @@
// Can be used to override the serialization used for storage.
// Typical value:
Wikibase\Lib\Serializers\LegacyInternalEntitySerializer
'internalEntitySerializerClass' => null,
+
+ 'transformLegacyFormatOnExport' => true,
);
return $defaults;
diff --git a/repo/includes/content/EntityHandler.php
b/repo/includes/content/EntityHandler.php
index dd14cb2..201fafc 100644
--- a/repo/includes/content/EntityHandler.php
+++ b/repo/includes/content/EntityHandler.php
@@ -5,7 +5,6 @@
use Content;
use ContentHandler;
use DataUpdate;
-use Deserializers\Exceptions\DeserializationException;
use IContextSource;
use InvalidArgumentException;
use Language;
@@ -13,7 +12,6 @@
use ParserOptions;
use RequestContext;
use Revision;
-use Serializers\Exceptions\SerializationException;
use Title;
use User;
use ValueValidators\Result;
@@ -60,12 +58,18 @@
private $errorLocalizer;
/**
+ * @var
+ */
+ private $transformOnExport;
+
+ /**
* @param string $modelId
* @param EntityPerPage $entityPerPage
* @param TermIndex $termIndex
* @param EntityContentDataCodec $contentCodec
* @param EntityValidator[] $preSaveValidators
* @param ValidatorErrorLocalizer $errorLocalizer
+ * @param bool $transformOnExport
*/
public function __construct(
$modelId,
@@ -73,7 +77,8 @@
TermIndex $termIndex,
EntityContentDataCodec $contentCodec,
array $preSaveValidators,
- ValidatorErrorLocalizer $errorLocalizer
+ ValidatorErrorLocalizer $errorLocalizer,
+ $transformOnExport
) {
$formats = $contentCodec->getSupportedFormats();
@@ -84,6 +89,7 @@
$this->entityPerPage = $entityPerPage;
$this->termIndex = $termIndex;
$this->errorLocalizer = $errorLocalizer;
+ $this->transformOnExport = $transformOnExport;
}
/**
@@ -192,6 +198,48 @@
return $options;
}
+
+ /**
+ * @see ContentHandler::exportTransform
+ *
+ * @param string $blob
+ * @param string|null $format
+ *
+ * @return string|void
+ */
+ public function exportTransform( $blob, $format = null ) {
+ if ( $this->transformOnExport &&
$this->isBlobUsingLegacyFormat( $blob, $format ) ) {
+ $format = ( $format === null ) ?
$this->getDefaultFormat() : $format;
+
+ $content = $this->unserializeContent( $blob, $format );
+ $blob = $this->serializeContent( $content );
+ }
+
+ return $blob;
+ }
+
+ /**
+ * Detects blobs that may be using a legacy serialization format.
+ *
+ * @note: False positives (detecting a legacy format when really no
legacy format was used)
+ * are acceptable, false negatives (failing to detect a legacy format
when one was used)
+ * are not acceptable.
+ *
+ * @param string $blob
+ * @param string $format
+ *
+ * @return bool True if $blob seems to be using a legacy serialization
format.
+ */
+ protected function isBlobUsingLegacyFormat( $blob, $format ) {
+ // The legacy serialization uses something like
"entity":["item",21] or
+ // even "entity":"p21" for the entity ID.
+ if ( preg_match( '/"entity"\s*:/s', $blob ) > 0 ) {
+ return true;
+ }
+
+ return false;
+ }
+
/**
* Creates a Content object for the given Entity object.
*
diff --git a/repo/includes/content/ItemHandler.php
b/repo/includes/content/ItemHandler.php
index 5da34e1..4d17be4 100644
--- a/repo/includes/content/ItemHandler.php
+++ b/repo/includes/content/ItemHandler.php
@@ -32,6 +32,7 @@
* @param EntityValidator[] $preSaveValidators
* @param ValidatorErrorLocalizer $errorLocalizer
* @param SiteLinkCache $siteLinkStore
+ * @param bool $transformOnExport
*/
public function __construct(
EntityPerPage $entityPerPage,
@@ -39,7 +40,8 @@
EntityContentDataCodec $contentCodec,
array $preSaveValidators,
ValidatorErrorLocalizer $errorLocalizer,
- SiteLinkCache $siteLinkStore
+ SiteLinkCache $siteLinkStore,
+ $transformOnExport
) {
parent::__construct(
CONTENT_MODEL_WIKIBASE_ITEM,
@@ -47,7 +49,8 @@
$termIndex,
$contentCodec,
$preSaveValidators,
- $errorLocalizer
+ $errorLocalizer,
+ $transformOnExport
);
$this->siteLinkStore = $siteLinkStore;
diff --git a/repo/includes/content/PropertyHandler.php
b/repo/includes/content/PropertyHandler.php
index ce54d6b..44f9431 100644
--- a/repo/includes/content/PropertyHandler.php
+++ b/repo/includes/content/PropertyHandler.php
@@ -43,6 +43,7 @@
* @param EntityValidator[] $preSaveValidators
* @param ValidatorErrorLocalizer $errorLocalizer
* @param PropertyInfoStore $infoStore
+ * @param bool $transformOnExport
*/
public function __construct(
EntityPerPage $entityPerPage,
@@ -50,7 +51,8 @@
EntityContentDataCodec $contentCodec,
$preSaveValidators,
ValidatorErrorLocalizer $errorLocalizer,
- PropertyInfoStore $infoStore
+ PropertyInfoStore $infoStore,
+ $transformOnExport
) {
parent::__construct(
CONTENT_MODEL_WIKIBASE_PROPERTY,
@@ -58,7 +60,8 @@
$termIndex,
$contentCodec,
$preSaveValidators,
- $errorLocalizer
+ $errorLocalizer,
+ $transformOnExport
);
$this->infoStore = $infoStore;
diff --git a/repo/tests/phpunit/includes/content/EntityHandlerTest.php
b/repo/tests/phpunit/includes/content/EntityHandlerTest.php
index d60c153..b2d7a2a 100644
--- a/repo/tests/phpunit/includes/content/EntityHandlerTest.php
+++ b/repo/tests/phpunit/includes/content/EntityHandlerTest.php
@@ -5,12 +5,15 @@
use ContentHandler;
use Language;
use Revision;
+use Symfony\Component\Yaml\Exception\RuntimeException;
use Title;
use Wikibase\Entity;
use Wikibase\EntityContent;
use Wikibase\EntityFactory;
use Wikibase\EntityHandler;
+use Wikibase\Lib\Serializers\LegacyInternalEntitySerializer;
use Wikibase\Repo\WikibaseRepo;
+use Wikibase\SettingsArray;
/**
* @covers Wikibase\EntityHandler
@@ -41,11 +44,11 @@
}
/**
+ * @param SettingsArray $settings
+ *
* @return EntityHandler
*/
- protected function getHandler() {
- return ContentHandler::getForModelID( $this->getModelId() );
- }
+ protected abstract function getHandler( SettingsArray $settings = null
);
/**
* @return Entity
@@ -274,4 +277,48 @@
$this->assertEquals( $this->getModelId(), $content->getModel()
);
}
+ public function exportTransformProvider() {
+ $entity = $this->newEntity();
+
+ //FIXME: We need a better way to set an ID. If091211c1d1d
changes how
+ // entity creation is handled in tests.
+ $entity->setId( 7 );
+
+ $legacySerializer = new LegacyInternalEntitySerializer();
+ $oldBlob = json_encode( $legacySerializer->serialize( $entity )
);
+
+ // replace "entity":["item",7] with "entity":"q7"
+ $id = $entity->getId()->getSerialization();
+ $veryOldBlob = preg_replace( '/"entity":\["\w+",\d+\]/',
'"entity":"' . strtolower( $id ) . '"', $oldBlob );
+
+ // sanity
+ if ( $oldBlob == $veryOldBlob ) {
+ throw new RuntimeException( 'Failed to fake very old
serialization format based on oldish serialization format.' );
+ }
+
+ $currentSerializer =
WikibaseRepo::getDefaultInstance()->getInternalEntitySerializer();
+ $newBlob = json_encode( $currentSerializer->serialize( $entity
) );
+
+ return array(
+ 'old serialization / ancient id format' => array(
$veryOldBlob, $newBlob ),
+ 'old serialization / new silly id format' => array(
$oldBlob, $newBlob ),
+ 'new serialization format, keep as is' => array(
$newBlob, $newBlob ),
+ );
+ }
+
+ /**
+ * @dataProvider exportTransformProvider
+ *
+ * @param $blob
+ * @param $expected
+ */
+ public function testExportTransform( $blob, $expected ) {
+ $settings = new SettingsArray();
+ $settings->setSetting( 'transformLegacyFormatOnExport', true );
+ $handler = $this->getHandler( $settings );
+
+ $actual = $handler->exportTransform( $blob );
+ $this->assertEquals( $expected, $actual );
+ }
+
}
diff --git a/repo/tests/phpunit/includes/content/ItemHandlerTest.php
b/repo/tests/phpunit/includes/content/ItemHandlerTest.php
index 8916710..4339dfa 100644
--- a/repo/tests/phpunit/includes/content/ItemHandlerTest.php
+++ b/repo/tests/phpunit/includes/content/ItemHandlerTest.php
@@ -6,7 +6,11 @@
use Wikibase\DataModel\Entity\Item;
use Wikibase\DataModel\Entity\ItemId;
use Wikibase\DataModel\SimpleSiteLink;
+use Wikibase\EntityHandler;
use Wikibase\ItemContent;
+use Wikibase\ItemHandler;
+use Wikibase\Repo\WikibaseRepo;
+use Wikibase\SettingsArray;
/**
* @covers Wikibase\ItemHandler
@@ -74,4 +78,35 @@
return Item::newEmpty();
}
+ /**
+ * @param SettingsArray $settings
+ *
+ * @return EntityHandler
+ */
+ protected function getHandler( SettingsArray $settings = null ) {
+ $repo = WikibaseRepo::getDefaultInstance();
+ $validator =
$repo->getEntityConstraintProvider()->getConstraints( Item::ENTITY_TYPE );
+ $codec = $repo->getEntityContentDataCodec();
+ $entityPerPage = $repo->getStore()->newEntityPerPage();
+ $termIndex = $repo->getStore()->getTermIndex();
+ $errorLocalizer = $repo->getValidatorErrorLocalizer();
+ $siteLinkStore = $repo->getStore()->newSiteLinkCache();
+
+ if ( !$settings ) {
+ $settings = $repo->getSettings();
+ }
+
+ $transformOnExport = $settings->getSetting(
'transformLegacyFormatOnExport' );
+
+ return new ItemHandler(
+ $entityPerPage,
+ $termIndex,
+ $codec,
+ array( $validator ),
+ $errorLocalizer,
+ $siteLinkStore,
+ $transformOnExport
+ );
+ }
+
}
diff --git a/repo/tests/phpunit/includes/content/PropertyHandlerTest.php
b/repo/tests/phpunit/includes/content/PropertyHandlerTest.php
index ad63bb9..aff6afd 100644
--- a/repo/tests/phpunit/includes/content/PropertyHandlerTest.php
+++ b/repo/tests/phpunit/includes/content/PropertyHandlerTest.php
@@ -5,7 +5,11 @@
use Title;
use Wikibase\DataModel\Entity\Property;
use Wikibase\DataModel\Entity\PropertyId;
+use Wikibase\EntityHandler;
use Wikibase\PropertyContent;
+use Wikibase\PropertyHandler;
+use Wikibase\Repo\WikibaseRepo;
+use Wikibase\SettingsArray;
/**
* @covers Wikibase\PropertyHandler
@@ -73,4 +77,36 @@
return Property::newFromType( 'string' );
}
+
+ /**
+ * @param SettingsArray $settings
+ *
+ * @return EntityHandler
+ */
+ protected function getHandler( SettingsArray $settings = null ) {
+ $repo = WikibaseRepo::getDefaultInstance();
+ $validator =
$repo->getEntityConstraintProvider()->getConstraints( Property::ENTITY_TYPE );
+ $codec = $repo->getEntityContentDataCodec();
+ $entityPerPage = $repo->getStore()->newEntityPerPage();
+ $termIndex = $repo->getStore()->getTermIndex();
+ $errorLocalizer = $repo->getValidatorErrorLocalizer();
+ $propertyInfoStore = $repo->getStore()->getPropertyInfoStore();
+
+ if ( !$settings ) {
+ $settings = $repo->getSettings();
+ }
+
+ $transformOnExport = $settings->getSetting(
'transformLegacyFormatOnExport' );
+
+ return new PropertyHandler(
+ $entityPerPage,
+ $termIndex,
+ $codec,
+ array( $validator ),
+ $errorLocalizer,
+ $propertyInfoStore,
+ $transformOnExport
+ );
+ }
+
}
--
To view, visit https://gerrit.wikimedia.org/r/140366
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I30d1c82700577e30788348a7a6bc2c5274783bcc
Gerrit-PatchSet: 10
Gerrit-Project: mediawiki/extensions/Wikibase
Gerrit-Branch: master
Gerrit-Owner: Daniel Kinzler <[email protected]>
Gerrit-Reviewer: Aude <[email protected]>
Gerrit-Reviewer: Daniel Kinzler <[email protected]>
Gerrit-Reviewer: Hoo man <[email protected]>
Gerrit-Reviewer: Thiemo Mättig (WMDE) <[email protected]>
Gerrit-Reviewer: WikidataJenkins <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
