[MediaWiki-commits] [Gerrit] Clean up CentralNotice Translation Metadata - change (mediawiki...CentralNotice)
Adamw has submitted this change and it was merged.
Change subject: Clean up CentralNotice Translation Metadata
..
Clean up CentralNotice Translation Metadata
Right now the translation metadata job is running very slowly --
part of this problem is that there is a lot of junk metadata
that we have to process.
So... the maintenance script cleans things up. And I'm now putting
the banner id in with the revtag object so that we can eventually
look to see if the banner is archived or attached only to archived
campaigns or something.
E.g. this patch alone will take 800 objects on meta down to ~250.
Bug: 53769
Bug: 53792
Change-Id: If9ae70977ee0f008f65ef8f742acafb991b4d221
---
M includes/Banner.php
A maintenance/CleanCNTranslateMetadata.php
2 files changed, 142 insertions(+), 5 deletions(-)
Approvals:
Adamw: Looks good to me, approved
Nikerabbit: Looks good to me, but someone else must approve
Mwalker: Looks good to me, but someone else must approve
diff --git a/includes/Banner.php b/includes/Banner.php
index 5cdbe59..9319485 100644
--- a/includes/Banner.php
+++ b/includes/Banner.php
@@ -842,7 +842,7 @@
$fields = $this-extractMessageFields(
$this-bodyContent );
if ( count( $fields ) 0 ) {
// Tag the banner for
translation
- Banner::addTag(
'banner:translate', $revisionId, $pageId );
+ Banner::addTag(
'banner:translate', $revisionId, $pageId, $this-getId() );
$this-runTranslateJob = true;
}
}
@@ -1147,25 +1147,27 @@
* @param string $tag The name of the tag
* @param integer $revisionId ID of the revision
* @param integer $pageId ID of the MediaWiki page for the banner
-* @param string $value Value to store for the tag
+* @param string $bannerId ID of banner this revtag belongs to
* @throws MWException
*/
- static function addTag( $tag, $revisionId, $pageId, $value = null ) {
+ static function addTag( $tag, $revisionId, $pageId, $bannerId ) {
$dbw = CNDatabase::getDb();
if ( is_object( $revisionId ) ) {
throw new MWException( 'Got object, excepted id' );
}
+ // There should only ever be one tag applied to a banner object
+ Banner::removeTag( $tag, $pageId );
+
$conds = array(
'rt_page' = $pageId,
'rt_type' = RevTag::getType( $tag ),
'rt_revision' = $revisionId
);
- $dbw-delete( 'revtag', $conds, __METHOD__ );
if ( $value !== null ) {
- $conds['rt_value'] = serialize( implode( '|', $value )
);
+ $conds['rt_value'] = $bannerId;
}
$dbw-insert( 'revtag', $conds, __METHOD__ );
diff --git a/maintenance/CleanCNTranslateMetadata.php
b/maintenance/CleanCNTranslateMetadata.php
new file mode 100644
index 000..b93dbed
--- /dev/null
+++ b/maintenance/CleanCNTranslateMetadata.php
@@ -0,0 +1,135 @@
+?php
+
+$IP = getenv( 'MW_INSTALL_PATH' );
+if ( $IP === false ) {
+ $IP = __DIR__ . '/../../..';
+}
+require_once $IP/maintenance/Maintenance.php;
+
+/**
+ * Cleans up the Revision Tag table which is where CentralNotice stores
+ * metadata required for the Translate extension.
+ *
+ * So far this class:
+ * * Removes duplicate revision entries (there should be only one per banner)
+ * * Associates entries with a banner by name
+ * * Removes entries that have no banner object
+ *
+ * Class CleanCNTranslateMetadata
+ */
+class CleanCNTranslateMetadata extends Maintenance {
+ protected $ttag;
+
+ public function execute() {
+ $this-ttag = RevTag::getType( 'banner:translate' );
+
+ $this-cleanDuplicates();
+ $this-populateIDs();
+ $this-deleteOrphans();
+ }
+
+ /**
+* Remove duplicated revtags
+*/
+ protected function cleanDuplicates() {
+ $this-output( Cleaning duplicates\n );
+
+ $db = CNDatabase::getDb( DB_MASTER );
+
+ $res = $db-select(
+ 'revtag',
+ array(
+ 'rt_page',
+ 'maxrev' = 'max(rt_revision)',
+ 'count' = 'count(*)'
+ ),
+ array( 'rt_type' = $this-ttag ),
+ __METHOD__,
+ array( 'GROUP BY' = 'rt_page' )
+ );
+
+ foreach (
[MediaWiki-commits] [Gerrit] Clean up CentralNotice Translation Metadata - change (mediawiki...CentralNotice)
Mwalker has uploaded a new change for review.
https://gerrit.wikimedia.org/r/83980
Change subject: Clean up CentralNotice Translation Metadata
..
Clean up CentralNotice Translation Metadata
Right now the translation metadata job is running very slowly --
part of this problem is that there is a lot of junk metadata
that we have to process.
So... the maintenance script cleans things up. And I'm now putting
the banner id in with the revtag object so that we can eventually
look to see if the banner is archived or attached only to archived
campaigns or something.
Bug: 53769
Bug: 53792
Change-Id: If9ae70977ee0f008f65ef8f742acafb991b4d221
---
M includes/Banner.php
A maintenance/CleanCNTranslateMetadata.php
2 files changed, 139 insertions(+), 5 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/CentralNotice
refs/changes/80/83980/1
diff --git a/includes/Banner.php b/includes/Banner.php
index 5cdbe59..9319485 100644
--- a/includes/Banner.php
+++ b/includes/Banner.php
@@ -842,7 +842,7 @@
$fields = $this-extractMessageFields(
$this-bodyContent );
if ( count( $fields ) 0 ) {
// Tag the banner for
translation
- Banner::addTag(
'banner:translate', $revisionId, $pageId );
+ Banner::addTag(
'banner:translate', $revisionId, $pageId, $this-getId() );
$this-runTranslateJob = true;
}
}
@@ -1147,25 +1147,27 @@
* @param string $tag The name of the tag
* @param integer $revisionId ID of the revision
* @param integer $pageId ID of the MediaWiki page for the banner
-* @param string $value Value to store for the tag
+* @param string $bannerId ID of banner this revtag belongs to
* @throws MWException
*/
- static function addTag( $tag, $revisionId, $pageId, $value = null ) {
+ static function addTag( $tag, $revisionId, $pageId, $bannerId ) {
$dbw = CNDatabase::getDb();
if ( is_object( $revisionId ) ) {
throw new MWException( 'Got object, excepted id' );
}
+ // There should only ever be one tag applied to a banner object
+ Banner::removeTag( $tag, $pageId );
+
$conds = array(
'rt_page' = $pageId,
'rt_type' = RevTag::getType( $tag ),
'rt_revision' = $revisionId
);
- $dbw-delete( 'revtag', $conds, __METHOD__ );
if ( $value !== null ) {
- $conds['rt_value'] = serialize( implode( '|', $value )
);
+ $conds['rt_value'] = $bannerId;
}
$dbw-insert( 'revtag', $conds, __METHOD__ );
diff --git a/maintenance/CleanCNTranslateMetadata.php
b/maintenance/CleanCNTranslateMetadata.php
new file mode 100644
index 000..6c6aa53
--- /dev/null
+++ b/maintenance/CleanCNTranslateMetadata.php
@@ -0,0 +1,132 @@
+?php
+
+if ( getenv( 'MW_INSTALL_PATH' ) !== false ) {
+ $IP = getenv( 'MW_INSTALL_PATH' );
+} else {
+ $dir = __DIR__;
+ $IP = $dir/../../..;
+}
+require_once $IP/maintenance/Maintenance.php;
+
+/**
+ * Cleans up the Revision Tag table which is where CentralNotice stores
+ * metadata required for the Translate extension.
+ *
+ * So far this class:
+ * * Removes duplicate revision entries (there should be only one per banner)
+ * * Associates entries with a banner by name
+ * * Removes entries that have no banner object
+ *
+ * Class CleanCNTranslateMetadata
+ */
+class CleanCNTranslateMetadata extends Maintenance {
+ protected $ttag;
+
+ public function execute() {
+ $this-ttag = RevTag::getType( 'banner:translate' );
+
+ $this-cleanDuplicates();
+ $this-populateIDs();
+ $this-deleteOrphans();
+ }
+
+ /**
+* Remove duplicated revtags
+*/
+ protected function cleanDuplicates() {
+ $this-output( Cleaning duplicates\n );
+
+ $db = CNDatabase::getDb( DB_MASTER );
+
+ $res = $db-select(
+ 'revtag',
+ array( 'rt_page', 'max(rt_revision) as maxrev',
'count(*) as count' ),
+ array( 'rt_type' = $this-ttag ),
+ __METHOD__,
+ array( 'GROUP BY' = 'rt_page' )
+ );
+
+ foreach ( $res as $row ) {
+ if ( (int)$row-count === 1 ) continue;
+
+ $db-delete(
+ 'revtag',
+
