jenkins-bot has submitted this change and it was merged.
Change subject: Gracefully handle outdated echo_unread_wikis rows
......................................................................
Gracefully handle outdated echo_unread_wikis rows
Adds $wgEchoSectionTransition and $wgEchoBundleTransition.
If either of these settings is enabled, we will disbelieve
the alert/message counts in the euw table and obtain them
using server-side cross-wiki API queries instead.
This affects both ApiEchoNotifications (for generating the cross-wiki
summary entry) and the count and timestamp computation in NotifUser.
In bundle transition mode, we trust that notifications are classified
correctly between alerts and messages, but we don't trust the
counts in the table. In section transition mode, we trust that
the sum of the alert and message counts is the correct count,
but we don't trust the alert and message counts individually.
If both modes are enabled, we mistrust anything that's mistrusted
by either mode and only trust what's trusted by both modes.
In any event, we do trust that only the wikis with rows in the
euw table have unread notifications.
Bug: T132954
Change-Id: Ibcc8ac102dac3cf06916d67427b42457fdb93db6
---
M Echo.php
M includes/NotifUser.php
M includes/api/ApiCrossWikiBase.php
M includes/api/ApiEchoNotifications.php
4 files changed, 218 insertions(+), 21 deletions(-)
Approvals:
Sbisson: Looks good to me, approved
jenkins-bot: Verified
diff --git a/Echo.php b/Echo.php
index 735ce17..9eeeaf4 100644
--- a/Echo.php
+++ b/Echo.php
@@ -151,6 +151,17 @@
// main one. Must be a key defined in $wgExternalServers
$wgEchoSharedTrackingCluster = false;
+// Enable this when you've changed the section (alert vs message) of a
notification
+// type, but haven't yet finished running backfillUnreadWikis.php. This setting
+// reduces performance but prevents glitchy and inaccurate information from
being
+// show to users while the unread_wikis table is being rebuilt.
+$wgEchoSectionTransition = false;
+
+// Enable this when you've changed the way bundled notifications are counted,
+// but haven't yet finished running backfillUnreadWikis.php. Like
$wgEchoSectionTransition,
+// this setting reduces performance but prevents glitches.
+$wgEchoBundleTransition = false;
+
// The max number of notifications allowed for a user to do a live update,
// this is also the number of max notifications allowed for a user to have
// @FIXME - the name is not intuitive, probably change it when the deleteJob
patch
diff --git a/includes/NotifUser.php b/includes/NotifUser.php
index 1755c90..546efaa 100644
--- a/includes/NotifUser.php
+++ b/includes/NotifUser.php
@@ -45,6 +45,11 @@
*/
private $cached;
+ /**
+ * @var array|null
+ */
+ private $mForeignData = null;
+
// The max notification count shown in badge
// The max number shown in bundled message, eg, <user> and 99+ others
<action>.
@@ -230,7 +235,7 @@
$count = (int)
$this->userNotifGateway->getCappedNotificationCount( $dbSource,
$eventTypesToLoad, MWEchoNotifUser::MAX_BADGE_COUNT + 1 );
if ( $global ) {
- $count += $this->getForeignNotifications()->getCount(
$section );
+ $count += $this->getForeignCount( $section );
}
$this->setInCache( $memcKey, $count, 86400 );
@@ -316,7 +321,8 @@
// Use timestamp of most recent foreign notification, if it's
more recent
if ( $global ) {
- $foreignTime =
$this->getForeignNotifications()->getTimestamp( $section );
+ $foreignTime = $this->getForeignTimestamp( $section );
+
if (
$foreignTime !== false &&
// $foreignTime < $timestamp = invert 0
@@ -478,16 +484,16 @@
if ( $wgEchoCrossWikiNotifications ) {
// For performance, compute the global counts by adding
foreign counts to the above
- $globalAlertCount = $alertCount +
$this->getForeignNotifications()->getCount( EchoAttributeManager::ALERT );
- $globalMsgCount = $msgCount +
$this->getForeignNotifications()->getCount( EchoAttributeManager::MESSAGE );
+ $globalAlertCount = $alertCount +
$this->getForeignCount( EchoAttributeManager::ALERT );
+ $globalMsgCount = $msgCount + $this->getForeignCount(
EchoAttributeManager::MESSAGE );
$globalAllCount = $globalAlertCount + $globalMsgCount;
// For performance, compute the global timestamps as
max( localTimestamp, foreignTimestamp )
- $foreignAlertUnread =
$this->getForeignNotifications()->getTimestamp( EchoAttributeManager::ALERT );
+ $foreignAlertUnread = $this->getForeignTimestamp(
EchoAttributeManager::ALERT );
$globalAlertUnread = $alertUnread !== false &&
( $foreignAlertUnread === false ||
$alertUnread->diff( $foreignAlertUnread )->invert === 1 ) ?
$alertUnread : $foreignAlertUnread;
- $foreignMsgUnread =
$this->getForeignNotifications()->getTimestamp( EchoAttributeManager::MESSAGE );
+ $foreignMsgUnread = $this->getForeignTimestamp(
EchoAttributeManager::MESSAGE );
$globalMsgUnread = $msgUnread !== false &&
( $foreignMsgUnread === false ||
$msgUnread->diff( $foreignMsgUnread )->invert === 1 ) ?
$msgUnread : $foreignMsgUnread;
@@ -670,4 +676,118 @@
}
return $this->foreignNotifications;
}
+
+ /**
+ * Get data about foreign notifications from the foreign wikis' APIs.
+ *
+ * This is used when $wgEchoSectionTransition or
$wgEchoBundleTransition is enabled,
+ * to deal with untrustworthy echo_unread_wikis entries. This method
fetches the list of
+ * wikis that have any unread notifications at all from the
echo_unread_wikis table, then
+ * queries their APIs to find the per-section counts and timestamps for
those wikis.
+ *
+ * The results of this function are cached in the NotifUser object.
+ * @return array [ (str) wiki => [ (str) section => [ 'count' => (int)
count, 'timestamp' => (str) ts ] ] ]
+ */
+ protected function getForeignData() {
+ if ( $this->mForeignData ) {
+ return $this->mForeignData;
+ }
+
+ $potentialWikis = $this->getForeignNotifications()->getWikis(
EchoAttributeManager::ALL );
+ $foreignReq = new EchoForeignWikiRequest(
+ $this->mUser,
+ array(
+ 'action' => 'query',
+ 'meta' => 'notifications',
+ 'notprop' => 'count|list',
+ 'notgroupbysection' => '1',
+ 'notunreadfirst' => '1',
+ ),
+ $potentialWikis,
+ 'notwikis'
+ );
+ $foreignResults = $foreignReq->execute();
+
+ $this->mForeignData = array();
+ foreach ( $foreignResults as $wiki => $result ) {
+ if ( !isset( $result['query']['notifications'] ) ) {
+ continue;
+ }
+ $data = $result['query']['notifications'];
+ foreach ( EchoAttributeManager::$sections as $section )
{
+ if ( isset( $data[$section]['rawcount'] ) ) {
+
$this->mForeignData[$wiki][$section]['count'] = $data[$section]['rawcount'];
+ }
+ if ( isset( $data[$section]['list'][0] ) ) {
+
$this->mForeignData[$wiki][$section]['timestamp'] =
$data[$section]['list'][0]['timestamp']['mw'];
+ }
+ }
+ }
+ return $this->mForeignData;
+ }
+
+ protected function getForeignCount( $section =
EchoAttributeManager::ALL ) {
+ global $wgEchoSectionTransition, $wgEchoBundleTransition;
+ $count = 0;
+ if (
+ // In section transition mode, we don't trust the
individual echo_unread_wikis rows
+ // but we do trust that alert+message=all. In bundle
transition mode, we don't trust
+ // that either, but we do trust that wikis with rows in
the table have unread notifications
+ // and wikis without rows in the table don't.
+ ( $wgEchoSectionTransition && $section !==
EchoAttributeManager::ALL ) ||
+ $wgEchoBundleTransition
+ ) {
+ $foreignData = $this->getForeignData();
+ foreach ( $foreignData as $data ) {
+ if ( $section === EchoAttributeManager::ALL ) {
+ foreach ( $data as $subData ) {
+ if ( isset( $subData['count'] )
) {
+ $count +=
$subData['count'];
+ }
+ }
+ } elseif ( isset( $data[$section]['count'] ) ) {
+ $count += $data[$section]['count'];
+ }
+ }
+ } else {
+ $count += $this->getForeignNotifications()->getCount(
$section );
+ }
+ return $count;
+ }
+
+ protected function getForeignTimestamp( $section =
EchoAttributeManager::ALL ) {
+ if (
+ // In section transition mode, we don't trust the
individual echo_unread_wikis rows
+ // but we do trust that alert+message=all. In bundle
transition mode, we don't trust
+ // that either, but we do trust that wikis with rows in
the table have unread notifications
+ // and wikis without rows in the table don't.
+ ( $wgEchoSectionTransition && $section !==
EchoAttributeManager::ALL ) ||
+ $wgEchoBundleTransition
+ ) {
+ $foreignTime = false;
+ $foreignData = $this->getForeignData();
+ foreach ( $foreignData as $data ) {
+ if ( $section === EchoAttributeManager::ALL ) {
+ foreach ( $data as $subData ) {
+ if ( isset(
$subData['timestamp'] ) ) {
+ $wikiTime = new
MWTimestamp( $data[$section]['timestamp'] );
+ // $wikiTime >
$foreignTime = invert 1
+ if ( $foreignTime ===
false || $wikiTime->diff( $foreignTime )->invert === 1 ) {
+ $foreignTime =
$wikiTime;
+ }
+ }
+ }
+ } elseif ( isset( $data[$section]['timestamp']
) ) {
+ $wikiTime = new MWTimestamp(
$data[$section]['timestamp'] );
+ // $wikiTime > $foreignTime = invert 1
+ if ( $foreignTime === false ||
$wikiTime->diff( $foreignTime )->invert === 1 ) {
+ $foreignTime = $wikiTime;
+ }
+ }
+ }
+ } else {
+ $foreignTime =
$this->getForeignNotifications()->getTimestamp( $section );
+ }
+ return $foreignTime;
+ }
}
diff --git a/includes/api/ApiCrossWikiBase.php
b/includes/api/ApiCrossWikiBase.php
index 84f0ef6..e5d9eeb 100644
--- a/includes/api/ApiCrossWikiBase.php
+++ b/includes/api/ApiCrossWikiBase.php
@@ -23,14 +23,16 @@
* This will turn the current API call (with all of it's params) and
execute
* it on all foreign wikis, returning an array of results per wiki.
*
+ * @param array $wikis List of wikis to query. Defaults to the result
of getRequestedForeignWikis().
+ * @param array $paramOverrides Request parameter overrides
* @return array
* @throws Exception
*/
- protected function getFromForeign() {
+ protected function getFromForeign( $wikis = null, array $paramOverrides
= array() ) {
$foreignReq = new EchoForeignWikiRequest(
$this->getUser(),
- $this->getForeignQueryParams(),
- $this->getRequestedForeignWikis(),
+ $paramOverrides + $this->getForeignQueryParams(),
+ $wikis !== null ? $wikis :
$this->getRequestedForeignWikis(),
$this->getModulePrefix() . 'wikis'
);
return $foreignReq->execute();
diff --git a/includes/api/ApiEchoNotifications.php
b/includes/api/ApiEchoNotifications.php
index 896fc68..3fc66d1 100644
--- a/includes/api/ApiEchoNotifications.php
+++ b/includes/api/ApiEchoNotifications.php
@@ -91,9 +91,12 @@
$titles, $params[$section .
'unreadfirst']
);
- if ( $this->crossWikiSummary &&
$this->foreignNotifications->getCount( $section ) > 0 ) {
+ if ( $this->crossWikiSummary ) {
// insert fake notification for
foreign notifications
- array_unshift(
$result[$section]['list'], $this->makeForeignNotification( $user,
$params['format'], $section ) );
+ $foreignNotification =
$this->makeForeignNotification( $user, $params['format'], $section );
+ if ( $foreignNotification ) {
+ array_unshift(
$result[$section]['list'], $foreignNotification );
+ }
}
}
} else {
@@ -108,8 +111,11 @@
// if exactly 1 section is specified, we
consider only that section, otherwise
// we pass ALL to consider all foreign
notifications
$section = count( $params['sections'] ) === 1 ?
reset( $params['sections'] ) : EchoAttributeManager::ALL;
- if ( $this->crossWikiSummary &&
$this->foreignNotifications->getCount( $section ) > 0 ) {
- array_unshift( $result['list'],
$this->makeForeignNotification( $user, $params['format'], $section ) );
+ if ( $this->crossWikiSummary ) {
+ $foreignNotification =
$this->makeForeignNotification( $user, $params['format'], $section );
+ if ( $foreignNotification ) {
+ array_unshift( $result['list'],
$foreignNotification );
+ }
}
}
}
@@ -283,15 +289,73 @@
return $result;
}
+ /**
+ * Build and format a "fake" notification to represent foreign
notifications.
+ * @param User $user
+ * @param string $format
+ * @param string $section
+ * @return array|false A formatted notification, or false if there are
no foreign notifications
+ */
protected function makeForeignNotification( User $user, $format,
$section = EchoAttributeManager::ALL ) {
- $wikis = $this->foreignNotifications->getWikis( $section );
- $count = $this->foreignNotifications->getCount( $section );
+ global $wgEchoSectionTransition, $wgEchoBundleTransition;
+ if (
+ ( $wgEchoSectionTransition && $section !==
EchoAttributeManager::ALL ) ||
+ $wgEchoBundleTransition
+ ) {
+ // In section transition mode we trust that
echo_unread_wikis is accurate for the total of alerts+messages,
+ // but not for each section individually (i.e. we don't
trust that notifications won't be misclassified).
+ // We get all wikis that have any notifications at all
according to the euw table,
+ // and query them to find out what's really there.
+ // In bundle transition mode, we trust that
notifications are classified correctly, but we don't
+ // trust the counts in the table.
+ $potentialWikis =
$this->foreignNotifications->getWikis( $wgEchoSectionTransition ?
EchoAttributeManager::ALL : $section );
+ if ( !$potentialWikis ) {
+ return false;
+ }
+ $foreignResults = $this->getFromForeign(
$potentialWikis, array( $this->getModulePrefix() . 'filter' => '!read' ) );
+
+ $countsByWiki = array();
+ $timestampsByWiki = array();
+ foreach ( $foreignResults as $wiki => $result ) {
+ if ( isset(
$result['query']['notifications']['list'] ) ) {
+ $notifs =
$result['query']['notifications']['list'];
+ } elseif ( isset(
$result['query']['notifications'][$section]['list'] ) ) {
+ $notifs =
$result['query']['notifications'][$section]['list'];
+ } else {
+ $notifs = false;
+ }
+ if ( $notifs ) {
+ $countsByWiki[$wiki] = count( $notifs );
+ $timestampsByWiki[$wiki] = max(
array_map( function ( $n ) {
+ return $n['timestamp']['mw'];
+ }, $notifs ) );
+ }
+ }
+
+ $wikis = array_keys( $countsByWiki );
+ $count = array_sum( $countsByWiki );
+ $maxTimestamp = new MWTimestamp( max( $timestampsByWiki
) );
+ $timestampsByWiki = array_map( function ( $ts ) {
+ return new MWTimestamp( $ts );
+ }, $timestampsByWiki );
+ } else {
+ // In non-transition mode, or when querying all
sections, we can trust the euw table
+ $wikis = $this->foreignNotifications->getWikis(
$section );
+ $count = $this->foreignNotifications->getCount(
$section );
+ $maxTimestamp =
$this->foreignNotifications->getTimestamp( $section );
+ $timestampsByWiki = array();
+ foreach ( $wikis as $wiki ) {
+ $timestampsByWiki[$wiki] =
$this->foreignNotifications->getWikiTimestamp( $wiki, $section );
+ }
+ }
+
+ if ( $count === 0 || count( $wikis ) === 0 ) {
+ return false;
+ }
// Sort wikis by timestamp, in descending order (newest first)
- usort( $wikis, function ( $a, $b ) use ( $section ) {
- $aTimestamp =
$this->foreignNotifications->getWikiTimestamp( $a, $section ) ?: new
MWTimestamp( 0 );
- $bTimestamp =
$this->foreignNotifications->getWikiTimestamp( $b, $section ) ?: new
MWTimestamp( 0 );
- return $bTimestamp->getTimestamp( TS_UNIX ) -
$aTimestamp->getTimestamp( TS_UNIX );
+ usort( $wikis, function ( $a, $b ) use ( $section,
$timestampsByWiki ) {
+ return $timestampsByWiki[$b]->getTimestamp( TS_UNIX ) -
$timestampsByWiki[$a]->getTimestamp( TS_UNIX );
} );
$row = new StdClass;
@@ -310,7 +374,7 @@
) );
$row->notification_user = $user->getId();
- $row->notification_timestamp =
$this->foreignNotifications->getTimestamp( $section );
+ $row->notification_timestamp = $maxTimestamp;
$row->notification_read_timestamp = null;
$row->notification_bundle_base = 1;
$row->notification_bundle_hash = md5( 'bogus' );
@@ -326,7 +390,7 @@
$output['sources'] = EchoForeignNotifications::getApiEndpoints(
$wikis );
// Add timestamp information
foreach ( $output['sources'] as $wiki => &$data ) {
- $data['ts'] =
$this->foreignNotifications->getWikiTimestamp( $wiki, $section )->getTimestamp(
TS_MW );
+ $data['ts'] = $timestampsByWiki[$wiki]->getTimestamp(
TS_MW );
}
return $output;
}
--
To view, visit https://gerrit.wikimedia.org/r/291676
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ibcc8ac102dac3cf06916d67427b42457fdb93db6
Gerrit-PatchSet: 12
Gerrit-Project: mediawiki/extensions/Echo
Gerrit-Branch: master
Gerrit-Owner: Catrope <roan.katt...@gmail.com>
Gerrit-Reviewer: Catrope <roan.katt...@gmail.com>
Gerrit-Reviewer: Mattflaschen <mflasc...@wikimedia.org>
Gerrit-Reviewer: Sbisson <sbis...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
