Smalyshev has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/361005 )
Change subject: Make tail poller not to use backoff
......................................................................
Make tail poller not to use backoff
Should not influence anything except removing some useless messages.
Change-Id: I22318e771011112874879290b7573ae4436a4967
---
M
tools/src/main/java/org/wikidata/query/rdf/tool/change/RecentChangesPoller.java
M
tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
2 files changed, 16 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikidata/query/rdf
refs/changes/05/361005/1
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/change/RecentChangesPoller.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/change/RecentChangesPoller.java
index 1970c8d..cb67259 100644
---
a/tools/src/main/java/org/wikidata/query/rdf/tool/change/RecentChangesPoller.java
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/change/RecentChangesPoller.java
@@ -85,6 +85,11 @@
*/
private TailingChangesPoller tailPoller;
+ /**
+ * Whether to use backoff.
+ */
+ private boolean useBackoff = true;
+
@SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "TODO: move
to LocalDateTime")
public RecentChangesPoller(WikibaseRepository wikibase, Date
firstStartTime,
int batchSize, Map<Long, Boolean> seenIDs, int tailSeconds) {
@@ -103,6 +108,13 @@
this(wikibase, firstStartTime, batchSize, createSeenMap(),
tailSeconds);
}
+ /**
+ * Whether to use backoff.
+ * @param useBackoff
+ */
+ public void setBackoff(boolean useBackoff) {
+ this.useBackoff = useBackoff;
+ }
/**
* Create map of seen IDs.
* @return
@@ -149,6 +161,7 @@
log.info("Started trailing poller with gap of {} seconds",
tailSeconds);
// Create new poller starting back tailSeconds and same IDs map.
final RecentChangesPoller poller = new
RecentChangesPoller(wikibase, DateUtils.addSeconds(new Date(), -tailSeconds),
batchSize, seenIDs, -1);
+ poller.setBackoff(false);
tailPoller = new TailingChangesPoller(poller, queue, tailSeconds);
tailPoller.start();
} else {
@@ -248,7 +261,7 @@
* @throws RetryableException on fetch failure
*/
private JSONObject fetchRecentChanges(Date lastNextStartTime, Batch
lastBatch) throws RetryableException {
- if (changeIsRecent(lastNextStartTime)) {
+ if (useBackoff && changeIsRecent(lastNextStartTime)) {
return wikibase.fetchRecentChangesByTime(
DateUtils.addSeconds(lastNextStartTime, -BACKOFF_TIME),
batchSize);
@@ -321,7 +334,7 @@
nextStartTime = Math.max(nextStartTime, timestamp.getTime());
}
ImmutableList<Change> changes =
ImmutableList.copyOf(changesByTitle.values());
- if (changes.size() == 0 && result.size() >= batchSize) {
+ if (useBackoff && changes.size() == 0 && result.size() >=
batchSize) {
// We have a problem here - due to backoff, we did not fetch
any new items
// Try to advance one second, even though we risk to lose a
change
log.info("Backoff overflow, advancing one second");
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
index 70d58b0..75d9d5b 100644
---
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
@@ -469,6 +469,7 @@
*
* @param startTime the first date to poll from - usually if
* continueObject isn't null this is ignored by wikibase
+ * @param continueObject Continue object from the last request
* @param batchSize maximum number of results we want back from
wikibase
*/
public URI recentChanges(Date startTime, JSONObject continueObject,
int batchSize) {
--
To view, visit https://gerrit.wikimedia.org/r/361005
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I22318e771011112874879290b7573ae4436a4967
Gerrit-PatchSet: 1
Gerrit-Project: wikidata/query/rdf
Gerrit-Branch: master
Gerrit-Owner: Smalyshev <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
