jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/399864 )
Change subject: Make WikibaseRepository type safe
......................................................................
Make WikibaseRepository type safe
Parse responses from Wikibase to Java objects to make them type safe. This
makes the client code less error prone, contains the JSON parser dependency
to the WikibaseRepository class and makes refactoring those classes easier.
This replaces json-simple with Jackson, which is already a dependency in
the project.
Still to check:
At this point, an invalid change will put the whole batch of changes in
error. This could happen if a field is invalid (non parseable date in a
date field, invalid integer in an integer field, ...). I'm not sure if
that's an error that is sufficiently frequent to need specific handling.
Change-Id: Ibd2a2e217931333dafa894dd003f3fdbf6c2157e
---
M pom.xml
M tools/pom.xml
M
tools/src/main/java/org/wikidata/query/rdf/tool/change/RecentChangesPoller.java
M tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
A tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/Continue.java
A
tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/CsrfTokenResponse.java
A tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/DeleteResponse.java
A tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/EditRequest.java
A tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/EditResponse.java
A
tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/RecentChangeResponse.java
A tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/SearchResponse.java
M
tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
A tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseResponse.java
M
tools/src/test/java/org/wikidata/query/rdf/tool/change/RecentChangesPollerUnitTest.java
M
tools/src/test/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepositoryIntegrationTest.java
A
tools/src/test/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepositoryWireIntegrationTest.java
A
tools/src/test/java/org/wikidata/query/rdf/tool/wikibase/WikibaseResponseTest.java
A
tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/recent_changes.json
A
tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/recent_changes_extra_fields.json
A
tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/response_with_complex_error.json
A
tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/response_with_string_error.json
21 files changed, 864 insertions(+), 347 deletions(-)
Approvals:
Smalyshev: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pom.xml b/pom.xml
index 77f5af9..cbfa1dc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -228,11 +228,6 @@
<version>22.0</version>
</dependency>
<dependency>
- <groupId>com.googlecode.json-simple</groupId>
- <artifactId>json-simple</artifactId>
- <version>1.1</version>
- </dependency>
- <dependency>
<groupId>com.lexicalscope.jewelcli</groupId>
<artifactId>jewelcli</artifactId>
<version>0.8.9</version>
@@ -379,6 +374,11 @@
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlets</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
@@ -553,6 +553,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>com.github.tomakehurst</groupId>
+ <artifactId>wiremock</artifactId>
+ <version>2.12.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
diff --git a/tools/pom.xml b/tools/pom.xml
index 222d3c2..f76c6b7 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -34,6 +34,14 @@
<artifactId>logback-core</artifactId>
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.github.rholder</groupId>
<artifactId>guava-retrying</artifactId>
</dependency>
@@ -44,10 +52,6 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>com.googlecode.json-simple</groupId>
- <artifactId>json-simple</artifactId>
</dependency>
<dependency>
<groupId>com.lexicalscope.jewelcli</groupId>
@@ -133,6 +137,17 @@
<scope>runtime</scope>
</dependency>
<dependency>
+ <groupId>com.github.tomakehurst</groupId>
+ <artifactId>wiremock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <!-- ensure compatible version for wiremock -->
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlets</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
<scope>test</scope>
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/change/RecentChangesPoller.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/change/RecentChangesPoller.java
index 85b6208..0c02bc1 100644
---
a/tools/src/main/java/org/wikidata/query/rdf/tool/change/RecentChangesPoller.java
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/change/RecentChangesPoller.java
@@ -1,21 +1,23 @@
package org.wikidata.query.rdf.tool.change;
+import static java.lang.Boolean.TRUE;
import static
org.wikidata.query.rdf.tool.wikibase.WikibaseRepository.inputDateFormat;
-import java.text.DateFormat;
import java.util.Collections;
import java.util.Date;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import org.apache.commons.lang3.time.DateUtils;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.query.rdf.tool.exception.RetryableException;
+import org.wikidata.query.rdf.tool.wikibase.Continue;
+import org.wikidata.query.rdf.tool.wikibase.RecentChangeResponse;
+import org.wikidata.query.rdf.tool.wikibase.RecentChangeResponse.RecentChange;
import org.wikidata.query.rdf.tool.wikibase.WikibaseRepository;
import com.google.common.collect.ImmutableList;
@@ -193,7 +195,7 @@
/**
* Continue from last request. Can be null.
*/
- private final JSONObject lastContinue;
+ private final Continue lastContinue;
/**
* Flag that states we have had changes, even though we didn't return
them.
@@ -204,7 +206,7 @@
* A batch that will next continue using the continue parameter.
*/
private Batch(ImmutableList<Change> changes, long advanced,
- String leftOff, Date nextStartTime, JSONObject lastContinue) {
+ String leftOff, Date nextStartTime, Continue lastContinue) {
super(changes, advanced, leftOff);
leftOffDate = nextStartTime;
this.lastContinue = lastContinue;
@@ -242,7 +244,7 @@
public String leftOffHuman() {
if (lastContinue != null) {
return WikibaseRepository.inputDateFormat().format(leftOffDate)
- + " (next: " + lastContinue.get("rccontinue") + ")";
+ + " (next: " + lastContinue.getRcContinue() + ")";
} else {
return
WikibaseRepository.inputDateFormat().format(leftOffDate);
}
@@ -266,7 +268,7 @@
* Get continue object.
* @return
*/
- public JSONObject getLastContinue() {
+ public Continue getLastContinue() {
return lastContinue;
}
}
@@ -290,7 +292,7 @@
* @return
* @throws RetryableException on fetch failure
*/
- private JSONObject fetchRecentChanges(Date lastNextStartTime, Batch
lastBatch) throws RetryableException {
+ private RecentChangeResponse fetchRecentChanges(Date lastNextStartTime,
Batch lastBatch) throws RetryableException {
if (useBackoff && changeIsRecent(lastNextStartTime)) {
return wikibase.fetchRecentChangesByTime(
DateUtils.addSeconds(lastNextStartTime, -BACKOFF_TIME),
@@ -309,98 +311,89 @@
*/
@SuppressWarnings({"checkstyle:npathcomplexity",
"checkstyle:cyclomaticcomplexity"})
private Batch batch(Date lastNextStartTime, Batch lastBatch) throws
RetryableException {
- try {
- JSONObject recentChanges = fetchRecentChanges(lastNextStartTime,
lastBatch);
- // Using LinkedHashMap here so that changes came out sorted by
order of arrival
- Map<String, Change> changesByTitle = new LinkedHashMap<>();
- JSONObject nextContinue = (JSONObject)
recentChanges.get("continue");
- long nextStartTime = lastNextStartTime.getTime();
- JSONArray result = (JSONArray) ((JSONObject)
recentChanges.get("query")).get("recentchanges");
- DateFormat df = inputDateFormat();
+ RecentChangeResponse recentChanges =
fetchRecentChanges(lastNextStartTime, lastBatch);
+ // Using LinkedHashMap here so that changes came out sorted by order
of arrival
+ Map<String, Change> changesByTitle = new LinkedHashMap<>();
+ Continue nextContinue = recentChanges.getContinue();
+ long nextStartTime = lastNextStartTime.getTime();
+ List<RecentChange> result =
recentChanges.getQuery().getRecentChanges();
- for (Object rco : result) {
- JSONObject rc = (JSONObject) rco;
- long namespace = (long) rc.get("ns");
- long rcid = (long)rc.get("rcid");
- Date timestamp = df.parse(rc.get("timestamp").toString());
- // Does not matter if the change matters for us or not, it
- // still advances the time since we've seen it.
- nextStartTime = Math.max(nextStartTime, timestamp.getTime());
- if (!wikibase.isEntityNamespace(namespace)) {
- log.info("Skipping change in irrelevant namespace: {}",
rc);
- continue;
- }
- if (!wikibase.isValidEntity(rc.get("title").toString())) {
- log.info("Skipping change with bogus title: {}",
rc.get("title").toString());
- continue;
- }
- if (seenIDs.containsKey(rcid)) {
- // This change was in the last batch
- log.debug("Skipping repeated change with rcid {}", rcid);
- continue;
- }
- seenIDs.put(rcid, true);
+ for (RecentChange rc : result) {
+ // Does not matter if the change matters for us or not, it
+ // still advances the time since we've seen it.
+ nextStartTime = Math.max(nextStartTime,
rc.getTimestamp().getTime());
+ if (!wikibase.isEntityNamespace(rc.getNs())) {
+ log.info("Skipping change in irrelevant namespace: {}", rc);
+ continue;
+ }
+ if (!wikibase.isValidEntity(rc.getTitle())) {
+ log.info("Skipping change with bogus title: {}",
rc.getTitle());
+ continue;
+ }
+ if (seenIDs.containsKey(rc.getRcId())) {
+ // This change was in the last batch
+ log.debug("Skipping repeated change with rcid {}",
rc.getRcId());
+ continue;
+ }
+ seenIDs.put(rc.getRcId(), TRUE);
// Looks like we can not rely on changes appearing in order, so we have to
take them all and let SPARQL
// sort out the dupes.
// if (continueChange != null && rcid < continueChange.rcid()) {
// // We've already seen this change, since it has older
rcid - so skip it
// continue;
// }
- Change change;
- if (rc.get("type").toString().equals("log") &&
(long)rc.get("revid") == 0) {
- // Deletes should always be processed, so put negative
revision
- change = new Change(rc.get("title").toString(), -1L,
timestamp, rcid);
- } else {
- change = new Change(rc.get("title").toString(), (long)
rc.get("revid"), timestamp, (long)rc.get("rcid"));
- }
- /*
- * Remove duplicate changes by title keeping the latest
- * revision. Note that negative revision means always update,
so those
- * are kept.
- */
- Change dupe = changesByTitle.put(change.entityId(), change);
- if (dupe != null && (dupe.revision() > change.revision() ||
dupe.revision() < 0)) {
- // need to remove so that order will be correct
- changesByTitle.remove(change.entityId());
- changesByTitle.put(change.entityId(), dupe);
- }
- }
- final ImmutableList<Change> changes =
ImmutableList.copyOf(changesByTitle.values());
- // Backoff overflow is when:
- // a. We use backoff
- // b. We got full batch of changes.
- // c. None of those were new changes.
- // In this case, sleeping and trying again is obviously useless.
- final boolean backoffOverflow = useBackoff && changes.size() == 0
&& result.size() >= batchSize;
- if (backoffOverflow) {
- // We have a problem here - due to backoff, we did not fetch
any new items
- // Try to advance one second, even though we risk to lose a
change - in hope
- // that trailing poller will pick them up.
- nextStartTime += 1000;
- log.info("Backoff overflow, advancing next time to {}",
inputDateFormat().format(new Date(nextStartTime)));
- }
-
- if (changes.size() != 0) {
- log.info("Got {} changes, from {} to {}", changes.size(),
- changes.get(0).toString(),
- changes.get(changes.size() - 1).toString());
+ Change change;
+ if (rc.getType().equals("log") && rc.getRevId() == 0) {
+ // Deletes should always be processed, so put negative revision
+ change = new Change(rc.getTitle(), -1L, rc.getTimestamp(),
rc.getRcId());
} else {
- log.info("Got no real changes");
+ change = new Change(rc.getTitle(), rc.getRevId(),
rc.getTimestamp(), rc.getRcId());
}
-
- // Show the user the polled time - one second because we can't
- // be sure we got the whole second
- String upTo = inputDateFormat().format(new Date(nextStartTime -
1000));
- long advanced = nextStartTime - lastNextStartTime.getTime();
- Batch batch = new Batch(changes, advanced, upTo, new
Date(nextStartTime), nextContinue);
- if (backoffOverflow && nextContinue != null) {
- // We will not sleep if continue is provided.
- log.info("Got only old changes, next is: {}",
nextContinue.toJSONString());
- batch.hasChanges(true);
+ /*
+ * Remove duplicate changes by title keeping the latest
+ * revision. Note that negative revision means always update, so
those
+ * are kept.
+ */
+ Change dupe = changesByTitle.put(change.entityId(), change);
+ if (dupe != null && (dupe.revision() > change.revision() ||
dupe.revision() < 0)) {
+ // need to remove so that order will be correct
+ changesByTitle.remove(change.entityId());
+ changesByTitle.put(change.entityId(), dupe);
}
- return batch;
- } catch (java.text.ParseException e) {
- throw new RetryableException("Parse error from api", e);
}
+ final ImmutableList<Change> changes =
ImmutableList.copyOf(changesByTitle.values());
+ // Backoff overflow is when:
+ // a. We use backoff
+ // b. We got full batch of changes.
+ // c. None of those were new changes.
+ // In this case, sleeping and trying again is obviously useless.
+ final boolean backoffOverflow = useBackoff && changes.size() == 0 &&
result.size() >= batchSize;
+ if (backoffOverflow) {
+ // We have a problem here - due to backoff, we did not fetch any
new items
+ // Try to advance one second, even though we risk to lose a change
- in hope
+ // that trailing poller will pick them up.
+ nextStartTime += 1000;
+ log.info("Backoff overflow, advancing next time to {}",
inputDateFormat().format(new Date(nextStartTime)));
+ }
+
+ if (changes.size() != 0) {
+ log.info("Got {} changes, from {} to {}", changes.size(),
+ changes.get(0).toString(),
+ changes.get(changes.size() - 1).toString());
+ } else {
+ log.info("Got no real changes");
+ }
+
+ // Show the user the polled time - one second because we can't
+ // be sure we got the whole second
+ String upTo = inputDateFormat().format(new Date(nextStartTime - 1000));
+ long advanced = nextStartTime - lastNextStartTime.getTime();
+ Batch batch = new Batch(changes, advanced, upTo, new
Date(nextStartTime), nextContinue);
+ if (backoffOverflow && nextContinue != null) {
+ // We will not sleep if continue is provided.
+ log.info("Got only old changes, next is: {}", nextContinue);
+ batch.hasChanges(true);
+ }
+ return batch;
}
}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
index b048f6d..8120f95 100644
--- a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
@@ -36,9 +36,6 @@
import org.eclipse.jetty.http.HttpStatus;
import org.eclipse.jetty.util.Fields;
import org.eclipse.jetty.util.ssl.SslContextFactory;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-import org.json.simple.parser.ParseException;
import org.openrdf.model.Literal;
import org.openrdf.model.Statement;
import org.openrdf.query.Binding;
@@ -59,6 +56,12 @@
import org.wikidata.query.rdf.tool.exception.ContainedException;
import org.wikidata.query.rdf.tool.exception.FatalException;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.rholder.retry.Attempt;
import com.github.rholder.retry.RetryException;
import com.github.rholder.retry.RetryListener;
@@ -920,11 +923,23 @@
@Override
public Boolean parse(ContentResponse entity) throws IOException {
try {
- JSONObject response = (JSONObject) new
JSONParser().parse(entity.getContentAsString());
- return (Boolean) response.get("boolean");
- } catch (ParseException e) {
+ ObjectMapper mapper = new ObjectMapper();
+
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ return mapper.readValue(entity.getContentAsString(),
Resp.class).aBoolean;
+
+ } catch (JsonParseException | JsonMappingException e) {
throw new IOException("Error parsing response", e);
}
}
+
+ public static class Resp {
+ private final Boolean aBoolean;
+
+ @JsonCreator
+ Resp(@JsonProperty("boolean") Boolean aBoolean) {
+ this.aBoolean = aBoolean;
+ }
+ }
}
+
}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/Continue.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/Continue.java
new file mode 100644
index 0000000..b97c8e1
--- /dev/null
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/Continue.java
@@ -0,0 +1,30 @@
+package org.wikidata.query.rdf.tool.wikibase;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class Continue {
+ private final String rcContinue;
+ private final String aContinue;
+
+ @JsonCreator
+ public Continue(
+ @JsonProperty("rccontinue") String rcContinue,
+ @JsonProperty("continue") String aContinue) {
+ this.rcContinue = rcContinue;
+ this.aContinue = aContinue;
+ }
+
+ public String getRcContinue() {
+ return rcContinue;
+ }
+
+ public String getContinue() {
+ return aContinue;
+ }
+
+ @Override
+ public String toString() {
+ return "Continue{rccontinue=" + rcContinue + ",continue=" + aContinue
+ "}";
+ }
+}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/CsrfTokenResponse.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/CsrfTokenResponse.java
new file mode 100644
index 0000000..dda5733
--- /dev/null
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/CsrfTokenResponse.java
@@ -0,0 +1,47 @@
+package org.wikidata.query.rdf.tool.wikibase;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class CsrfTokenResponse extends WikibaseResponse {
+
+ private final Query query;
+
+ @JsonCreator
+ public CsrfTokenResponse(
+ @JsonProperty("error") Object error,
+ @JsonProperty("query") Query query) {
+ super(error);
+ this.query = query;
+ }
+
+ public Query getQuery() {
+ return query;
+ }
+
+ public static class Query {
+ private final Tokens tokens;
+
+ @JsonCreator
+ public Query(@JsonProperty("tokens") Tokens tokens) {
+ this.tokens = tokens;
+ }
+
+ public Tokens getTokens() {
+ return tokens;
+ }
+ }
+
+ public static class Tokens {
+ private final String csrfToken;
+
+ @JsonCreator
+ public Tokens(@JsonProperty("csrftoken") String csrfToken) {
+ this.csrfToken = csrfToken;
+ }
+
+ public String getCsrfToken() {
+ return csrfToken;
+ }
+ }
+}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/DeleteResponse.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/DeleteResponse.java
new file mode 100644
index 0000000..a23e3e3
--- /dev/null
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/DeleteResponse.java
@@ -0,0 +1,12 @@
+package org.wikidata.query.rdf.tool.wikibase;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class DeleteResponse extends WikibaseResponse {
+ @JsonCreator
+ public DeleteResponse(
+ @JsonProperty("error") Object error) {
+ super(error);
+ }
+}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/EditRequest.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/EditRequest.java
new file mode 100644
index 0000000..dfe4fb6
--- /dev/null
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/EditRequest.java
@@ -0,0 +1,51 @@
+package org.wikidata.query.rdf.tool.wikibase;
+
+import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
+
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class EditRequest {
+
+ @JsonProperty @Nullable @JsonInclude(NON_NULL)
+ private final String datatype;
+ @JsonProperty
+ private final Map<String, Label> labels;
+
+ public EditRequest(String datatype, Map<String, Label> labels) {
+ this.datatype = datatype;
+ this.labels = labels;
+ }
+
+ public String getDatatype() {
+ return datatype;
+ }
+
+ public Map<String, Label> getLabels() {
+ return labels;
+ }
+
+ public static class Label {
+ @JsonProperty
+ private final String language;
+ @JsonProperty
+ private final String value;
+
+ public Label(String language, String value) {
+ this.language = language;
+ this.value = value;
+ }
+
+ public String getLanguage() {
+ return language;
+ }
+
+ public String getValue() {
+ return value;
+ }
+ }
+}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/EditResponse.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/EditResponse.java
new file mode 100644
index 0000000..6d6aa09
--- /dev/null
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/EditResponse.java
@@ -0,0 +1,34 @@
+package org.wikidata.query.rdf.tool.wikibase;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class EditResponse extends WikibaseResponse {
+
+ private final Entity entity;
+
+ @JsonCreator
+ public EditResponse(
+ @JsonProperty("error") Object error,
+ @JsonProperty("entity") Entity entity) {
+ super(error);
+ this.entity = entity;
+ }
+
+ public Entity getEntity() {
+ return entity;
+ }
+
+ public static class Entity {
+ private final String id;
+
+ @JsonCreator
+ public Entity(@JsonProperty("id") String id) {
+ this.id = id;
+ }
+
+ public String getId() {
+ return id;
+ }
+ }
+}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/RecentChangeResponse.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/RecentChangeResponse.java
new file mode 100644
index 0000000..d5f25a6
--- /dev/null
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/RecentChangeResponse.java
@@ -0,0 +1,101 @@
+package org.wikidata.query.rdf.tool.wikibase;
+
+import static com.fasterxml.jackson.annotation.JsonFormat.Shape.STRING;
+import static
org.wikidata.query.rdf.tool.wikibase.WikibaseRepository.INPUT_DATE_FORMAT;
+
+import java.util.Date;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+public class RecentChangeResponse extends WikibaseResponse {
+
+ private final Continue aContinue;
+ private final Query query;
+
+ @JsonCreator
+ public RecentChangeResponse(
+ @JsonProperty("error") Object error,
+ @JsonProperty("continue") Continue aContinue,
+ @JsonProperty("query") Query query) {
+ super(error);
+ this.aContinue = aContinue;
+ this.query = query;
+ }
+
+ public Continue getContinue() {
+ return aContinue;
+ }
+
+ public Query getQuery() {
+ return query;
+ }
+
+ public static class Query {
+ private final List<RecentChange> recentChanges;
+
+ @JsonCreator
+ public Query(@JsonProperty("recentchanges") List<RecentChange>
recentChanges) {
+ this.recentChanges = recentChanges;
+ }
+
+ public List<RecentChange> getRecentChanges() {
+ return recentChanges;
+ }
+ }
+
+ @SuppressFBWarnings(value = {"EI_EXPOSE_REP", "EI_EXPOSE_REP2"},
justification = "We'll need to migrate to JSR310 at some point")
+ public static class RecentChange {
+ private final Long ns;
+ private final String title;
+ private final Date timestamp;
+ private final Long revId;
+ private final Long rcId;
+ private final String type;
+
+ @JsonCreator
+ public RecentChange(
+ @JsonProperty("ns") Long ns,
+ @JsonProperty("title") String title,
+ @JsonProperty("timestamp") @JsonFormat(shape = STRING, pattern
= INPUT_DATE_FORMAT) Date timestamp,
+ @JsonProperty("revid") Long revId,
+ @JsonProperty("rcid") Long rcId,
+ @JsonProperty("type") String type
+ ) {
+ this.ns = ns;
+ this.title = title;
+ this.timestamp = timestamp;
+ this.revId = revId;
+ this.rcId = rcId;
+ this.type = type;
+ }
+
+ public Long getNs() {
+ return ns;
+ }
+
+ public String getTitle() {
+ return title;
+ }
+
+ public Date getTimestamp() {
+ return timestamp;
+ }
+
+ public Long getRevId() {
+ return revId;
+ }
+
+ public Long getRcId() {
+ return rcId;
+ }
+
+ public String getType() {
+ return type;
+ }
+ }
+}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/SearchResponse.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/SearchResponse.java
new file mode 100644
index 0000000..44c5cf8
--- /dev/null
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/SearchResponse.java
@@ -0,0 +1,39 @@
+package org.wikidata.query.rdf.tool.wikibase;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class SearchResponse extends WikibaseResponse {
+
+ private final List<SearchResult> search;
+
+ @JsonCreator
+ public SearchResponse(
+ @JsonProperty("error") Object error,
+ @JsonProperty("search") List<SearchResult> search) {
+ super(error);
+ this.search = search;
+ }
+
+ public List<SearchResult> getSearch() {
+ return search;
+ }
+
+
+ public static class SearchResult {
+ private final String id;
+
+ @JsonCreator
+ public SearchResult(
+ @JsonProperty("id") String id
+ ) {
+ this.id = id;
+ }
+
+ public String getId() {
+ return id;
+ }
+ }
+}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
index 58c3cfb..b7d3e16 100644
---
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
@@ -16,7 +16,6 @@
import java.util.Date;
import java.util.List;
import java.util.Locale;
-import java.util.Map;
import java.util.TimeZone;
import javax.net.ssl.SSLException;
@@ -44,10 +43,6 @@
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HttpContext;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-import org.json.simple.parser.ParseException;
import org.openrdf.model.Statement;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFHandlerException;
@@ -62,8 +57,15 @@
import org.wikidata.query.rdf.tool.exception.FatalException;
import org.wikidata.query.rdf.tool.exception.RetryableException;
import org.wikidata.query.rdf.tool.rdf.NormalizingRdfHandler;
+import org.wikidata.query.rdf.tool.wikibase.EditRequest.Label;
+import org.wikidata.query.rdf.tool.wikibase.SearchResponse.SearchResult;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableMap;
import com.google.common.primitives.Longs;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@@ -87,6 +89,11 @@
private static final int RETRY_INTERVAL = 500;
/**
+ * Standard representation of dates in Mediawiki API (ISO 8601).
+ */
+ public static final String INPUT_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'";
+
+ /**
* HTTP client for wikibase.
*/
private final CloseableHttpClient client = HttpClients.custom()
@@ -102,16 +109,30 @@
*/
private final Uris uris;
+ /**
+ * Object mapper used to deserialize JSON messages from Wikidata.
+ *
+ * Note that this mapper is configured to ignore unknown properties.
+ */
+ private final ObjectMapper mapper = new ObjectMapper();
+
public WikibaseRepository(String scheme, String host) {
uris = new Uris(scheme, host);
+ configureObjectMapper(mapper);
}
public WikibaseRepository(String scheme, String host, int port) {
uris = new Uris(scheme, host, port);
+ configureObjectMapper(mapper);
}
public WikibaseRepository(String scheme, String host, int port, long[]
entityNamespaces) {
uris = new Uris(scheme, host, port, entityNamespaces);
+ configureObjectMapper(mapper);
+ }
+
+ private void configureObjectMapper(ObjectMapper mapper) {
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES,
false);
}
/**
@@ -194,7 +215,7 @@
* @throws RetryableException thrown if there is an error communicating
with
* wikibase
*/
- public JSONObject fetchRecentChangesByTime(Date nextStartTime, int
batchSize) throws RetryableException {
+ public RecentChangeResponse fetchRecentChangesByTime(Date nextStartTime,
int batchSize) throws RetryableException {
return fetchRecentChanges(nextStartTime, null, batchSize);
}
@@ -211,16 +232,19 @@
* @throws RetryableException thrown if there is an error communicating
with
* wikibase
*/
- public JSONObject fetchRecentChanges(Date nextStartTime, JSONObject
lastContinue, int batchSize)
+ public RecentChangeResponse fetchRecentChanges(Date nextStartTime,
Continue lastContinue, int batchSize)
throws RetryableException {
URI uri = uris.recentChanges(nextStartTime, lastContinue, batchSize);
log.debug("Polling for changes from {}", uri);
try {
- return checkApi(getJson(new HttpGet(uri)));
+ return checkApi(getJson(new HttpGet(uri),
RecentChangeResponse.class));
} catch (UnknownHostException | SocketException e) {
// We want to bail on this, since it happens to be sticky for some
reason
throw new RuntimeException(e);
- } catch (IOException | ParseException e) {
+ } catch (JsonParseException | JsonMappingException e) {
+ // An invalid response will probably not fix itself with a retry,
so let's bail
+ throw new RuntimeException(e);
+ } catch (IOException e) {
throw new RetryableException("Error fetching recent changes", e);
}
}
@@ -275,14 +299,13 @@
URI uri = uris.searchForLabel(label, language, type);
log.debug("Searching for entity using {}", uri);
try {
- JSONObject result = checkApi(getJson(new HttpGet(uri)));
- JSONArray resultList = (JSONArray) result.get("search");
+ SearchResponse result = checkApi(getJson(new HttpGet(uri),
SearchResponse.class));
+ List<SearchResult> resultList = result.getSearch();
if (resultList.isEmpty()) {
return null;
}
- result = (JSONObject) resultList.get(0);
- return result.get("id").toString();
- } catch (IOException | ParseException e) {
+ return resultList.get(0).getId();
+ } catch (IOException e) {
throw new RetryableException("Error searching for page", e);
}
}
@@ -301,23 +324,20 @@
*/
@SuppressWarnings("unchecked")
public String setLabel(String entityId, String type, String label, String
language) throws RetryableException {
- JSONObject data = new JSONObject();
- JSONObject labels = new JSONObject();
- data.put("labels", labels);
- JSONObject labelObject = new JSONObject();
- labels.put("en", labelObject);
- labelObject.put("language", language);
- labelObject.put("value", label + System.currentTimeMillis());
- if (type.equals("property")) {
- // A data type is required for properties so lets just pick one
- data.put("datatype", "string");
- }
- URI uri = uris.edit(entityId, type, data.toJSONString());
- log.debug("Editing entity using {}", uri);
+ String datatype = type.equals("property") ? "string" : null;
+
+ EditRequest data = new EditRequest(
+ datatype,
+ ImmutableMap.of(
+ language,
+ new Label(language, label)));
+
try {
- JSONObject result = checkApi(getJson(postWithToken(uri)));
- return ((JSONObject) result.get("entity")).get("id").toString();
- } catch (IOException | ParseException e) {
+ URI uri = uris.edit(entityId, type,
mapper.writeValueAsString(data));
+ log.debug("Editing entity using {}", uri);
+ EditResponse result = checkApi(getJson(postWithToken(uri),
EditResponse.class));
+ return result.getEntity().getId();
+ } catch (IOException e) {
throw new RetryableException("Error adding page", e);
}
}
@@ -332,9 +352,9 @@
URI uri = uris.delete(entityId);
log.debug("Deleting entity {} using {}", entityId, uri);
try {
- JSONObject result = checkApi(getJson(postWithToken(uri)));
+ DeleteResponse result = checkApi(getJson(postWithToken(uri),
DeleteResponse.class));
log.debug("Deleted: {}", result);
- } catch (IOException | ParseException e) {
+ } catch (IOException e) {
throw new RetryableException("Error deleting page", e);
}
}
@@ -343,9 +363,8 @@
* Post with a csrf token.
*
* @throws IOException if its thrown while communicating with wikibase
- * @throws ParseException if wikibase's response can't be parsed
*/
- private HttpPost postWithToken(URI uri) throws IOException, ParseException
{
+ private HttpPost postWithToken(URI uri) throws IOException {
HttpPost request = new HttpPost(uri);
List<NameValuePair> entity = new ArrayList<>();
entity.add(new BasicNameValuePair("token", csrfToken()));
@@ -357,13 +376,11 @@
* Fetch a csrf token.
*
* @throws IOException if its thrown while communicating with wikibase
- * @throws ParseException if wikibase's response can't be parsed
*/
- private String csrfToken() throws IOException, ParseException {
+ private String csrfToken() throws IOException {
URI uri = uris.csrfToken();
log.debug("Fetching csrf token from {}", uri);
- return ((JSONObject) ((JSONObject) getJson(new
HttpGet(uri)).get("query")).get("tokens")).get("csrftoken")
- .toString();
+ return getJson(new HttpGet(uri),
CsrfTokenResponse.class).getQuery().getTokens().getCsrfToken();
}
/**
@@ -373,12 +390,11 @@
* @return json response
* @throws IOException if there is an error parsing the json or if one is
* thrown receiving the data
- * @throws ParseException the json was malformed and couldn't be parsed
*/
- private JSONObject getJson(HttpRequestBase request) throws IOException,
ParseException {
+ private <T extends WikibaseResponse> T getJson(HttpRequestBase request,
Class<T> valueType)
+ throws IOException {
try (CloseableHttpResponse response = client.execute(request)) {
- return (JSONObject) new JSONParser().parse(new
InputStreamReader(response.getEntity().getContent(),
- Charsets.UTF_8));
+ return mapper.readValue(response.getEntity().getContent(),
valueType);
}
}
@@ -390,8 +406,8 @@
* @throws RetryableException thrown if there is an error communicating
with
* wikibase
*/
- private JSONObject checkApi(JSONObject response) throws RetryableException
{
- Object error = response.get("error");
+ private <T extends WikibaseResponse> T checkApi(T response) throws
RetryableException {
+ Object error = response.getError();
if (error != null) {
throw new RetryableException("Error result from Mediawiki: " +
error);
}
@@ -471,7 +487,7 @@
* @param continueObject Continue object from the last request
* @param batchSize maximum number of results we want back from
wikibase
*/
- public URI recentChanges(Date startTime, Map continueObject, int
batchSize) {
+ public URI recentChanges(Date startTime, Continue continueObject, int
batchSize) {
URIBuilder builder = apiBuilder();
builder.addParameter("action", "query");
builder.addParameter("list", "recentchanges");
@@ -483,8 +499,8 @@
builder.addParameter("continue", "");
builder.addParameter("rcstart",
outputDateFormat().format(startTime));
} else {
- builder.addParameter("continue",
continueObject.get("continue").toString());
- builder.addParameter("rccontinue",
continueObject.get("rccontinue").toString());
+ builder.addParameter("continue", continueObject.getContinue());
+ builder.addParameter("rccontinue",
continueObject.getRcContinue());
}
return build(builder);
}
@@ -642,7 +658,7 @@
* in the format that wikibase returns.
*/
public static DateFormat inputDateFormat() {
- return utc(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'",
Locale.ROOT));
+ return utc(new SimpleDateFormat(INPUT_DATE_FORMAT, Locale.ROOT));
}
/**
@@ -659,31 +675,17 @@
}
/**
- * Create JSON change description for continuing.
- * @param lastChange
- * @return Change description that can be used to continue from the next
change.
- */
- @SuppressWarnings("unchecked")
- public JSONObject getContinueObject(Change lastChange) {
- JSONObject nextContinue = new JSONObject();
- nextContinue.put("rccontinue",
outputDateFormat().format(lastChange.timestamp()) + "|" + (lastChange.rcid() +
1));
- nextContinue.put("continue", "-||");
- return nextContinue;
- }
-
- /**
* Extract timestamp from continue JSON object.
* @param nextContinue
* @return Timestamp as date
* @throws java.text.ParseException When data is in is wrong format
*/
@SuppressFBWarnings(value = "STT_STRING_PARSING_A_FIELD", justification =
"low priority to fix")
- public Change getChangeFromContinue(Map<String, Object> nextContinue)
throws java.text.ParseException {
+ public Change getChangeFromContinue(Continue nextContinue) throws
java.text.ParseException {
if (nextContinue == null) {
return null;
}
- final String rccontinue = (String)nextContinue.get("rccontinue");
- final String[] parts = rccontinue.split("\\|");
+ final String[] parts = nextContinue.getRcContinue().split("\\|");
return new Change("DUMMY", -1, outputDateFormat().parse(parts[0]),
Long.parseLong(parts[1]));
}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseResponse.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseResponse.java
new file mode 100644
index 0000000..4071a0a
--- /dev/null
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseResponse.java
@@ -0,0 +1,33 @@
+package org.wikidata.query.rdf.tool.wikibase;
+
+import javax.annotation.Nullable;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Base class for all responses from Wikibase.
+ *
+ * All Wikibase responses can return errors in the same format. Subclasses
+ * should implement the more specific fields.
+ *
+ * Note that error message is parsed as a very generic {@link Object}. This
+ * Object will be a String in all known cases, but might be a more complex
+ * structure if the response contains some complex structure.
+ */
+public abstract class WikibaseResponse {
+
+ @Nullable private final Object error;
+
+ @JsonCreator
+ public WikibaseResponse(
+ @JsonProperty("error") Object error
+ ) {
+ this.error = error;
+ }
+
+ /** A representation of an error if it occured, <code>null</code>
otherwise. */
+ public Object getError() {
+ return error;
+ }
+}
diff --git
a/tools/src/test/java/org/wikidata/query/rdf/tool/change/RecentChangesPollerUnitTest.java
b/tools/src/test/java/org/wikidata/query/rdf/tool/change/RecentChangesPollerUnitTest.java
index 80bec94..1e6c474 100644
---
a/tools/src/test/java/org/wikidata/query/rdf/tool/change/RecentChangesPollerUnitTest.java
+++
b/tools/src/test/java/org/wikidata/query/rdf/tool/change/RecentChangesPollerUnitTest.java
@@ -1,15 +1,16 @@
package org.wikidata.query.rdf.tool.change;
+import static java.util.Collections.emptyList;
+import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.lessThan;
-import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import static
org.wikidata.query.rdf.tool.wikibase.WikibaseRepository.outputDateFormat;
import java.util.ArrayList;
@@ -19,13 +20,15 @@
import java.util.List;
import org.apache.commons.lang3.time.DateUtils;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.wikidata.query.rdf.tool.change.RecentChangesPoller.Batch;
import org.wikidata.query.rdf.tool.exception.RetryableException;
+import org.wikidata.query.rdf.tool.wikibase.RecentChangeResponse;
+import org.wikidata.query.rdf.tool.wikibase.Continue;
+import org.wikidata.query.rdf.tool.wikibase.RecentChangeResponse.Query;
+import org.wikidata.query.rdf.tool.wikibase.RecentChangeResponse.RecentChange;
import org.wikidata.query.rdf.tool.wikibase.WikibaseRepository;
public class RecentChangesPollerUnitTest {
@@ -39,9 +42,9 @@
* @param result
* @throws RetryableException
*/
- private void firstBatchReturns(Date startTime, JSONObject result) throws
RetryableException {
+ private void firstBatchReturns(Date startTime, RecentChangeResponse
result) throws RetryableException {
when(repository.fetchRecentChangesByTime(any(Date.class),
eq(batchSize))).thenCallRealMethod();
- when(repository.fetchRecentChanges(any(Date.class),
(JSONObject)eq(null), eq(batchSize))).thenReturn(result);
+ when(repository.fetchRecentChanges(any(Date.class), eq(null),
eq(batchSize))).thenReturn(result);
when(repository.isEntityNamespace(0)).thenReturn(true);
when(repository.isValidEntity(any(String.class))).thenReturn(true);
@@ -58,23 +61,17 @@
public void dedups() throws RetryableException {
Date startTime = new Date();
// Build a result from wikibase with duplicate recent changes
- JSONObject result = new JSONObject();
- JSONObject query = new JSONObject();
- result.put("query", query);
- JSONArray recentChanges = new JSONArray();
- query.put("recentchanges", recentChanges);
- String date = WikibaseRepository.inputDateFormat().format(new Date());
+ List<RecentChange> recentChanges = new ArrayList<>();
// 20 entries with 10 total Q ids
- for (int i = 0; i < 20; i++) {
- JSONObject rc = new JSONObject();
- rc.put("ns", Long.valueOf(0));
- rc.put("title", "Q" + (i / 2));
- rc.put("timestamp", date);
- rc.put("revid", Long.valueOf(i));
- rc.put("rcid", Long.valueOf(i));
- rc.put("type", "edit");
+ for (long i = 0; i < 20; i++) {
+ RecentChange rc = new RecentChange(
+ 0L, "Q" + (i / 2), new Date(), i, i, "edit");
recentChanges.add(rc);
}
+ Query query = new Query(recentChanges);
+ String error = null;
+ Continue aContinue = null;
+ RecentChangeResponse result = new RecentChangeResponse(error,
aContinue, query);
firstBatchReturns(startTime, result);
RecentChangesPoller poller = new RecentChangesPoller(repository,
startTime, batchSize);
@@ -82,12 +79,7 @@
assertThat(batch.changes(), hasSize(10));
List<Change> changes = new ArrayList<>(batch.changes());
- Collections.sort(changes, new Comparator<Change>() {
- @Override
- public int compare(Change lhs, Change rhs) {
- return lhs.entityId().compareTo(rhs.entityId());
- }
- });
+ Collections.sort(changes, Comparator.comparing(Change::entityId));
for (int i = 0; i < 10; i++) {
assertEquals(changes.get(i).entityId(), "Q" + i);
assertEquals(changes.get(i).revision(), 2 * i + 1);
@@ -106,36 +98,20 @@
Date startTime = DateUtils.addDays(new Date(), -10);
int batchSize = 10;
- JSONObject result = new JSONObject();
- JSONObject rc = new JSONObject();
- JSONArray recentChanges = new JSONArray();
- JSONObject query = new JSONObject();
-
Date revDate = DateUtils.addSeconds(startTime, 20);
+
+ String error = null;
+ Continue aContinue = new Continue(
+ outputDateFormat().format(revDate) + "|8",
+ "-||");
+ List<RecentChange> recentChanges = new ArrayList<>();
+ recentChanges.add(new RecentChange(0L, "Q666", revDate, 1L, 1L,
"edit"));
+ recentChanges.add(new RecentChange(0L, "Q667", revDate, 7L, 7L,
"edit"));
+ Query query = new Query(recentChanges);
+
+ RecentChangeResponse result = new RecentChangeResponse(error,
aContinue, query);
+
String date = WikibaseRepository.inputDateFormat().format(revDate);
- rc.put("ns", Long.valueOf(0));
- rc.put("title", "Q666");
- rc.put("timestamp", date);
- rc.put("revid", 1L);
- rc.put("rcid", 1L);
- rc.put("type", "edit");
- recentChanges.add(rc);
- rc = new JSONObject();
- rc.put("ns", Long.valueOf(0));
- rc.put("title", "Q667");
- rc.put("timestamp", date);
- rc.put("revid", 7L);
- rc.put("rcid", 7L);
- rc.put("type", "edit");
- recentChanges.add(rc);
-
- query.put("recentchanges", recentChanges);
- result.put("query", query);
-
- JSONObject contJson = new JSONObject();
- contJson.put("rccontinue", outputDateFormat().format(revDate) + "|8");
- contJson.put("continue", "-||");
- result.put("continue", contJson);
firstBatchReturns(startTime, result);
@@ -144,18 +120,18 @@
assertThat(batch.changes(), hasSize(2));
assertEquals(7, batch.changes().get(1).rcid());
assertEquals(date,
WikibaseRepository.inputDateFormat().format(batch.leftOffDate()));
- assertEquals(contJson, batch.getLastContinue());
+ assertEquals(aContinue, batch.getLastContinue());
ArgumentCaptor<Date> argumentDate =
ArgumentCaptor.forClass(Date.class);
- ArgumentCaptor<JSONObject> argumentJson =
ArgumentCaptor.forClass(JSONObject.class);
+ ArgumentCaptor<Continue> continueCaptor =
ArgumentCaptor.forClass(Continue.class);
recentChanges.clear();
- when(repository.fetchRecentChanges(argumentDate.capture(),
argumentJson.capture(), eq(batchSize))).thenReturn(result);
+ when(repository.fetchRecentChanges(argumentDate.capture(),
continueCaptor.capture(), eq(batchSize))).thenReturn(result);
// check that poller passes the continue object to the next batch
batch = poller.nextBatch(batch);
assertThat(batch.changes(), hasSize(0));
assertEquals(date,
WikibaseRepository.inputDateFormat().format(argumentDate.getValue()));
- assertEquals(contJson, argumentJson.getValue());
+ assertEquals(aContinue, continueCaptor.getValue());
}
/**
@@ -168,29 +144,14 @@
// Use old date to remove backoff
Date startTime = DateUtils.addDays(new Date(), -10);
// Build a result from wikibase with duplicate recent changes
- JSONObject result = new JSONObject();
- JSONObject query = new JSONObject();
- result.put("query", query);
- JSONArray recentChanges = new JSONArray();
- query.put("recentchanges", recentChanges);
+ String error = null;
+ Continue aContinue = null;
+ List<RecentChange> recentChanges = new ArrayList<>();
+ recentChanges.add(new RecentChange(0L, "Q424242", new Date(), 0L, 42L,
"log"));
+ recentChanges.add(new RecentChange(0L, "Q424242", new Date(), 7L, 45L,
"edit"));
+ Query query = new Query(recentChanges);
+ RecentChangeResponse result = new RecentChangeResponse(error,
aContinue, query);
String date = WikibaseRepository.inputDateFormat().format(new Date());
- JSONObject rc = new JSONObject();
- rc.put("ns", Long.valueOf(0));
- rc.put("title", "Q424242");
- rc.put("timestamp", date);
- rc.put("revid", Long.valueOf(0)); // 0 means delete
- rc.put("rcid", 42L);
- rc.put("type", "log");
- recentChanges.add(rc);
-
- rc = new JSONObject();
- rc.put("ns", Long.valueOf(0));
- rc.put("title", "Q424242");
- rc.put("timestamp", date);
- rc.put("revid", 7L);
- rc.put("rcid", 45L);
- rc.put("type", "edit");
- recentChanges.add(rc);
firstBatchReturns(startTime, result);
@@ -213,22 +174,15 @@
Date startTime = new Date();
RecentChangesPoller poller = new RecentChangesPoller(repository,
startTime, batchSize);
- JSONObject result = new JSONObject();
- JSONObject query = new JSONObject();
- result.put("query", query);
- JSONArray recentChanges = new JSONArray();
- query.put("recentchanges", recentChanges);
-
Date nextStartTime = DateUtils.addSeconds(startTime, 20);
String date =
WikibaseRepository.inputDateFormat().format(nextStartTime);
- JSONObject rc = new JSONObject();
- rc.put("ns", Long.valueOf(0));
- rc.put("title", "Q424242");
- rc.put("timestamp", date);
- rc.put("revid", 42L);
- rc.put("rcid", 42L);
- rc.put("type", "edit");
- recentChanges.add(rc);
+
+ String error = null;
+ Continue aContinue = null;
+ List<RecentChange> recentChanges = new ArrayList<>();
+ recentChanges.add(new RecentChange(0L, "Q424242", nextStartTime, 42L,
42L, "edit"));
+ Query query = new Query(recentChanges);
+ RecentChangeResponse result = new RecentChangeResponse(error,
aContinue, query);
ArgumentCaptor<Date> argument = ArgumentCaptor.forClass(Date.class);
when(repository.fetchRecentChangesByTime(argument.capture(),
eq(batchSize))).thenReturn(result);
@@ -258,14 +212,13 @@
Date startTime = DateUtils.addDays(new Date(), -1);
RecentChangesPoller poller = new RecentChangesPoller(repository,
startTime, 10);
- JSONObject result = new JSONObject();
- JSONObject query = new JSONObject();
- result.put("query", query);
- JSONArray recentChanges = new JSONArray();
- query.put("recentchanges", recentChanges);
+ String error = null;
+ Continue aContinue = null;
+ Query query = new Query(emptyList());
+ RecentChangeResponse result = new RecentChangeResponse(error,
aContinue, query);
ArgumentCaptor<Date> argument = ArgumentCaptor.forClass(Date.class);
- when(repository.fetchRecentChanges(argument.capture(),
any(JSONObject.class), eq(batchSize))).thenReturn(result);
+ when(repository.fetchRecentChanges(argument.capture(), any(),
eq(batchSize))).thenReturn(result);
when(repository.isEntityNamespace(0)).thenReturn(true);
when(repository.isValidEntity(any(String.class))).thenReturn(true);
Batch batch = poller.firstBatch();
@@ -285,21 +238,12 @@
batchSize = 1;
RecentChangesPoller poller = new RecentChangesPoller(repository,
startTime, batchSize);
- JSONObject result = new JSONObject();
- JSONObject query = new JSONObject();
- result.put("query", query);
- JSONArray recentChanges = new JSONArray();
- query.put("recentchanges", recentChanges);
-
- String date = WikibaseRepository.inputDateFormat().format(startTime);
- JSONObject rc = new JSONObject();
- rc.put("ns", Long.valueOf(0));
- rc.put("title", "Q424242");
- rc.put("timestamp", date);
- rc.put("revid", 42L);
- rc.put("rcid", 42L);
- rc.put("type", "edit");
- recentChanges.add(rc);
+ String error = null;
+ Continue aContinue = null;
+ ArrayList<RecentChange> recentChanges = new ArrayList<>();
+ recentChanges.add(new RecentChange(0L, "Q424242", startTime, 42L, 42L,
"edit"));
+ Query query = new Query(recentChanges);
+ RecentChangeResponse result = new RecentChangeResponse(error,
aContinue, query);
firstBatchReturns(startTime, result);
Batch batch = poller.firstBatch();
diff --git
a/tools/src/test/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepositoryIntegrationTest.java
b/tools/src/test/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepositoryIntegrationTest.java
index 85ff47e..973c8bf 100644
---
a/tools/src/test/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepositoryIntegrationTest.java
+++
b/tools/src/test/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepositoryIntegrationTest.java
@@ -2,27 +2,17 @@
import static org.hamcrest.Matchers.either;
import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasEntry;
-import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.isA;
-import static org.hamcrest.Matchers.not;
import static org.wikidata.query.rdf.test.CloseableRule.autoClose;
-import static
org.wikidata.query.rdf.tool.wikibase.WikibaseRepository.inputDateFormat;
import java.io.IOException;
-import java.text.DateFormat;
import java.text.ParseException;
import java.util.Collection;
import java.util.Date;
import java.util.List;
-import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.time.DateUtils;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
import org.junit.Rule;
import org.junit.Test;
import org.openrdf.model.Statement;
@@ -31,6 +21,7 @@
import org.wikidata.query.rdf.tool.change.Change;
import org.wikidata.query.rdf.tool.exception.ContainedException;
import org.wikidata.query.rdf.tool.exception.RetryableException;
+import org.wikidata.query.rdf.tool.wikibase.RecentChangeResponse.RecentChange;
import com.carrotsearch.randomizedtesting.RandomizedTest;
@@ -53,27 +44,25 @@
* is probably ok.
*/
int batchSize = randomIntBetween(3, 30);
- JSONObject changes = repo.get().fetchRecentChanges(new
Date(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30)),
+ RecentChangeResponse changes = repo.get().fetchRecentChanges(new
Date(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30)),
null, batchSize);
- Map<String, Object> c = changes;
- assertThat(c, hasKey("continue"));
- assertThat((Map<String, Object>) changes.get("continue"),
hasKey("rccontinue"));
- assertThat(c, hasKey("query"));
- Map<String, Object> query = (Map<String, Object>) c.get("query");
- assertThat(query, hasKey("recentchanges"));
- List<Object> recentChanges = (JSONArray) ((Map<String, Object>)
c.get("query")).get("recentchanges");
+ assertNotNull(changes.getContinue());
+ assertNotNull(changes.getContinue());
+ assertNotNull(changes.getQuery());
+ RecentChangeResponse.Query query = changes.getQuery();
+ assertNotNull(query.getRecentChanges());
+ List<RecentChange> recentChanges =
changes.getQuery().getRecentChanges();
assertThat(recentChanges, hasSize(batchSize));
- for (Object rco : recentChanges) {
- Map<String, Object> rc = (Map<String, Object>) rco;
- assertThat(rc, hasEntry(equalTo("ns"), either(equalTo((Object)
0L)).or(equalTo((Object) 120L))));
- assertThat(rc, hasEntry(equalTo("title"),
instanceOf(String.class)));
- assertThat(rc, hasEntry(equalTo("timestamp"),
instanceOf(String.class)));
- assertThat(rc, hasEntry(equalTo("revid"), instanceOf(Long.class)));
+ for (RecentChange rc : recentChanges) {
+ assertThat(rc.getNs(), either(equalTo(0L)).or(equalTo(120L)));
+ assertNotNull(rc.getTitle());
+ assertNotNull(rc.getTimestamp());
+ assertNotNull(rc.getRevId());
}
- final Date nextDate = repo.get().getChangeFromContinue((Map<String,
Object>)changes.get("continue")).timestamp();
+ final Date nextDate =
repo.get().getChangeFromContinue(changes.getContinue()).timestamp();
changes = repo.get().fetchRecentChanges(nextDate, null, batchSize);
- assertThat(c, hasKey("query"));
- assertThat((Map<String, Object>) c.get("query"),
hasKey("recentchanges"));
+ assertNotNull(changes.getQuery());
+ assertNotNull(changes.getQuery().getRecentChanges());
}
@Test
@@ -83,11 +72,10 @@
* This relies on there being very few changes in the current
* second.
*/
- JSONObject changes = repo.get().fetchRecentChanges(new
Date(System.currentTimeMillis()), null, 500);
- Map<String, Object> c = changes;
- assertThat(c, not(hasKey("continue")));
- assertThat(c, hasKey("query"));
- assertThat((Map<String, Object>) c.get("query"),
hasKey("recentchanges"));
+ RecentChangeResponse changes = repo.get().fetchRecentChanges(new
Date(System.currentTimeMillis()), null, 500);
+ assertNull(changes.getContinue());
+ assertNotNull(changes.getQuery());
+ assertNotNull(changes.getQuery().getRecentChanges());
}
@Test
@@ -100,7 +88,7 @@
editShowsUpInRecentChangesTestCase("QueryTestProperty", "property");
}
- private JSONArray getRecentChanges(Date date, int batchSize) throws
RetryableException,
+ private List<RecentChange> getRecentChanges(Date date, int batchSize)
throws RetryableException,
ContainedException {
// Add a bit of a wait to try and improve Jenkins test stability.
try {
@@ -108,8 +96,8 @@
} catch (InterruptedException e) {
// nothing to do here, sorry. I know it looks bad.
}
- JSONObject result = repo.get().fetchRecentChanges(date, null,
batchSize);
- return (JSONArray) ((JSONObject)
result.get("query")).get("recentchanges");
+ RecentChangeResponse result = repo.get().fetchRecentChanges(date,
null, batchSize);
+ return result.getQuery().getRecentChanges();
}
@SuppressWarnings({ "unchecked", "rawtypes" })
@@ -118,18 +106,16 @@
long now = System.currentTimeMillis();
String entityId = repo.get().firstEntityIdForLabelStartingWith(label,
"en", type);
repo.get().setLabel(entityId, type, label + now, "en");
- JSONArray changes = getRecentChanges(new Date(now - 10000), 10);
+ List<RecentChange> changes = getRecentChanges(new Date(now - 10000),
10);
boolean found = false;
String title = entityId;
if (type.equals("property")) {
title = "Property:" + title;
}
- for (Object changeObject : changes) {
- JSONObject change = (JSONObject) changeObject;
- if (change.get("title").equals(title)) {
+ for (RecentChange change : changes) {
+ if (change.getTitle().equals(title)) {
found = true;
- Map<String, Object> c = change;
- assertThat(c, hasEntry(equalTo("revid"), isA((Class)
Long.class)));
+ assertNotNull(change.getRevId());
break;
}
}
@@ -178,19 +164,16 @@
long now = System.currentTimeMillis();
String entityId =
repo.get().firstEntityIdForLabelStartingWith("QueryTestItem", "en", "item");
repo.get().setLabel(entityId, "item", "QueryTestItem" + now, "en");
- JSONArray changes = getRecentChanges(new Date(now - 10000), 10);
+ List<RecentChange> changes = getRecentChanges(new Date(now - 10000),
10);
Change change = null;
- long oldRevid = 0;
- long oldRcid = 0;
+ Long oldRevid = 0L;
+ Long oldRcid = 0L;
- for (Object changeObject : changes) {
- JSONObject rc = (JSONObject) changeObject;
- if (rc.get("title").equals(entityId)) {
- DateFormat df = inputDateFormat();
- Date timestamp = df.parse(rc.get("timestamp").toString());
- oldRevid = (long) rc.get("revid");
- oldRcid = (long)rc.get("rcid");
- change = new Change(rc.get("title").toString(), oldRevid,
timestamp, oldRcid);
+ for (RecentChange rc : changes) {
+ if (rc.getTitle().equals(entityId)) {
+ oldRevid = rc.getRevId();
+ oldRcid = rc.getRcId();
+ change = new Change(rc.getTitle(), oldRevid,
rc.getTimestamp(), oldRcid);
break;
}
}
@@ -202,11 +185,10 @@
changes = getRecentChanges(DateUtils.addSeconds(change.timestamp(),
1), 10);
// check that new result does not contain old edit but contains new
edit
boolean found = false;
- for (Object changeObject : changes) {
- JSONObject rc = (JSONObject) changeObject;
- if (rc.get("title").equals(entityId)) {
- assertNotEquals("Found old edit after continue: revid",
oldRevid, (long) rc.get("revid"));
- assertNotEquals("Found old edit after continue: rcid",
oldRcid, (long) rc.get("rcid"));
+ for (RecentChange rc: changes) {
+ if (rc.getTitle().equals(entityId)) {
+ assertNotEquals("Found old edit after continue: revid",
oldRevid, rc.getRevId());
+ assertNotEquals("Found old edit after continue: rcid",
oldRcid, rc.getRcId());
found = true;
}
}
@@ -216,11 +198,10 @@
@Test
@SuppressWarnings({ "unchecked", "rawtypes" })
public void recentChangesWithErrors() throws RetryableException,
ContainedException {
- JSONObject changes = proxyRepo.get().fetchRecentChanges(new
Date(System.currentTimeMillis()), null, 500);
- Map<String, Object> c = changes;
- assertThat(c, not(hasKey("continue")));
- assertThat(c, hasKey("query"));
- assertThat((Map<String, Object>) c.get("query"),
hasKey("recentchanges"));
+ RecentChangeResponse changes = proxyRepo.get().fetchRecentChanges(new
Date(System.currentTimeMillis()), null, 500);
+ assertNull(changes.getContinue());
+ assertNotNull(changes.getQuery());
+ assertNotNull(changes.getQuery().getRecentChanges());
}
// TODO we should verify the RDF dump format against a stored file
diff --git
a/tools/src/test/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepositoryWireIntegrationTest.java
b/tools/src/test/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepositoryWireIntegrationTest.java
new file mode 100644
index 0000000..fe31c65
--- /dev/null
+++
b/tools/src/test/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepositoryWireIntegrationTest.java
@@ -0,0 +1,83 @@
+package org.wikidata.query.rdf.tool.wikibase;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
+import static
com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
+import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.io.Resources.getResource;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasSize;
+import static
org.wikidata.query.rdf.tool.wikibase.WikibaseRepository.inputDateFormat;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.Date;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.wikidata.query.rdf.tool.exception.RetryableException;
+
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
+import com.google.common.io.Resources;
+
+public class WikibaseRepositoryWireIntegrationTest {
+
+ @Rule public WireMockRule wireMockRule = new WireMockRule(wireMockConfig()
+ .dynamicPort()
+ .dynamicHttpsPort());
+ private WikibaseRepository repository;
+
+
+ @Before
+ public void createWikibaseRepository() {
+ repository = new WikibaseRepository("http", "localhost",
wireMockRule.port());
+ }
+
+ @After
+ public void shutdownWikibaseRepository() throws IOException {
+ repository.close();
+ }
+
+ @Test
+ public void recentChangesAreParsed() throws IOException,
RetryableException, ParseException {
+ stubFor(get(anyUrl())
+
.willReturn(aResponse().withBody(load("recent_changes.json"))));
+
+ RecentChangeResponse response = repository.fetchRecentChanges(new
Date(), null, 10);
+
+ assertThat(response.getContinue().getRcContinue(),
is("20171126140446|634268213"));
+ assertThat(response.getContinue().getContinue(), is("-||"));
+ assertThat(response.getQuery().getRecentChanges(), hasSize(2));
+ RecentChangeResponse.RecentChange change =
response.getQuery().getRecentChanges().get(0);
+
+ assertThat(change.getTitle(), is("Q16013051"));
+ assertThat(change.getType(), is("edit"));
+ assertThat(change.getNs(), is(0L));
+ assertThat(change.getRevId(), is(598908952L));
+ assertThat(change.getRcId(), is(634268202L));
+ assertThat(change.getTimestamp(),
is(inputDateFormat().parse("2017-11-26T14:04:45Z")));
+ }
+
+ @Test
+ public void unknownFieldsAreIgnored() throws RetryableException,
IOException {
+ stubFor(get(anyUrl())
+
.willReturn(aResponse().withBody(load("recent_changes_extra_fields.json"))));
+
+ RecentChangeResponse response = repository.fetchRecentChanges(new
Date(), null, 10);
+
+ assertThat(response.getQuery().getRecentChanges(), hasSize(2));
+ RecentChangeResponse.RecentChange change =
response.getQuery().getRecentChanges().get(0);
+
+ assertThat(change.getTitle(), is("Q16013051"));
+ }
+
+ private String load(String name) throws IOException {
+ String prefix = this.getClass().getPackage().getName().replace(".",
"/");
+ return Resources.toString(getResource(prefix + "/" + name), UTF_8);
+ }
+}
diff --git
a/tools/src/test/java/org/wikidata/query/rdf/tool/wikibase/WikibaseResponseTest.java
b/tools/src/test/java/org/wikidata/query/rdf/tool/wikibase/WikibaseResponseTest.java
new file mode 100644
index 0000000..5581310
--- /dev/null
+++
b/tools/src/test/java/org/wikidata/query/rdf/tool/wikibase/WikibaseResponseTest.java
@@ -0,0 +1,58 @@
+package org.wikidata.query.rdf.tool.wikibase;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.junit.Test;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableMap;
+
+public class WikibaseResponseTest {
+
+ private final ObjectMapper mapper = new ObjectMapper();
+
+ @Test
+ public void stringErrorIsParsedCorrectly() throws IOException {
+ MockResponse response =
mapper.readValue(load("response_with_string_error.json"), MockResponse.class);
+ assertThat(response.getError(), is(equalTo("some error")));
+ }
+
+ @Test
+ public void complexErrorIsParsedCorrectly() throws IOException {
+ MockResponse response =
mapper.readValue(load("response_with_complex_error.json"), MockResponse.class);
+ assertThat(
+ response.getError(),
+ is(equalTo(
+ ImmutableMap.of(
+ "code", 123,
+ "reason", "some reason"))));
+ }
+
+ @Test
+ public void complexErrorIsDumpedToString() throws IOException {
+ MockResponse response =
mapper.readValue(load("response_with_complex_error.json"), MockResponse.class);
+ assertThat(response.getError().toString(), is(equalTo("{code=123,
reason=some reason}")));
+ }
+
+ private InputStream load(String name) {
+ String prefix = this.getClass().getPackage().getName().replace(".",
"/");
+ return getClass().getClassLoader().getResourceAsStream(prefix + "/" +
name);
+ }
+
+ private static final class MockResponse extends WikibaseResponse {
+ @JsonCreator
+ MockResponse(
+ @JsonProperty("error") Object error
+ ) {
+ super(error);
+ }
+ }
+
+}
diff --git
a/tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/recent_changes.json
b/tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/recent_changes.json
new file mode 100644
index 0000000..3b333fc
--- /dev/null
+++
b/tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/recent_changes.json
@@ -0,0 +1,31 @@
+{
+ "batchcomplete": "",
+ "continue": {
+ "rccontinue": "20171126140446|634268213",
+ "continue": "-||"
+ },
+ "query": {
+ "recentchanges": [
+ {
+ "type": "edit",
+ "ns": 0,
+ "title": "Q16013051",
+ "pageid": 17630512,
+ "revid": 598908952,
+ "old_revid": 582982420,
+ "rcid": 634268202,
+ "timestamp": "2017-11-26T14:04:45Z"
+ },
+ {
+ "type": "new",
+ "ns": 0,
+ "title": "Q43758077",
+ "pageid": 44966689,
+ "revid": 598908953,
+ "old_revid": 0,
+ "rcid": 634268204,
+ "timestamp": "2017-11-26T14:04:45Z"
+ }
+ ]
+ }
+}
diff --git
a/tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/recent_changes_extra_fields.json
b/tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/recent_changes_extra_fields.json
new file mode 100644
index 0000000..603ea03
--- /dev/null
+++
b/tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/recent_changes_extra_fields.json
@@ -0,0 +1,33 @@
+{
+ "batchcomplete": "",
+ "continue": {
+ "rccontinue": "20171126140446|634268213",
+ "continue": "-||"
+ },
+ "query": {
+ "recentchanges": [
+ {
+ "type": "edit",
+ "ns": 0,
+ "title": "Q16013051",
+ "pageid": 17630512,
+ "revid": 598908952,
+ "old_revid": 582982420,
+ "rcid": 634268202,
+ "timestamp": "2017-11-26T14:04:45Z",
+ "extra_field": 1234
+ },
+ {
+ "type": "new",
+ "ns": 0,
+ "title": "Q43758077",
+ "pageid": 44966689,
+ "revid": 598908953,
+ "old_revid": 0,
+ "rcid": 634268204,
+ "timestamp": "2017-11-26T14:04:45Z",
+ "extra_field": 5678
+ }
+ ]
+ }
+}
diff --git
a/tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/response_with_complex_error.json
b/tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/response_with_complex_error.json
new file mode 100644
index 0000000..c241171
--- /dev/null
+++
b/tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/response_with_complex_error.json
@@ -0,0 +1,6 @@
+{
+ "error": {
+ "code": 123,
+ "reason": "some reason"
+ }
+}
\ No newline at end of file
diff --git
a/tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/response_with_string_error.json
b/tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/response_with_string_error.json
new file mode 100644
index 0000000..6035fef
--- /dev/null
+++
b/tools/src/test/resources/org/wikidata/query/rdf/tool/wikibase/response_with_string_error.json
@@ -0,0 +1,3 @@
+{
+ "error": "some error"
+}
--
To view, visit https://gerrit.wikimedia.org/r/399864
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ibd2a2e217931333dafa894dd003f3fdbf6c2157e
Gerrit-PatchSet: 14
Gerrit-Project: wikidata/query/rdf
Gerrit-Branch: master
Gerrit-Owner: Gehel <[email protected]>
Gerrit-Reviewer: Gehel <[email protected]>
Gerrit-Reviewer: Smalyshev <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
