jenkins-bot has submitted this change and it was merged.
Change subject: T96713: add retries in HTTP calls
......................................................................
T96713: add retries in HTTP calls
Change-Id: I0c681559d0d3c9d93250dd80e2be67d1b7e318d2
---
M tools/src/main/java/org/wikidata/query/rdf/tool/rdf/NormalizingRdfHandler.java
M tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
2 files changed, 72 insertions(+), 9 deletions(-)
Approvals:
Manybubbles: Looks good to me, approved
jenkins-bot: Verified
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/NormalizingRdfHandler.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/NormalizingRdfHandler.java
index ed838ab..d6d8899 100644
---
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/NormalizingRdfHandler.java
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/NormalizingRdfHandler.java
@@ -68,7 +68,7 @@
r = new URIImpl(r.stringValue().replace("ontology-beta",
"ontology"));
}
// Temporary bugfix for dump URLs having bad characters in them
- String fixed = StringUtils.replaceEach(r.stringValue(), new
String[]{"\n", "|", "\\"}, new String[]{"", "%7C", "%5C"});
+ String fixed = StringUtils.replaceEach(r.stringValue(), new
String[]{"\n", "|", "\\", "{", "}"}, new String[]{"", "%7C", "%5C", "%7B",
"%7D"});
if (!fixed.equals(r.stringValue())) {
r = new URIImpl(fixed);
}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
index aa408d7..9c142b7 100644
--- a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
@@ -106,6 +106,16 @@
private final String updateLeftOffTimeBody;
/**
+ * How many times we retry a failed HTTP call.
+ */
+ private int maxRetries = 5;
+ /**
+ * How long to delay after failing first HTTP call, in milliseconds.
+ * Next retries would be slower by 2x, 3x, 4x etc. until maxRetries is
exhausted.
+ */
+ private int delay = 1000;
+
+ /**
* Allow subclass access to the HTTP client.
*/
protected CloseableHttpClient client() {
@@ -120,6 +130,42 @@
getValues = loadBody("GetValues");
getRefs = loadBody("GetRefs");
cleanUnused = loadBody("CleanUnused");
+ }
+
+ /**
+ * Get max retries count.
+ * @return How many times we retry a failed HTTP call.
+ */
+ public int getMaxRetries() {
+ return maxRetries;
+ }
+
+ /**
+ * Set how many times we retry a failed HTTP call.
+ * @return this
+ */
+ public RdfRepository setMaxRetries(int maxRetries) {
+ this.maxRetries = maxRetries;
+ return this;
+ }
+
+ /**
+ * Get retry delay.
+ * @return How long to delay after failing first HTTP call, in
milliseconds.
+ */
+ public int getDelay() {
+ return delay;
+ }
+
+ /**
+ * Set retry delay.
+ * Specifies how long to delay after failing first HTTP call, in
milliseconds.
+ * Next retries would be slower by 2x, 3x, 4x etc. until maxRetries is
exhausted.
+ * @return
+ */
+ public RdfRepository setDelay(int delay) {
+ this.delay = delay;
+ return this;
}
/**
@@ -331,17 +377,34 @@
List<NameValuePair> entity = new ArrayList<>();
entity.add(new BasicNameValuePair(type, sparql));
post.setEntity(new UrlEncodedFormEntity(entity, Consts.UTF_8));
- try {
- try (CloseableHttpResponse response = client.execute(post)) {
- if (response.getStatusLine().getStatusCode() != 200) {
- throw new ContainedException("Non-200 response from triple
store: " + response + " body=\n"
- + responseBodyAsString(response));
+ int retries = 0;
+ while (true) {
+ try {
+ try (CloseableHttpResponse response = client.execute(post)) {
+ if (response.getStatusLine().getStatusCode() != 200) {
+ throw new ContainedException("Non-200 response from
triple store: " + response + " body=\n"
+ + responseBodyAsString(response));
+ }
+ return responseHandler.parse(response.getEntity());
}
- return responseHandler.parse(response.getEntity());
+ } catch (IOException e) {
+ if (retries < maxRetries) {
+ // Increasing delay, with random 10% variation so threads
won't all get restarts
+ // at the same time.
+ int retryIn = (int)Math.ceil(delay * (retries + 1) * (1 +
Math.random() * 0.1));
+ log.info("HTTP request failed: {}, retrying in {} ms", e,
retryIn);
+ retries++;
+ try {
+ Thread.sleep(retryIn);
+ } catch (InterruptedException e1) {
+ throw new FatalException("Interrupted", e);
+ }
+ continue;
+ }
+ throw new FatalException("Error updating triple store", e);
}
- } catch (IOException e) {
- throw new RuntimeException("Error updating triple store", e);
}
+
}
/**
--
To view, visit https://gerrit.wikimedia.org/r/208826
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I0c681559d0d3c9d93250dd80e2be67d1b7e318d2
Gerrit-PatchSet: 3
Gerrit-Project: wikidata/query/rdf
Gerrit-Branch: master
Gerrit-Owner: Smalyshev <[email protected]>
Gerrit-Reviewer: Jdouglas <[email protected]>
Gerrit-Reviewer: Manybubbles <[email protected]>
Gerrit-Reviewer: Smalyshev <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
