jenkins-bot has submitted this change and it was merged.
Change subject: Add a simple proxy for testing
......................................................................
Add a simple proxy for testing
We boot the proxy before integration testing start and let it run during
the tests so we can make requests to it. All it does is send requests to
wikibase or general 503s. As configured right now it generates 503s every
other request.
Change-Id: Iabe712ccb6f74bb431fe7ef96ccd6536bd82381c
---
M tools/pom.xml
A tools/src/main/java/org/wikidata/query/rdf/tool/HttpClientUtils.java
M
tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
A tools/src/test/java/org/wikidata/query/rdf/tool/Proxy.java
4 files changed, 306 insertions(+), 16 deletions(-)
Approvals:
Smalyshev: Looks good to me, approved
jenkins-bot: Verified
diff --git a/tools/pom.xml b/tools/pom.xml
index 1804527..e403219 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -239,6 +239,34 @@
</executions>
</plugin>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.1.1</version>
+ <executions>
+ <execution>
+ <id>start-proxy</id>
+ <phase>pre-integration-test</phase>
+ <goals>
+ <goal>java</goal>
+ </goals>
+ <configuration>
+ <classpathScope>test</classpathScope>
+ <mainClass>org.wikidata.query.rdf.tool.Proxy</mainClass>
+ <arguments>
+ <argument>--port</argument>
+ <argument>8812</argument>
+ <argument>--error</argument>
+ <argument>503</argument>
+ <argument>--errorMod</argument>
+ <argument>2</argument>
+ <argument>--embedded</argument>
+ <argument>-v</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/HttpClientUtils.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/HttpClientUtils.java
new file mode 100644
index 0000000..805a379
--- /dev/null
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/HttpClientUtils.java
@@ -0,0 +1,22 @@
+package org.wikidata.query.rdf.tool;
+
+import org.apache.http.client.config.CookieSpecs;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpRequestBase;
+
+/**
+ * Utilities for dealing with HttpClient.
+ */
+public final class HttpClientUtils {
+ /**
+ * Configure request to ignore cookies.
+ */
+ public static void ignoreCookies(HttpRequestBase request) {
+ RequestConfig noCookiesConfig =
RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES).build();
+ request.setConfig(noCookiesConfig);
+ }
+
+ private HttpClientUtils() {
+ // Uncallable utility constructor
+ }
+}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
index 5e25b06..4b0a4b4 100644
---
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
@@ -16,8 +16,6 @@
import org.apache.http.Consts;
import org.apache.http.NameValuePair;
-import org.apache.http.client.config.CookieSpecs;
-import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
@@ -40,6 +38,7 @@
import org.openrdf.rio.helpers.StatementCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.wikidata.query.rdf.tool.HttpClientUtils;
import org.wikidata.query.rdf.tool.change.Change;
import org.wikidata.query.rdf.tool.exception.ContainedException;
import org.wikidata.query.rdf.tool.exception.FatalException;
@@ -108,7 +107,7 @@
StatementCollector collector = new StatementCollector();
parser.setRDFHandler(new NormalizingRdfHandler(collector));
HttpGet request = new HttpGet(uri);
- setNoCookies(request);
+ HttpClientUtils.ignoreCookies(request);
try {
try (CloseableHttpResponse response = client.execute(request)) {
if (response.getStatusLine().getStatusCode() == 404) {
@@ -215,16 +214,6 @@
}
/**
- * Configure request to ignore cookies.
- * @param request
- */
- private void setNoCookies(HttpRequestBase request) {
- RequestConfig noCookiesConfig = RequestConfig.custom()
- .setCookieSpec(CookieSpecs.IGNORE_COOKIES).build();
- request.setConfig(noCookiesConfig);
- }
-
- /**
* Perform an HTTP request and return the JSON in the response body.
*
* @param request request to perform
@@ -234,7 +223,7 @@
* @throws ParseException the json was malformed and couldn't be parsed
*/
private JSONObject getJson(HttpRequestBase request) throws IOException,
ParseException {
- setNoCookies(request);
+ HttpClientUtils.ignoreCookies(request);
try (CloseableHttpResponse response = client.execute(request)) {
return (JSONObject) new JSONParser().parse(new
InputStreamReader(response.getEntity().getContent(),
Charsets.UTF_8));
@@ -260,7 +249,7 @@
/**
* URIs used for accessing wikibase.
*/
- private class Uris {
+ public static class Uris {
/**
* Uri scheme for wikibase.
*/
@@ -379,7 +368,7 @@
/**
* Build a URIBuilder for wikibase requests.
*/
- private URIBuilder builder() {
+ public URIBuilder builder() {
URIBuilder builder = new URIBuilder();
builder.setHost(host);
builder.setScheme(scheme);
@@ -397,6 +386,21 @@
throw new FatalException("Unable to build url!?", e);
}
}
+
+ /**
+ * The wikibase host.
+ */
+ public String getHost() {
+ return host;
+ }
+
+ /**
+ * The uri scheme for the wikibase instance.
+ */
+ public String getScheme() {
+ return scheme;
+ }
+
}
/**
diff --git a/tools/src/test/java/org/wikidata/query/rdf/tool/Proxy.java
b/tools/src/test/java/org/wikidata/query/rdf/tool/Proxy.java
new file mode 100644
index 0000000..239922f
--- /dev/null
+++ b/tools/src/test/java/org/wikidata/query/rdf/tool/Proxy.java
@@ -0,0 +1,236 @@
+package org.wikidata.query.rdf.tool;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.wikidata.query.rdf.tool.OptionsUtils.BasicOptions;
+import org.wikidata.query.rdf.tool.OptionsUtils.WikibaseOptions;
+import org.wikidata.query.rdf.tool.wikibase.WikibaseRepository;
+
+import com.lexicalscope.jewel.cli.Option;
+
+import fi.iki.elonen.NanoHTTPD;
+import fi.iki.elonen.NanoHTTPD.Response.IStatus;
+import fi.iki.elonen.NanoHTTPD.Response.Status;
+
+/**
+ * Simply http proxy that returns preprogrammed error messages every few
+ * requests.
+ */
+public class Proxy extends NanoHTTPD {
+ private static final Logger log = LoggerFactory.getLogger(Proxy.class);
+
+ /**
+ * Options for this proxy for parsing with JewelCLI.
+ */
+ @SuppressWarnings("checkstyle:javadocmethod")
+ public interface ProxyOptions extends BasicOptions, WikibaseOptions {
+ @Option(shortName = "p", description = "Port")
+ int port();
+
+ @Option(shortName = "e", description = "Error")
+ int error();
+
+ @Option(shortName = "m", description = "Error thrown every m requests")
+ int errorMod();
+
+ @Option(defaultValue = "https", description = "Wikidata url scheme")
+ String wikibaseScheme();
+
+ @Option(defaultValue = "false", description = "Immediately return and
leave the server running in a thread")
+ boolean embedded();
+ }
+
+ /**
+ * Run from the CLI.
+ *
+ * @throws IOException if one is thrown trying to get the port
+ */
+ public static void main(String[] args) throws IOException {
+ ProxyOptions options = OptionsUtils.handleOptions(ProxyOptions.class,
args);
+ WikibaseRepository.Uris wikibase = new
WikibaseRepository.Uris(options.wikibaseScheme(), options.wikibaseHost());
+ Proxy p = new Proxy(options.port(), wikibase,
buildErrorStatus(options.error()), options.errorMod());
+ p.start();
+ if (options.embedded()) {
+ return;
+ }
+ while (true) {
+ try {
+ Thread.sleep(TimeUnit.MINUTES.toSeconds(1));
+ } catch (InterruptedException e) {
+ break;
+ }
+ }
+ p.stop();
+ }
+
+ /**
+ * Build a NanoHTTPD compatible IStatus for an error code.
+ */
+ private static IStatus buildErrorStatus(int errorCode) {
+ /*
+ * Some codes aren't supported by NanoHTTPD "natively" so we add them
+ * ourselves.
+ */
+ switch (errorCode) {
+ case 503:
+ return new SimpleStatus(503, "Internal server error");
+ default:
+ }
+ // If it is supported by NanoHTTPD use its status
+ for (Status status : Status.values()) {
+ if (status.getRequestStatus() == errorCode) {
+ return status;
+ }
+ }
+ // Otherwise throw the user an error
+ CliUtils.ForbiddenOk.systemDotErr().printf(Locale.ROOT, "Unknown error
code: %s\n", errorCode);
+ System.exit(1);
+ return null;
+ }
+
+ /**
+ * How many requests we've served?
+ */
+ private final AtomicLong requestCount = new AtomicLong(0);
+ /**
+ * Actually makes the http calls. We can afford to use the default client
in
+ * this simple test tool.
+ */
+ private final CloseableHttpClient client = HttpClients.createDefault();
+ /**
+ * The root contact point for wikibase.
+ */
+ private final WikibaseRepository.Uris wikibase;
+ /**
+ * The status to return for the error.
+ */
+ private final IStatus errorStatus;
+ /**
+ * Throw an error every this many requests.
+ */
+ private final int errorMod;
+
+ public Proxy(int port, WikibaseRepository.Uris wikibase, IStatus
errorStatus, int errorMod) {
+ super(port);
+ this.wikibase = wikibase;
+ this.errorStatus = errorStatus;
+ this.errorMod = errorMod;
+ }
+
+ @Override
+ public void start() throws IOException {
+ super.start();
+ log.info("Started proxy to {} on {}", wikibase.getHost(),
getListeningPort());
+ }
+
+ @Override
+ public Response serve(IHTTPSession session) {
+ log.debug("Serving {} {}", session.getMethod(), session.getUri());
+ long currentRequest = requestCount.incrementAndGet();
+ if (currentRequest % errorMod == 0) {
+ log.debug("Returning an {}", errorStatus);
+ return new Response(errorStatus, NanoHTTPD.MIME_PLAINTEXT, "dummy
error");
+ }
+ try {
+ URI uri = buildUri(session.getUri(), session.getParms());
+ log.debug("Proxying to {}", uri);
+ HttpRequestBase request = buildRequest(session.getMethod(), uri);
+ // TODO we totally ignore headers
+ HttpClientUtils.ignoreCookies(request);
+ CloseableHttpResponse response = client.execute(request);
+ /*
+ * Note that I'm intentionally not closing the response because the
+ * caller will close the wrapped input stream which is probably
good
+ * enough.
+ */
+ IStatus status = new SimpleStatus(response.getStatusLine());
+ String mimeType = response.getEntity().getContentType().getValue();
+ Response result = new Response(status, mimeType,
response.getEntity().getContent());
+ result.setChunkedTransfer(true);
+ return result;
+ } catch (URISyntaxException e) {
+ return new Response(NanoHTTPD.Response.Status.INTERNAL_ERROR,
NanoHTTPD.MIME_PLAINTEXT, "Invalid URI: "
+ + e.getMessage());
+ } catch (IOException e) {
+ return new Response(NanoHTTPD.Response.Status.INTERNAL_ERROR,
NanoHTTPD.MIME_PLAINTEXT,
+ "Errroc communicating with other side of proxy: " +
e.getMessage());
+ }
+ }
+
+ /**
+ * Build the URI for the proxied request.
+ *
+ * @throws URISyntaxException when the builder doesn't like the URI syntax.
+ * It is unlikely to happen but we catch it so we can send an
+ * error back to the client anyway.
+ */
+ private URI buildUri(String path, Map<String, String> params) throws
URISyntaxException {
+ URIBuilder builder = wikibase.builder().setPath(path);
+ for (Map.Entry<String, String> param : params.entrySet()) {
+ builder.addParameter(param.getKey(), param.getValue());
+ }
+ return builder.build();
+ }
+
+ /**
+ * Build the proxied request.
+ */
+ private HttpRequestBase buildRequest(Method method, URI uri) {
+ // Add extra case arms to support more types of requests
+ switch (method) {
+ case GET:
+ return new HttpGet(uri);
+ default:
+ throw new UnsupportedOperationException("Unsupported method: " +
method);
+ }
+ }
+
+ /**
+ * Wraps response from the upstream connection.
+ */
+ private static class SimpleStatus implements IStatus {
+ /**
+ * HTTP error code.
+ */
+ private final int code;
+ /**
+ * Description of the error.
+ */
+ private final String description;
+
+ public SimpleStatus(StatusLine status) {
+ this(status.getStatusCode(), status.getReasonPhrase());
+ }
+
+ public SimpleStatus(int code, String description) {
+ this.code = code;
+ this.description = description;
+ }
+
+ @Override
+ public int getRequestStatus() {
+ return code;
+ }
+
+ @Override
+ public String getDescription() {
+ // Silly API
+ return code + " " + description;
+ }
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/226020
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Iabe712ccb6f74bb431fe7ef96ccd6536bd82381c
Gerrit-PatchSet: 2
Gerrit-Project: wikidata/query/rdf
Gerrit-Branch: master
Gerrit-Owner: Manybubbles <[email protected]>
Gerrit-Reviewer: Manybubbles <[email protected]>
Gerrit-Reviewer: Smalyshev <[email protected]>
Gerrit-Reviewer: Tjones <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
