jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/367664 )
Change subject: throttling filter to protect access to sparql endpoint
......................................................................
throttling filter to protect access to sparql endpoint
This provides a way to throttle client which consume too much resources.
See the description in the ThrittlingFilter class for details.
Bug: T170860
Change-Id: If3c0c28c47f953fdb7f3b6186da8a9535cc18bdf
---
M blazegraph/pom.xml
A
blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/Bucketing.java
A
blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ClientIPFilter.java
A
blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/Throttler.java
A
blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlingFilter.java
A
blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlingState.java
A
blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/UserAgentIpAddressBucketing.java
A
blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ClientIPFilterTest.java
A
blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlerTest.java
A
blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlingStateTest.java
A
blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/UserAgentIpAddressBucketingTest.java
M pom.xml
M war/src/main/webapp/WEB-INF/web.xml
13 files changed, 1,162 insertions(+), 1 deletion(-)
Approvals:
Smalyshev: Looks good to me, approved
jenkins-bot: Verified
diff --git a/blazegraph/pom.xml b/blazegraph/pom.xml
index 86187ef..5318026 100644
--- a/blazegraph/pom.xml
+++ b/blazegraph/pom.xml
@@ -74,6 +74,10 @@
<artifactId>jetty-http</artifactId>
</dependency>
<dependency>
+ <groupId>org.isomorphism</groupId>
+ <artifactId>token-bucket</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.linkeddatafragments</groupId>
<artifactId>ldfserver</artifactId>
<classifier>classes</classifier>
@@ -127,6 +131,21 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.wikidata.query.rdf</groupId>
<artifactId>testTools</artifactId>
<scope>test</scope>
diff --git
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/Bucketing.java
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/Bucketing.java
new file mode 100644
index 0000000..ace9eeb
--- /dev/null
+++
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/Bucketing.java
@@ -0,0 +1,22 @@
+package org.wikidata.query.rdf.blazegraph.throttling;
+
+import javax.servlet.http.HttpServletRequest;
+
+/**
+ * Segmentation of requests.
+ *
+ * Resource consumption is done by <i>client</i>. This interface defines how we
+ * segment clients in different buckets.
+ *
+ * @param <T> the type of the bucket identifier
+ */
+public interface Bucketing<T> {
+ /**
+ * Compute a identifier for the bucket in which this request needs to be
+ * stored.
+ *
+ * @param request the request for which to compute the bucket
+ * @return an object identifying the bucket
+ */
+ T bucket(HttpServletRequest request);
+}
diff --git
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ClientIPFilter.java
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ClientIPFilter.java
new file mode 100644
index 0000000..ce2e67e
--- /dev/null
+++
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ClientIPFilter.java
@@ -0,0 +1,80 @@
+package org.wikidata.query.rdf.blazegraph.throttling;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ * Wrap {@link HttpServletRequest} so that it honors the "X-Real-IP" header.
+ */
+public class ClientIPFilter implements Filter {
+
+ /** Header name for X-Client-IP. */
+ private static final String X_CLIENT_IP = "X-Client-IP";
+
+ /** {@inheritDoc} */
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+ // Do nothing
+ }
+
+ /**
+ * Wrap {@link HttpServletRequest} so that it honors the "X-Real-IP"
header.
+ *
+ * @param request {@inheritDoc}
+ * @param response {@inheritDoc}
+ * @param chain {@inheritDoc}
+ * @throws IOException {@inheritDoc}
+ * @throws ServletException {@inheritDoc}
+ */
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response,
FilterChain chain)
+ throws IOException, ServletException {
+ if (request instanceof HttpServletRequest) {
+ chain.doFilter(new RealIPHttpRequestWrapper((HttpServletRequest)
request), response);
+ } else {
+ chain.doFilter(request, response);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void destroy() {
+ // Do nothing
+ }
+
+
+ /** Wrapping the request and implementing lookup of remoteAddr with
x-client-ip header. */
+ private static final class RealIPHttpRequestWrapper extends
HttpServletRequestWrapper {
+
+ /** Constructor. */
+ private RealIPHttpRequestWrapper(HttpServletRequest request) {
+ super(request);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String getRemoteAddr() {
+ String realIP = getHeader(X_CLIENT_IP);
+ return realIP != null ? realIP : super.getRemoteAddr();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String getRemoteHost() {
+ try {
+ return InetAddress.getByName(getRemoteAddr()).getHostName();
+ } catch (UnknownHostException e) {
+ return getRemoteAddr();
+ }
+ }
+ }
+}
diff --git
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/Throttler.java
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/Throttler.java
new file mode 100644
index 0000000..da237d2
--- /dev/null
+++
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/Throttler.java
@@ -0,0 +1,131 @@
+package org.wikidata.query.rdf.blazegraph.throttling;
+
+import com.google.common.cache.Cache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
+import java.time.Duration;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+
+import static java.time.temporal.ChronoUnit.MILLIS;
+
+/**
+ * Implement throttling logic.
+ *
+ * @see ThrottlingFilter for a more complete description of how throttling
+ * works.
+ * @param <B> type of the bucket used to differentiate clients
+ */
+public class Throttler<B> {
+
+ private static final Logger log = LoggerFactory.getLogger(Throttler.class);
+
+ /** How to associate a request with a specific bucket. */
+ private final Bucketing<B> bucketing;
+ /**
+ * Stores the throttling state by buckets.
+ *
+ * This is a slight abuse of Guava {@link Cache}, but makes it easy to have
+ * an LRU map with an automatic cleanup mechanism.
+ */
+ // TODO: we probably want to expose metrics on the size / usage of this
cache
+ private final Cache<B, ThrottlingState> state;
+ /** Requests longer than this will trigger tracking resource consumption.
*/
+ private final Duration requestTimeThreshold;
+ /** How to create the initial throttling state when we start tracking a
specific client. */
+ private final Callable<ThrottlingState> createThrottlingState;
+
+ /**
+ * Constructor.
+ *
+ * Note that a bucket represent our approximation of a single client.
+ *
+ * @param requestTimeThreshold requests longer than this will trigger
+ * tracking resource consumption
+ * @param bucketing how to associate a request with a specific bucket
+ * @param createThrottlingState how to create the initial throttling state
+ * when we start tracking a specific client
+ * @param stateStore the cache in which we store the per client state of
+ * throttling
+ */
+ public Throttler(
+ Duration requestTimeThreshold,
+ Bucketing<B> bucketing,
+ Callable<ThrottlingState> createThrottlingState,
+ Cache<B, ThrottlingState> stateStore) {
+ this.requestTimeThreshold = requestTimeThreshold;
+ this.bucketing = bucketing;
+ this.state = stateStore;
+ this.createThrottlingState = createThrottlingState;
+ }
+
+ /**
+ * Should this request be throttled.
+ *
+ * @param request the request to check
+ * @return true if the request should be throttled
+ */
+ public boolean isThrottled(HttpServletRequest request) {
+ ThrottlingState throttlingState =
state.getIfPresent(bucketing.bucket(request));
+ if (throttlingState == null) return false;
+
+ return throttlingState.isThrottled();
+ }
+
+ /**
+ * Notify this throttler that a request has been completed successfully.
+ *
+ * @param request the request
+ * @param elapsed how long that request took
+ */
+ public void success(HttpServletRequest request, Duration elapsed) {
+ try {
+ B bucket = bucketing.bucket(request);
+ ThrottlingState throttlingState;
+ // only start to keep track of time usage if requests are expensive
+ if (elapsed.compareTo(requestTimeThreshold) > 0) {
+ throttlingState = state.get(bucket, createThrottlingState);
+ } else {
+ throttlingState = state.getIfPresent(bucket);
+ }
+ if (throttlingState != null) {
+ throttlingState.consumeTime(elapsed);
+ }
+ } catch (ExecutionException ee) {
+ log.warn("Could not create throttling state", ee);
+ }
+ }
+
+ /**
+ * Notify this throttler that a request has completed in error.
+ *
+ * @param request the request
+ * @param elapsed how long that request took
+ */
+ public void failure(HttpServletRequest request, Duration elapsed) {
+ try {
+ ThrottlingState throttlingState =
state.get(bucketing.bucket(request), createThrottlingState);
+
+ throttlingState.consumeError();
+ throttlingState.consumeTime(elapsed);
+ } catch (ExecutionException ee) {
+ log.warn("Could not create throttling state", ee);
+ }
+ }
+
+ /**
+ * How long should this client wait before his next request.
+ *
+ * @param request the request
+ * @return 0 if no throttling, the backoff delay otherwise
+ */
+ public Duration getBackoffDelay(HttpServletRequest request) {
+ ThrottlingState throttlingState =
state.getIfPresent(bucketing.bucket(request));
+ if (throttlingState == null) return Duration.of(0, MILLIS);
+
+ return throttlingState.getBackoffDelay();
+ }
+
+}
diff --git
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlingFilter.java
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlingFilter.java
new file mode 100644
index 0000000..72a1198
--- /dev/null
+++
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlingFilter.java
@@ -0,0 +1,268 @@
+package org.wikidata.query.rdf.blazegraph.throttling;
+
+
+import com.google.common.base.Stopwatch;
+import com.google.common.cache.CacheBuilder;
+import org.isomorphism.util.TokenBuckets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import
org.wikidata.query.rdf.blazegraph.throttling.UserAgentIpAddressBucketing.Bucket;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import static java.lang.Boolean.parseBoolean;
+import static java.lang.Integer.parseInt;
+import static java.lang.String.format;
+import static java.time.temporal.ChronoUnit.SECONDS;
+import static java.util.Locale.ENGLISH;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.MINUTES;
+
+/**
+ * A Servlet Filter that applies throttling.
+ *
+ * The throttling is based on the request time consumed and the number of
+ * errors. The rational is:
+ *
+ * <dl>
+ * <dt>request time</dt>
+ * <dd>This is a good proxy for how much resource (CPU, IO) are
consumed.</dd>
+ * <dt>errors</dt>
+ * <dd>A client always in error indicates a problem client side, which
+ * should be fixed client side.</dd>
+ * </dl>
+ *
+ * Resource consumption is based on <a
+ * href="https://en.wikipedia.org/wiki/Token_bucket";>token buckets</a> as
+ * implemented by <a href="https://github.com/bbeck/token-bucket";>bbeck</a>. A
+ * token bucket is defined by:
+ *
+ * <dl>
+ * <dt>capacity</dt>
+ * <dd>the maximum number of tokes in the bucket</dd>
+ * <dt>refill amount</dt>
+ * <dd>the number of tokens to add to the bucket when refilling</dd>
+ * <dt>refill period</dt>
+ * <dd>how often to refill the bucket</dd>
+ * </dl>
+ *
+ * This filter has two buckets, one to keep track of time, and one to keep
+ * track of errors. Each time an error occurs, a token is taken out of the
+ * error bucket. Each refill period, tokens are added again. The time bucket
+ * has a similar behaviour. As an optimization, we start keeping track of
+ * resource consumption only if:
+ *
+ * <ol>
+ * <li>a request is taking a significant time</li>
+ * <li>a request is in error</li>
+ * </ol>
+ *
+ * The client is throttled if either the time bucket or the error bucket is
+ * empty. Since we don't know in advance the cost of a request or if it is
+ * going to be in error, the throttling will only occur for the next requests.
+ *
+ * In case of throttling, the client is notified by an HTTP 429 status code and
+ * is presented with a <code>Retry-After</code> HTTP header giving a backoff
+ * time in seconds.
+ *
+ * The clients are segmented in different buckets and resource consumption is
+ * tracked individually for each of those buckets. The segmentation is done by
+ * [IP address, User Agent], but could be extended to support more complex
+ * strategies. A bucket is only kept while its client is active. After a period
+ * of inactivity, the bucket is deleted.
+ *
+ * All state is limited to a single JVM, this filter is not cluster aware.
+ */
+public class ThrottlingFilter implements Filter {
+
+ private static final Logger log =
LoggerFactory.getLogger(ThrottlingFilter.class);
+
+ /** Is throttling enabled. */
+ private boolean enabled;
+ /** To delegate throttling logic. */
+ private Throttler<Bucket> throttler;
+
+ /**
+ * Initialise the filter.
+ *
+ * Configuration is loaded from TODO.
+ * @param filterConfig {@inheritDoc}
+ */
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+ // TODO: the value of the parameters are mostly invented, we need to
+ // find the correct ones.
+
+ int requestDurationThresholdInSeconds =
loadIntParam("request-duration-threshold-in-seconds", filterConfig, 10);
+ int timeBucketCapacityInSeconds =
loadIntParam("time-bucket-capacity-in-seconds", filterConfig, 60);
+ int timeBucketRefillAmountInSeconds =
loadIntParam("time-bucket-refill-amount-in-seconds", filterConfig, 60);
+ int timeBucketRefillPeriodInMinutes =
loadIntParam("time-bucket-refill-period-in-minutes", filterConfig, 1);
+ int errorBucketCapacity = loadIntParam("error-bucket-capacity",
filterConfig, 10);
+ int errorBucketRefillAmount =
loadIntParam("error-bucket-refill-amount", filterConfig, 100);
+ int errorBucketRefillPeriodInMinutes =
loadIntParam("error-bucket-refill-period-in-minutes", filterConfig, 1);
+ int maxStateSize = loadIntParam("max-state-size", filterConfig, 10000);
+ int stateExpirationInMinutes =
loadIntParam("state-expiration-in-minutes", filterConfig, 15);
+
+ this.enabled = loadBooleanParam("enabled", filterConfig, true);
+ throttler = new Throttler<>(
+ Duration.of(requestDurationThresholdInSeconds, SECONDS),
+ new UserAgentIpAddressBucketing(),
+ createThrottlingState(
+ timeBucketCapacityInSeconds,
+ timeBucketRefillAmountInSeconds,
+ timeBucketRefillPeriodInMinutes,
+ errorBucketCapacity,
+ errorBucketRefillAmount,
+ errorBucketRefillPeriodInMinutes),
+ CacheBuilder.newBuilder()
+ .maximumSize(maxStateSize)
+ .expireAfterAccess(stateExpirationInMinutes,
TimeUnit.MINUTES)
+ .build());
+ }
+
+ /**
+ * See {@link ThrottlingFilter#loadStringParam(String, FilterConfig)}.
+ *
+ * @param name
+ * @param filterConfig
+ * @param defaultValue
+ * @return
+ */
+ private int loadIntParam(String name, FilterConfig filterConfig, int
defaultValue) {
+ String result = loadStringParam(name, filterConfig);
+ return result != null ? parseInt(result) : defaultValue;
+ }
+
+ /**
+ * See {@link ThrottlingFilter#loadStringParam(String, FilterConfig)}.
+ *
+ * @param name
+ * @param filterConfig
+ * @param defaultValue
+ * @return
+ */
+ private boolean loadBooleanParam(String name, FilterConfig filterConfig,
boolean defaultValue) {
+ String result = loadStringParam(name, filterConfig);
+ return result != null ? parseBoolean(result) : defaultValue;
+ }
+
+ /**
+ * Load a parameter from multiple locations.
+ *
+ * System properties have the highest priority, filter config is used if no
+ * system property is found.
+ *
+ * The system property used is
<code>wdqs.<filter-name>.<name></code>.
+ *
+ * @param name name of the property
+ * @param filterConfig used to get the filter config
+ * @return the value of the parameter
+ */
+ private String loadStringParam(String name, FilterConfig filterConfig) {
+ String result = null;
+ String fParam = filterConfig.getInitParameter(name);
+ if (fParam != null) {
+ result = fParam;
+ }
+ String sParam = System.getProperty("wdqs." +
filterConfig.getFilterName() + "." + name);
+ if (sParam != null) {
+ result = sParam;
+ }
+ return result;
+ }
+
+ /**
+ * Create Callable to initialize throttling state.
+ *
+ * @param timeBucketCapacityInSeconds
+ * @param timeBucketRefillAmountInSeconds
+ * @param timeBucketRefillPeriodInMinutes
+ * @param errorBucketCapacity
+ * @param errorBucketRefillAmount
+ * @param errorBucketRefillPeriodInMinutes
+ */
+ public static Callable<ThrottlingState> createThrottlingState(
+ int timeBucketCapacityInSeconds,
+ int timeBucketRefillAmountInSeconds,
+ int timeBucketRefillPeriodInMinutes,
+ int errorBucketCapacity,
+ int errorBucketRefillAmount,
+ int errorBucketRefillPeriodInMinutes) {
+ return () -> new ThrottlingState(
+ TokenBuckets.builder()
+ .withCapacity(
+
MILLISECONDS.convert(timeBucketCapacityInSeconds, TimeUnit.SECONDS))
+ .withFixedIntervalRefillStrategy(
+
MILLISECONDS.convert(timeBucketRefillAmountInSeconds, TimeUnit.SECONDS),
+ timeBucketRefillPeriodInMinutes, MINUTES)
+ .build(),
+ TokenBuckets.builder()
+ .withCapacity(errorBucketCapacity)
+ .withFixedIntervalRefillStrategy(
+ errorBucketRefillAmount,
+ errorBucketRefillPeriodInMinutes, MINUTES)
+ .build());
+ }
+
+ /**
+ * Check resource consumption and throttle requests as needed.
+ *
+ * @param request {@inheritDoc}
+ * @param response {@inheritDoc}
+ * @param chain {@inheritDoc}
+ * @throws IOException {@inheritDoc}
+ * @throws ServletException {@inheritDoc}
+ */
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response,
FilterChain chain) throws IOException, ServletException {
+ HttpServletRequest httpRequest = (HttpServletRequest) request;
+ HttpServletResponse httpResponse = (HttpServletResponse) response;
+
+ if (throttler.isThrottled(httpRequest)) {
+ log.info("A request is being throttled.");
+ if (enabled) {
+ notifyUser(httpResponse,
throttler.getBackoffDelay(httpRequest));
+ return;
+ }
+ }
+
+ Stopwatch stopwatch = Stopwatch.createStarted();
+ try {
+ chain.doFilter(request, response);
+ throttler.success(httpRequest, stopwatch.elapsed());
+ } catch (IOException | ServletException e) {
+ throttler.failure(httpRequest, stopwatch.elapsed());
+ throw e;
+ }
+ }
+
+ /**
+ * Notify the user that he is being throttled.
+ *
+ * @param response the response
+ * @param backoffDelay the backoff delay
+ * @throws IOException if the response cannot be written
+ */
+ private void notifyUser(HttpServletResponse response, Duration
backoffDelay) throws IOException {
+ String retryAfter = Long.toString(backoffDelay.getSeconds());
+ response.setHeader("Retry-After", retryAfter);
+ response.sendError(429, format(ENGLISH, "Too Many Requests - Please
retry in %s seconds.", retryAfter));
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void destroy() {
+ // Nothing to destroy
+ }
+}
diff --git
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlingState.java
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlingState.java
new file mode 100644
index 0000000..58e552e
--- /dev/null
+++
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlingState.java
@@ -0,0 +1,90 @@
+package org.wikidata.query.rdf.blazegraph.throttling;
+
+import org.isomorphism.util.TokenBucket;
+
+import java.time.Duration;
+
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static java.time.temporal.ChronoUnit.MILLIS;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+/**
+ * Keeps track of resource useage for a client.
+ *
+ * This class tracks of the request time consumed and of the errors.
+ */
+public final class ThrottlingState {
+
+ /** Bucket to track requests time. */
+ private final TokenBucket timeBucket;
+ /** Bucket to track errors. */
+ private final TokenBucket errorsBucket;
+
+ /**
+ * Constructor.
+ *
+ * @param timeBucket bucket to track requests time
+ * @param errorsBucket bucket to track errors
+ */
+ public ThrottlingState(TokenBucket timeBucket, TokenBucket errorsBucket) {
+ this.timeBucket = timeBucket;
+ this.errorsBucket = errorsBucket;
+ }
+
+ /**
+ * Should this client be throttled.
+ */
+ public synchronized boolean isThrottled() {
+ return timeBucket.getNumTokens() == 0 || errorsBucket.getNumTokens()
== 0;
+ }
+
+ /**
+ * How long should a client wait before sending the next request.
+ *
+ * @return 0 if the client is not throttled, or the backoff delay
otherwise.
+ */
+ public synchronized Duration getBackoffDelay() {
+ return Duration.of(
+ max(backoffDelayMillis(timeBucket),
backoffDelayMillis(errorsBucket)),
+ MILLIS
+ );
+ }
+
+ /**
+ * Consumes request time from the time bucket.
+ *
+ * If the time bucket contains less tokens than what the request consumed,
+ * all available tokens will be consumed:
+ * <code>min(elapsed.toMillis(), timeBucket.getNumTokens())</code>.
+ *
+ * @param elapsed time elapsed during the request
+ */
+ public synchronized void consumeTime(Duration elapsed) {
+ long tokenToConsume = min(elapsed.toMillis(),
timeBucket.getNumTokens());
+ // consuming zero tokens is not allowed (and would be a NOOP anyway)
+ if (tokenToConsume > 0) {
+ timeBucket.consume(tokenToConsume);
+ }
+ }
+
+ /**
+ * Consumes errors from the error bucket.
+ *
+ * If the error bucket is already empty, does nothing.
+ */
+ public synchronized void consumeError() {
+ errorsBucket.tryConsume();
+ }
+
+ /**
+ * How long to wait before next refill for the specified bucket.
+ * @param bucket the bucket to check
+ * @return 0 if the bucket is not empty, the delay otherwise
+ */
+ private static long backoffDelayMillis(TokenBucket bucket) {
+ if (bucket.getNumTokens() > 0) return 0;
+ return bucket.getDurationUntilNextRefill(MILLISECONDS);
+ }
+
+}
diff --git
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/UserAgentIpAddressBucketing.java
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/UserAgentIpAddressBucketing.java
new file mode 100644
index 0000000..2dad56a
--- /dev/null
+++
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/throttling/UserAgentIpAddressBucketing.java
@@ -0,0 +1,58 @@
+package org.wikidata.query.rdf.blazegraph.throttling;
+
+import javax.servlet.http.HttpServletRequest;
+import java.util.Objects;
+
+/**
+ * Segmentation of requests by user-agent and IP address.
+ */
+public class UserAgentIpAddressBucketing implements
Bucketing<UserAgentIpAddressBucketing.Bucket> {
+
+ /** {@inheritDoc} */
+ @Override
+ public Bucket bucket(HttpServletRequest request) {
+ return new Bucket(request.getRemoteAddr(),
request.getHeader("User-Agent"));
+ }
+
+ /**
+ * A bucket based on user-agent and IP address.
+ *
+ * This is a simple class to wrap the user agent and IP address to avoid
+ * doing string concatenation.
+ */
+ public static final class Bucket {
+ /** IP address. */
+ private final String remoteAddr;
+ /** User-agent. */
+ private final String userAgent;
+
+ /**
+ * Constructor.
+ *
+ * @param remoteAddr IP address
+ * @param userAgent user-agent
+ */
+ private Bucket(String remoteAddr, String userAgent) {
+ this.remoteAddr = remoteAddr;
+ this.userAgent = userAgent;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ Bucket bucket = (Bucket) o;
+
+ return Objects.equals(remoteAddr, bucket.remoteAddr)
+ && Objects.equals(userAgent, bucket.userAgent);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int hashCode() {
+ return Objects.hash(remoteAddr, userAgent);
+ }
+ }
+}
diff --git
a/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ClientIPFilterTest.java
b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ClientIPFilterTest.java
new file mode 100644
index 0000000..7001ac8
--- /dev/null
+++
b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ClientIPFilterTest.java
@@ -0,0 +1,83 @@
+package org.wikidata.query.rdf.blazegraph.throttling;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.springframework.mock.web.MockHttpServletRequest;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletResponse;
+
+import java.io.IOException;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.sameInstance;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ClientIPFilterTest {
+
+ private Filter filter = new ClientIPFilter();
+ @Mock private HttpServletResponse response;
+ @Mock private FilterChain chain;
+ @Captor private ArgumentCaptor<ServletRequest> filteredRequest;
+
+ @Test
+ public void remoteAddrIsReturnedWhenNoRealIpHeaderIsPresent() throws
Exception {
+ MockHttpServletRequest request = new MockHttpServletRequest();
+ request.setRemoteAddr("1.2.3.4");
+
+ filter.doFilter(request, response, chain);
+
+ verify(chain).doFilter(filteredRequest.capture(),
any(ServletResponse.class));
+ assertThat(filteredRequest.getValue().getRemoteAddr(),
equalTo("1.2.3.4"));
+ }
+
+ @Test
+ public void realIpIsReturnedWhenHeaderIsPresent() throws Exception {
+ MockHttpServletRequest request = new MockHttpServletRequest();
+ request.setRemoteAddr("1.2.3.4");
+ request.addHeader("X-Client-IP", "4.3.2.1");
+
+ filter.doFilter(request, response, chain);
+
+ verify(chain).doFilter(filteredRequest.capture(),
any(ServletResponse.class));
+ assertThat(filteredRequest.getValue().getRemoteAddr(),
equalTo("4.3.2.1"));
+ }
+
+ @Test
+ public void caseOfHeaderIsIgnored() throws Exception {
+ MockHttpServletRequest request = new MockHttpServletRequest();
+ request.setRemoteAddr("1.2.3.4");
+ request.addHeader("x-client-ip", "4.3.2.1");
+
+ filter.doFilter(request, response, chain);
+
+ verify(chain).doFilter(filteredRequest.capture(),
any(ServletResponse.class));
+ assertThat(filteredRequest.getValue().getRemoteAddr(),
equalTo("4.3.2.1"));
+ }
+
+ @Test
+ public void dontWrapNonHttpRequests() throws IOException, ServletException
{
+ ServletRequest request = mock(ServletRequest.class);
+ ServletResponse response = mock(ServletResponse.class);
+ ArgumentCaptor<ServletRequest> filteredRequest =
ArgumentCaptor.forClass(ServletRequest.class);
+
+ filter.doFilter(request, response, chain);
+
+ verify(chain).doFilter(filteredRequest.capture(),
any(ServletResponse.class));
+
+ assertThat(filteredRequest.getValue(), sameInstance(request));
+ }
+
+}
diff --git
a/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlerTest.java
b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlerTest.java
new file mode 100644
index 0000000..53b0a5a
--- /dev/null
+++
b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlerTest.java
@@ -0,0 +1,168 @@
+package org.wikidata.query.rdf.blazegraph.throttling;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import org.isomorphism.util.TokenBuckets;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.mock.web.MockHttpServletRequest;
+import
org.wikidata.query.rdf.blazegraph.throttling.UserAgentIpAddressBucketing.Bucket;
+
+import java.time.Duration;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import static java.time.temporal.ChronoUnit.SECONDS;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThan;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class ThrottlerTest {
+
+ private Cache<Bucket, ThrottlingState> stateStore;
+ private Throttler throttler;
+
+ @Before
+ public void createThrottlerUnderTest() {
+ stateStore = CacheBuilder.newBuilder()
+ .maximumSize(10000)
+ .expireAfterAccess(15, TimeUnit.MINUTES)
+ .build();
+ throttler = new Throttler<>(
+ Duration.of(20, SECONDS),
+ new UserAgentIpAddressBucketing(),
+ createThrottlingState(),
+ stateStore);
+ }
+
+ private static Callable<ThrottlingState> createThrottlingState() {
+ return () -> new ThrottlingState(
+ TokenBuckets.builder()
+ .withCapacity(MILLISECONDS.convert(40,
TimeUnit.SECONDS))
+ .withFixedIntervalRefillStrategy(1000000, 1, MINUTES)
+ .build(),
+ TokenBuckets.builder()
+ .withCapacity(10)
+ .withFixedIntervalRefillStrategy(100, 1, MINUTES)
+ .build());
+ }
+
+ @Test
+ public void newClientIsNotThrottled() {
+ MockHttpServletRequest request = createRequest("UA1", "1.2.3.4");
+
+ assertFalse(throttler.isThrottled(request));
+ }
+
+ @Test
+ public void shortRequestsDoNotCreateState() {
+ MockHttpServletRequest request = createRequest("UA1", "1.2.3.4");
+
+ throttler.success(request, Duration.of(10, SECONDS));
+
+ assertThat(stateStore.size(), equalTo(0L));
+ }
+
+ @Test
+ public void backoffDelayIsZeroForNewClient() {
+ MockHttpServletRequest request = createRequest("UA1", "1.2.3.4");
+
+ assertThat(throttler.getBackoffDelay(request), equalTo(Duration.of(0,
SECONDS)));
+ }
+
+ @Test
+ public void
requestOverThresholdButBelowThrottlingRateEnablesUserTracking() {
+ MockHttpServletRequest request = createRequest("UA1", "1.2.3.4");
+
+ throttler.success(request, Duration.of(30, SECONDS));
+
+ assertThat(stateStore.size(), equalTo(1L));
+ assertFalse(throttler.isThrottled(request));
+ }
+
+ @Test
+ public void requestOverThrottlingRateWillThrottleNextRequest() {
+ MockHttpServletRequest request = createRequest("UA1", "1.2.3.4");
+
+ throttler.success(request, Duration.of(60, SECONDS));
+
+ assertTrue(throttler.isThrottled(request));
+ }
+
+ @Test
+ public void backoffDelayIsGivenForTimeThrottledClient() {
+ MockHttpServletRequest request = createRequest("UA1", "1.2.3.4");
+
+ throttler.success(request, Duration.of(60, SECONDS));
+
+ Duration backoffDelay = throttler.getBackoffDelay(request);
+ assertThat(backoffDelay.compareTo(Duration.of(0, SECONDS)),
greaterThan(0));
+ assertThat(backoffDelay.compareTo(Duration.of(60, SECONDS)),
lessThan(0));
+ }
+
+ @Test
+ public void
onceTrackingIsEnabledEvenShortRequestsAreTrackedAndEnableThrottling() {
+ MockHttpServletRequest request = createRequest("UA1", "1.2.3.4");
+
+ throttler.success(request, Duration.of(30, SECONDS));
+
+ assertThat(stateStore.size(), equalTo(1L));
+ assertFalse(throttler.isThrottled(request));
+
+ for (int i = 0; i < 200; i++) {
+ throttler.success(request, Duration.of(1, SECONDS));
+ }
+
+ assertThat(stateStore.size(), equalTo(1L));
+ assertTrue(throttler.isThrottled(request));
+ }
+
+ @Test
+ public void errorEnablesTrackingOfRequests() {
+ MockHttpServletRequest request = createRequest("UA1", "1.2.3.4");
+
+ throttler.failure(request, Duration.of(10, SECONDS));
+
+ assertThat(stateStore.size(), equalTo(1L));
+ assertFalse(throttler.isThrottled(request));
+ }
+
+ @Test
+ public void multipleErrorsEnablesThrottling() {
+ MockHttpServletRequest request = createRequest("UA1", "1.2.3.4");
+
+ for (int i = 0; i < 100; i++) {
+ throttler.failure(request, Duration.of(10, SECONDS));
+ }
+
+ assertThat(stateStore.size(), equalTo(1L));
+ assertTrue(throttler.isThrottled(request));
+ }
+
+ @Test
+ public void backoffDelayIsGivenForErrorThrottledClient() {
+ MockHttpServletRequest request = createRequest("UA1", "1.2.3.4");
+
+ for (int i = 0; i < 100; i++) {
+ throttler.failure(request, Duration.of(10, SECONDS));
+ }
+
+ Duration backoffDelay = throttler.getBackoffDelay(request);
+ assertThat(backoffDelay.compareTo(Duration.of(0, SECONDS)),
greaterThan(0));
+ assertThat(backoffDelay.compareTo(Duration.of(60, SECONDS)),
lessThan(0));
+ }
+
+
+ private MockHttpServletRequest createRequest(String userAgent, String
remoteAddr) {
+ MockHttpServletRequest request = new MockHttpServletRequest();
+ request.addHeader("User-Agent", userAgent);
+ request.setRemoteAddr(remoteAddr);
+ return request;
+ }
+
+}
diff --git
a/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlingStateTest.java
b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlingStateTest.java
new file mode 100644
index 0000000..32af484
--- /dev/null
+++
b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/ThrottlingStateTest.java
@@ -0,0 +1,121 @@
+package org.wikidata.query.rdf.blazegraph.throttling;
+
+import org.isomorphism.util.TokenBucket;
+import org.isomorphism.util.TokenBuckets;
+import org.junit.Test;
+
+import java.time.Duration;
+
+import static java.time.temporal.ChronoUnit.MILLIS;
+import static java.time.temporal.ChronoUnit.SECONDS;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ThrottlingStateTest {
+
+ @Test
+ public void fullBucketsAreNotThrottled() {
+ assertTrue(!new ThrottlingState(fullBucket(),
fullBucket()).isThrottled());
+ }
+
+ @Test
+ public void anyEmptyBucketIsThrottled() {
+ assertTrue(new ThrottlingState(fullBucket(),
emptyBucket()).isThrottled());
+ assertTrue(new ThrottlingState(emptyBucket(),
fullBucket()).isThrottled());
+ assertTrue(new ThrottlingState(emptyBucket(),
emptyBucket()).isThrottled());
+
+ }
+
+ @Test
+ public void backoffDelayIsZeroWhenBucketsAreNonEmpty() {
+ ThrottlingState state = new ThrottlingState(fullBucket(),
fullBucket());
+ assertThat(state.getBackoffDelay(), equalTo(Duration.of(0, SECONDS)));
+ }
+
+ @Test
+ public void backoffDelayIsTheLargestDelay() {
+ TokenBucket emptyBucket1 = mock(TokenBucket.class);
+ when(emptyBucket1.getNumTokens()).thenReturn(0L);
+
when(emptyBucket1.getDurationUntilNextRefill(MILLISECONDS)).thenReturn(1000L);
+
+ TokenBucket emptyBucket2 = mock(TokenBucket.class);
+ when(emptyBucket2.getNumTokens()).thenReturn(0L);
+
when(emptyBucket2.getDurationUntilNextRefill(MILLISECONDS)).thenReturn(10000L);
+
+ assertThat(
+ new ThrottlingState(fullBucket(),
emptyBucket1).getBackoffDelay(),
+ equalTo(Duration.of(1, SECONDS)));
+
+ assertThat(
+ new ThrottlingState(emptyBucket1,
emptyBucket2).getBackoffDelay(),
+ equalTo(Duration.of(10, SECONDS)));
+ }
+
+ @Test
+ public void canConsumeTime() {
+ TokenBucket timeBucket = fullBucket();
+ ThrottlingState state = new ThrottlingState(timeBucket, fullBucket());
+ long tokensBefore = timeBucket.getNumTokens();
+
+ state.consumeTime(Duration.of(500, MILLIS));
+
+ assertThat(timeBucket.getNumTokens() + 500, equalTo(tokensBefore));
+ }
+
+ @Test
+ public void canConsumeTimeEvenIfNotEnoughTokensAvailable() {
+ TokenBucket timeBucket = fullBucket();
+ ThrottlingState state = new ThrottlingState(timeBucket, fullBucket());
+
+ state.consumeTime(Duration.of(5, SECONDS));
+
+ assertThat(timeBucket.getNumTokens(), equalTo(0L));
+ }
+
+ @Test
+ public void canConsumeErrors() {
+ TokenBucket errorsBucket = fullBucket();
+ ThrottlingState state = new ThrottlingState(fullBucket(),
errorsBucket);
+ long tokensBefore = errorsBucket.getNumTokens();
+
+ state.consumeError();
+
+ assertThat(errorsBucket.getNumTokens() + 1, equalTo(tokensBefore));
+ }
+
+ @Test
+ public void canConsumeErrorsEvenIfNotEnoughTokensAvailable() {
+ TokenBucket errorsBucket = emptyBucket();
+ ThrottlingState state = new ThrottlingState(errorsBucket,
fullBucket());
+
+ state.consumeError();
+
+ assertThat(errorsBucket.getNumTokens(), equalTo(0L));
+ }
+
+ private TokenBucket fullBucket() {
+ return TokenBuckets.builder()
+ .withCapacity(1000)
+ .withFixedIntervalRefillStrategy(1000, 1, MINUTES)
+ .build();
+ }
+
+ private TokenBucket emptyBucket() {
+ TokenBucket bucket = TokenBuckets.builder()
+ .withCapacity(1000)
+ .withFixedIntervalRefillStrategy(1000, 1, MINUTES)
+ .build();
+ while (bucket.tryConsume()) {
+ // consume all tokens
+ }
+ return bucket;
+ }
+
+
+
+}
diff --git
a/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/UserAgentIpAddressBucketingTest.java
b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/UserAgentIpAddressBucketingTest.java
new file mode 100644
index 0000000..f8c90ee
--- /dev/null
+++
b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/throttling/UserAgentIpAddressBucketingTest.java
@@ -0,0 +1,76 @@
+package org.wikidata.query.rdf.blazegraph.throttling;
+
+import org.junit.Before;
+import org.junit.Test;
+import
org.wikidata.query.rdf.blazegraph.throttling.UserAgentIpAddressBucketing.Bucket;
+
+import javax.servlet.http.HttpServletRequest;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class UserAgentIpAddressBucketingTest {
+
+ private UserAgentIpAddressBucketing bucketting;
+
+ @Before
+ public void createBuckettingUnderTest() {
+ bucketting = new UserAgentIpAddressBucketing();
+ }
+
+ @Test
+ public void sameUserAgentAndIpAddressAreInTheSameBucket() {
+ Bucket bucket1 = bucketting.bucket(mockRequest("1.2.3.4", "UA1"));
+ Bucket bucket2 = bucketting.bucket(mockRequest("1.2.3.4", "UA1"));
+
+ assertThat(bucket1, equalTo(bucket2));
+ assertThat(bucket1.hashCode(), equalTo(bucket2.hashCode()));
+ }
+
+ @Test
+ public void nullUserAgentAndNonNullIpAddressAreInTheSameBucket() {
+ Bucket bucket1 = bucketting.bucket(mockRequest("1.2.3.4", null));
+ Bucket bucket2 = bucketting.bucket(mockRequest("1.2.3.4", null));
+
+ assertThat(bucket1, equalTo(bucket2));
+ assertThat(bucket1.hashCode(), equalTo(bucket2.hashCode()));
+ }
+
+ @Test
+ public void nullUserAgentAndNullIpAddressAreInTheSameBucket() {
+ Bucket bucket1 = bucketting.bucket(mockRequest(null, null));
+ Bucket bucket2 = bucketting.bucket(mockRequest(null, null));
+
+ assertThat(bucket1, equalTo(bucket2));
+ assertThat(bucket1.hashCode(), equalTo(bucket2.hashCode()));
+ }
+
+ @Test
+ public void differentUserAgentsAreInDifferentBuckets() {
+ Bucket bucket1 = bucketting.bucket(mockRequest("1.2.3.4", "UA1"));
+ Bucket bucket2 = bucketting.bucket(mockRequest("1.2.3.4", "UA2"));
+
+ assertThat(bucket1, not(equalTo(bucket2)));
+ assertThat(bucket1.hashCode(), not(equalTo(bucket2.hashCode())));
+ }
+
+ @Test
+ public void differentIpAddressesAreInDifferentBuckets() {
+ Bucket bucket1 = bucketting.bucket(mockRequest("1.2.3.4", "UA1"));
+ Bucket bucket2 = bucketting.bucket(mockRequest("4.3.2.1", "UA1"));
+
+ assertThat(bucket1, not(equalTo(bucket2)));
+ assertThat(bucket1.hashCode(), not(equalTo(bucket2.hashCode())));
+ }
+
+ private HttpServletRequest mockRequest(String ipAddress, String userAgent)
{
+ HttpServletRequest request1 = mock(HttpServletRequest.class);
+ when(request1.getRemoteAddr()).thenReturn(ipAddress);
+ when(request1.getHeader("User-Agent")).thenReturn(userAgent);
+ return request1;
+ }
+
+}
diff --git a/pom.xml b/pom.xml
index 7d65391..866e8a8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -247,7 +247,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>21.0</version>
+ <version>22.0</version>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
@@ -422,6 +422,11 @@
<version>${jetty.version}</version>
</dependency>
<dependency>
+ <groupId>org.isomorphism</groupId>
+ <artifactId>token-bucket</artifactId>
+ <version>1.6</version>
+ </dependency>
+ <dependency>
<groupId>org.jolokia</groupId>
<artifactId>jolokia-jvm</artifactId>
<version>1.3.1</version>
@@ -460,6 +465,10 @@
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -600,6 +609,24 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-test</artifactId>
+ <version>4.3.10.RELEASE</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>org.wikidata.query.rdf</groupId>
<artifactId>testTools</artifactId>
<version>${project.version}</version>
diff --git a/war/src/main/webapp/WEB-INF/web.xml
b/war/src/main/webapp/WEB-INF/web.xml
index 91efc26..b6f4484 100644
--- a/war/src/main/webapp/WEB-INF/web.xml
+++ b/war/src/main/webapp/WEB-INF/web.xml
@@ -84,6 +84,24 @@
<listener>
<listener-class>org.wikidata.query.rdf.blazegraph.WikibaseContextListener</listener-class>
</listener>
+ <filter>
+ <filter-name>real-ip-filter</filter-name>
+
<filter-class>org.wikidata.query.rdf.blazegraph.throttling.ClientIPFilter</filter-class>
+ </filter>
+ <filter-mapping>
+ <filter-name>real-ip-filter</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+ <filter>
+ <filter-name>throttling-filter</filter-name>
+
<filter-class>org.wikidata.query.rdf.blazegraph.throttling.ThrottlingFilter</filter-class>
+ </filter>
+ <filter-mapping>
+ <filter-name>throttling-filter</filter-name>
+ <url-pattern>/sparql</url-pattern>
+ <url-pattern>/namespace</url-pattern>
+ <url-pattern>/namespace/*</url-pattern>
+ </filter-mapping>
<servlet>
<servlet-name>REST API</servlet-name>
<display-name>REST API</display-name>
--
To view, visit https://gerrit.wikimedia.org/r/367664
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: If3c0c28c47f953fdb7f3b6186da8a9535cc18bdf
Gerrit-PatchSet: 11
Gerrit-Project: wikidata/query/rdf
Gerrit-Branch: master
Gerrit-Owner: Gehel <[email protected]>
Gerrit-Reviewer: Gehel <[email protected]>
Gerrit-Reviewer: Smalyshev <[email protected]>
Gerrit-Reviewer: Volans <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
