Smalyshev has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/338030 )
Change subject: Allow whitelisted remote services
......................................................................
Allow whitelisted remote services
Bug: T155127
Change-Id: Iae1b19c0f6674bb14db0fd5fb308f08e1e621f9d
---
M
blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseContextListener.java
M blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/geo/GeoService.java
M
blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/label/LabelService.java
M dist/src/script/runBlazegraph.sh
4 files changed, 80 insertions(+), 9 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikidata/query/rdf
refs/changes/30/338030/1
diff --git
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseContextListener.java
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseContextListener.java
index c5d651d..7bf98ff 100644
---
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseContextListener.java
+++
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseContextListener.java
@@ -1,5 +1,11 @@
package org.wikidata.query.rdf.blazegraph;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
import java.util.Map;
import javax.servlet.ServletContextEvent;
@@ -24,6 +30,7 @@
import com.bigdata.bop.BOpContextBase;
import com.bigdata.bop.IValueExpression;
+import com.bigdata.rdf.graph.impl.bd.GASService;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.constraints.DateBOp.DateOp;
import com.bigdata.rdf.sail.sparql.PrefixDeclProcessor;
@@ -34,31 +41,58 @@
import com.bigdata.rdf.sparql.ast.FunctionRegistry.Factory;
import com.bigdata.rdf.sparql.ast.eval.AST2BOpUtility;
import com.bigdata.rdf.sparql.ast.eval.AbstractServiceFactoryBase;
+import com.bigdata.rdf.sparql.ast.eval.SampleServiceFactory;
+import com.bigdata.rdf.sparql.ast.eval.SliceServiceFactory;
+import com.bigdata.rdf.sparql.ast.eval.ValuesServiceFactory;
import com.bigdata.rdf.sparql.ast.service.IServiceOptions;
+import com.bigdata.rdf.sparql.ast.service.RemoteServiceFactoryImpl;
import com.bigdata.rdf.sparql.ast.service.RemoteServiceOptions;
+import com.bigdata.rdf.sparql.ast.service.SPARQLVersion;
import com.bigdata.rdf.sparql.ast.service.ServiceCall;
import com.bigdata.rdf.sparql.ast.service.ServiceCallCreateParams;
+import com.bigdata.rdf.sparql.ast.service.ServiceFactory;
import com.bigdata.rdf.sparql.ast.service.ServiceRegistry;
-import com.bigdata.service.fts.FTS;
+import com.bigdata.rdf.store.BDS;
/**
* Context listener to enact configurations we need on initialization.
*/
+@SuppressWarnings("checkstyle:classfanoutcomplexity")
public class WikibaseContextListener extends BigdataRDFServletContextListener {
private static final Logger log =
LoggerFactory.getLogger(WikibaseContextListener.class);
/**
- * Replaces the default Blazegraph services with ones that do not allow
- * remote services and a label resolution service.
+ * Default service whitelist filename.
+ */
+ public static final String WHITELIST_DEFAULT = "whitelist.txt";
+
+ /**
+ * Whitelist configuration name.
+ */
+ public static final String WHITELIST =
System.getProperty("wikibaseServiceWhitelist", WHITELIST_DEFAULT);
+
+ /**
+ * Initializes BG service setup to allow whitelisted services.
+ * Also add additional custom services and functions.
*/
public static void initializeServices() {
- ServiceRegistry.getInstance().setDefaultServiceFactory(new
DisableRemotesServiceFactory());
+ // Enable service whitelisting
+ final ServiceRegistry reg = ServiceRegistry.getInstance();
+ reg.setWhitelistEnabled(true);
LabelService.register();
GeoService.register();
- // Remove FTS service for now since it allows arbitrary endpoints
- ServiceRegistry.getInstance().remove(FTS.SEARCH);
+ // Whitelist services we like by default
+ reg.addWhitelistURL(GASService.Options.SERVICE_KEY.toString());
+ reg.addWhitelistURL(ValuesServiceFactory.SERVICE_KEY.toString());
+ reg.addWhitelistURL(BDS.SEARCH_IN_SEARCH.toString());
+ reg.addWhitelistURL(SliceServiceFactory.SERVICE_KEY.toString());
+ reg.addWhitelistURL(SampleServiceFactory.SERVICE_KEY.toString());
+ loadWhitelist(reg);
+
+ // Initialize remote services
+ reg.setDefaultServiceFactory(getDefaultServiceFactory());
// Override date functions so that we can handle them
// via WikibaseDate
@@ -103,6 +137,36 @@
}
/**
+ * Get default service factory, with proper options.
+ * @return
+ */
+ private static ServiceFactory getDefaultServiceFactory() {
+ final RemoteServiceOptions options = new RemoteServiceOptions();
+ options.setSPARQLVersion(SPARQLVersion.SPARQL_11);
+ options.setGET(true);
+ return new RemoteServiceFactoryImpl(options);
+ }
+
+ /**
+ * Load whitelist from file.
+ * @param reg
+ */
+ private static void loadWhitelist(final ServiceRegistry reg) {
+ try {
+ List<String> lines = Files.readAllLines(Paths.get(WHITELIST),
+ StandardCharsets.UTF_8);
+ for (String line : lines) {
+ reg.addWhitelistURL(line);
+ }
+ } catch (FileNotFoundException e) {
+ // ignore file not found
+ log.info("Whitelist file {} not found, ignoring.", WHITELIST);
+ } catch (IOException e) {
+ log.warn("Failed reading from whitelist file");
+ }
+ }
+
+ /**
* Add standard prefixes to the system.
* @param uris Wikidata URIs to use
*/
diff --git
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/geo/GeoService.java
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/geo/GeoService.java
index a2751f4..330a6f0 100644
---
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/geo/GeoService.java
+++
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/geo/GeoService.java
@@ -59,8 +59,12 @@
* Register the service so it is recognized by Blazegraph.
*/
public static void register() {
- ServiceRegistry.getInstance().add(GeoAroundService.SERVICE_KEY, new
GeoAroundService());
- ServiceRegistry.getInstance().add(GeoBoxService.SERVICE_KEY, new
GeoBoxService());
+ final ServiceRegistry reg = ServiceRegistry.getInstance();
+
+ reg.add(GeoAroundService.SERVICE_KEY, new GeoAroundService());
+ reg.addWhitelistURL(GeoAroundService.SERVICE_KEY.toString());
+ reg.add(GeoBoxService.SERVICE_KEY, new GeoBoxService());
+ reg.addWhitelistURL(GeoBoxService.SERVICE_KEY.toString());
}
@Override
diff --git
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/label/LabelService.java
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/label/LabelService.java
index 6b6b773..82e5df9 100644
---
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/label/LabelService.java
+++
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/label/LabelService.java
@@ -106,7 +106,9 @@
* Register the service so it is recognized by Blazegraph.
*/
public static void register() {
- ServiceRegistry.getInstance().add(SERVICE_KEY, new LabelService());
+ final ServiceRegistry reg = ServiceRegistry.getInstance();
+ reg.add(SERVICE_KEY, new LabelService());
+ reg.addWhitelistURL(SERVICE_KEY.toString());
}
@Override
diff --git a/dist/src/script/runBlazegraph.sh b/dist/src/script/runBlazegraph.sh
index 9d1c053..575c1fe 100755
--- a/dist/src/script/runBlazegraph.sh
+++ b/dist/src/script/runBlazegraph.sh
@@ -42,6 +42,7 @@
-Dcom.bigdata.rdf.sparql.ast.QueryHints.analyticMaxMemoryPerQuery=1073741824 \
-DASTOptimizerClass=org.wikidata.query.rdf.blazegraph.WikibaseOptimizers \
-Dorg.wikidata.query.rdf.blazegraph.inline.literal.WKTSerializer.noGlobe=$DEFAULT_GLOBE
\
+
-Dcom.bigdata.rdf.sail.webapp.client.RemoteRepository.maxRequestURLLength=7168 \
-DwikibasePrefixes=$DIR/prefixes.conf \
${BLAZEGRAPH_OPTS} \
-jar jetty-runner*.jar \
--
To view, visit https://gerrit.wikimedia.org/r/338030
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Iae1b19c0f6674bb14db0fd5fb308f08e1e621f9d
Gerrit-PatchSet: 1
Gerrit-Project: wikidata/query/rdf
Gerrit-Branch: master
Gerrit-Owner: Smalyshev <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
