JulianFeinauer commented on a change in pull request #46: PLC4X-88: Add
Triggering to PLC Scraper
URL: https://github.com/apache/incubator-plc4x/pull/46#discussion_r262821820
##########
File path:
plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java
##########
@@ -19,156 +19,23 @@
package org.apache.plc4x.java.scraper;
-import org.apache.commons.collections4.MultiValuedMap;
-import org.apache.commons.collections4.multimap.ArrayListValuedHashMap;
-import org.apache.commons.lang3.Validate;
-import org.apache.commons.lang3.concurrent.BasicThreadFactory;
-import org.apache.commons.lang3.tuple.Triple;
-import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
-import org.apache.commons.pool2.impl.GenericKeyedObjectPool;
-import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig;
-import org.apache.plc4x.java.PlcDriverManager;
-import org.apache.plc4x.java.api.PlcConnection;
-import org.apache.plc4x.java.scraper.config.ScraperConfiguration;
-import org.apache.plc4x.java.scraper.util.PercentageAboveThreshold;
-import org.apache.plc4x.java.utils.connectionpool.PooledPlcDriverManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.*;
-
/**
- * Main class that orchestrates scraping.
+ * Created by timbo on 2019-03-01
*/
-public class Scraper {
-
- private static final Logger LOGGER =
LoggerFactory.getLogger(Scraper.class);
-
- private final ScheduledExecutorService scheduler =
Executors.newScheduledThreadPool(10,
- new BasicThreadFactory.Builder()
- .namingPattern("scheduler-thread-%d")
- .daemon(false)
- .build()
- );
- private final ExecutorService handlerPool = Executors.newFixedThreadPool(4,
- new BasicThreadFactory.Builder()
- .namingPattern("handler-thread-%d")
- .daemon(true)
- .build()
- );
-
- private final ResultHandler resultHandler;
-
- private final MultiValuedMap<ScrapeJob, ScraperTask> tasks = new
ArrayListValuedHashMap<>();
- private final MultiValuedMap<ScraperTask, ScheduledFuture<?>> futures =
new ArrayListValuedHashMap<>();
- private final PlcDriverManager driverManager;
- private final List<ScrapeJob> jobs;
-
- /**
- * Creates a Scraper instance from a configuration.
- * By default a {@link PooledPlcDriverManager} is used.
- * @param config Configuration to use.
- * @param resultHandler
- */
- public Scraper(ScraperConfiguration config, ResultHandler resultHandler) {
- this(resultHandler, createPooledDriverManager(), config.getJobs());
- }
-
+public interface Scraper {
/**
- * Min Idle per Key is set to 1 for situations where the network is broken.
- * Then, on reconnect we can fail all getConnection calls (in the
ScraperTask) fast until
- * (in the background) the idle connection is created and the
getConnection call returns fast.
- */
- private static PooledPlcDriverManager createPooledDriverManager() {
- return new PooledPlcDriverManager(pooledPlcConnectionFactory -> {
- GenericKeyedObjectPoolConfig<PlcConnection> poolConfig = new
GenericKeyedObjectPoolConfig<>();
- poolConfig.setMinIdlePerKey(1); // This should avoid problems
with long running connect attempts??
- poolConfig.setTestOnBorrow(true);
- poolConfig.setTestOnReturn(true);
- return new GenericKeyedObjectPool<>(pooledPlcConnectionFactory,
poolConfig);
- });
- }
-
- /**
- *
- * @param resultHandler
- * @param driverManager
Review comment:
same here and for all methods below... I usually use the "copy" for the
comments when I do the extract interface in IntelliJ : )
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
