chrismattmann closed pull request #152: Added crawling progress bar
URL: https://github.com/apache/drat/pull/152
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/crawler/src/main/resources/bin/dump_repo_details.py
b/crawler/src/main/resources/bin/dump_repo_details.py
index efaf64e3..dac623a5 100644
--- a/crawler/src/main/resources/bin/dump_repo_details.py
+++ b/crawler/src/main/resources/bin/dump_repo_details.py
@@ -19,6 +19,7 @@ def main(argv=None):
rep = eval(data)
reponame = os.path.basename(os.path.normpath(argv[0]))
+ rep["id"] = "id:"+os.path.normpath(argv[0])
rep["repo"] = os.path.normpath(argv[0])
rep["name"] = reponame
diff --git a/distribution/src/main/assembly/assembly.xml
b/distribution/src/main/assembly/assembly.xml
index 30a75aa6..8acf7ca3 100644
--- a/distribution/src/main/assembly/assembly.xml
+++ b/distribution/src/main/assembly/assembly.xml
@@ -32,6 +32,13 @@
<exclude>**/*</exclude>
</excludes>
</fileSet>
+ <fileSet>
+ <directory>target</directory>
+ <outputDirectory>data/clones</outputDirectory>
+ <excludes>
+ <exclude>**/*</exclude>
+ </excludes>
+ </fileSet>
<fileSet>
<directory>target</directory>
<outputDirectory>data/archive</outputDirectory>
diff --git a/proteus/src/main/java/backend/AbstractDratWrapper.java
b/proteus/src/main/java/backend/AbstractDratWrapper.java
index bc323490..6ef4b333 100644
--- a/proteus/src/main/java/backend/AbstractDratWrapper.java
+++ b/proteus/src/main/java/backend/AbstractDratWrapper.java
@@ -17,6 +17,8 @@
package backend;
+import drat.proteus.rest.DratRequestWrapper;
+
/**
* This interface abstracts the main communication methods between Apache DRAT
* and Proteus
@@ -38,7 +40,5 @@
public String getIndexablePath();
- public String getUrlLoc();
-
- public void setUrlLoc(String urlLoc);
+ void setData(DratRequestWrapper body);
}
diff --git a/proteus/src/main/java/backend/FileConstants.java
b/proteus/src/main/java/backend/FileConstants.java
index 5dcd8d65..b502e3ee 100644
--- a/proteus/src/main/java/backend/FileConstants.java
+++ b/proteus/src/main/java/backend/FileConstants.java
@@ -28,6 +28,7 @@
public static final String OODT_PATH =
buildDratSubdirectoryPath("/deploy/bin/oodt");
public static final String WORKFLOW_PATH =
buildDratSubdirectoryPath("/deploy/workflow/bin/wmgr-client");
public static final String DRAT_PATH =
buildDratSubdirectoryPath("/deploy/bin/drat");
+ public static final String DRAT_CLONES =
buildDratSubdirectoryPath("/deploy/data/clones");
public static final String DRAT_TEMP_UNZIPPED_PATH =
buildDratSubdirectoryPath("/deploy/data/staging");
public static final String CURRENT_REPO_DETAILS_FILE =
buildDratSubdirectoryPath("/deploy/data/repo");
public static final String DRAT_TEMP_LOG_OUTPUT =
buildDratSubdirectoryPath("/deploy/data/drat_output.log");
diff --git a/proteus/src/main/java/backend/ProcessDratWrapper.java
b/proteus/src/main/java/backend/ProcessDratWrapper.java
index c71cfde5..4ebd571b 100644
--- a/proteus/src/main/java/backend/ProcessDratWrapper.java
+++ b/proteus/src/main/java/backend/ProcessDratWrapper.java
@@ -17,6 +17,8 @@
package backend;
+import com.google.gson.Gson;
+import drat.proteus.rest.DratRequestWrapper;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.PumpStreamHandler;
@@ -48,6 +50,7 @@
import java.io.IOException;
import java.net.URL;
import java.nio.charset.Charset;
+import java.nio.file.Files;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
@@ -77,10 +80,13 @@
"RatAggregateLog" };
private String status;
- public String urlLoc;
+
private FileManagerUtils fm;
private String path;
+ private DratRequestWrapper body;
+
private static ProcessDratWrapper singletonDratWrapper = new
ProcessDratWrapper();
+ private boolean runnning = false;
public static ProcessDratWrapper getInstance() {
return singletonDratWrapper;
@@ -89,7 +95,7 @@ public static ProcessDratWrapper getInstance() {
private ProcessDratWrapper() {
super(DRAT);
this.path = "";
- this.urlLoc="";
+ body = new DratRequestWrapper();
this.status = "IDLE";
this.fm = new FileManagerUtils(
PathUtils.replaceEnvVariables("[FILEMGR_URL]"));
@@ -102,15 +108,13 @@ public void setIndexablePath(String canonicalPath) {
public String getIndexablePath() {
return this.path;
}
-
- public String getUrlLoc() {
- return urlLoc;
- }
-
- public void setUrlLoc(String urlLoc) {
- this.urlLoc = urlLoc;
+
+ @Override
+ public void setData(DratRequestWrapper body) {
+ this.body = body;
}
-
+
+
public String getStatus() {
return this.status;
}
@@ -121,51 +125,44 @@ public synchronized void setStatus(String status) {
@Override
public void crawl() throws Exception {
- DratLog crawlLog = new DratLog("CRAWLING");
-
- versionControlCheck();
- try{
- setStatus(CRAWL_CMD);
-
- crawlLog.logInfo("Configuring");
- String beanRepo =
System.getProperty("org.apache.oodt.cas.crawl.bean.repo",
- FileConstants.CRAWLER_CONFIG);
- String crawlerId = "MetExtractorProductCrawler";
- System.setProperty("DRAT_EXCLUDE","");
- FileSystemXmlApplicationContext appContext = new
FileSystemXmlApplicationContext("file:"+beanRepo);
-
- MetExtractorProductCrawler crawler = new
MetExtractorProductCrawler();
- crawler.setApplicationContext(appContext);
- crawler.setId(crawlerId);
-
crawler.setMetExtractor("org.apache.oodt.cas.metadata.extractors.CopyAndRewriteExtractor");
- crawler.setMetExtractorConfig(FileConstants.MET_EXT_CONFIG_PATH);
- crawler.setFilemgrUrl(FileConstants.FILEMGR_URL);
-
crawler.setClientTransferer("org.apache.oodt.cas.filemgr.datatransfer.InPlaceDataTransferFactory");
- crawler.setPreCondIds(Arrays.asList("RegExExcludeComparator"));
- crawler.setProductPath(this.path);
- crawlLog.logInfo("STARTING ",null);
- crawler.crawl();
- crawlLog.logInfo("COMPLETED",null);
- }catch (Exception ex) {
- crawlLog.logSevere("ERROR ",ex.getLocalizedMessage());
- ex.printStackTrace();
- throw ex;
- }
+ reset();
+ versionControlCheck();
+
+ dumpToFile(this.body);
+ simpleCrawl();
+
}
-
- private void versionControlCheck() throws Exception {
- File dirPathFile = new File(this.path);
- boolean clone=false;
- if(dirPathFile.exists() ){
- if(dirPathFile.isDirectory()&& dirPathFile.list().length==0){
- clone=true;
- }
- }else{
- dirPathFile.createNewFile();
- clone =true;
- }
- if(clone){
- parseAsVersionControlledRepo();
+
+ private void simpleCrawl() throws Exception{
+ DratLog crawlLog = new DratLog("CRAWLING");
+
+
+ try{
+ setStatus(CRAWL_CMD);
+
+ crawlLog.logInfo("Configuring");
+ String beanRepo =
System.getProperty("org.apache.oodt.cas.crawl.bean.repo",
+ FileConstants.CRAWLER_CONFIG);
+ String crawlerId = "MetExtractorProductCrawler";
+ System.setProperty("DRAT_EXCLUDE","");
+ FileSystemXmlApplicationContext appContext = new
FileSystemXmlApplicationContext("file:"+beanRepo);
+
+ MetExtractorProductCrawler crawler = new MetExtractorProductCrawler();
+ crawler.setApplicationContext(appContext);
+ crawler.setId(crawlerId);
+
crawler.setMetExtractor("org.apache.oodt.cas.metadata.extractors.CopyAndRewriteExtractor");
+ crawler.setMetExtractorConfig(FileConstants.MET_EXT_CONFIG_PATH);
+ crawler.setFilemgrUrl(FileConstants.FILEMGR_URL);
+
crawler.setClientTransferer("org.apache.oodt.cas.filemgr.datatransfer.InPlaceDataTransferFactory");
+ crawler.setPreCondIds(Arrays.asList("RegExExcludeComparator"));
+ crawler.setProductPath(this.path);
+ crawlLog.logInfo("STARTING ",null);
+ crawler.crawl();
+ crawlLog.logInfo("COMPLETED",null);
+ }catch (Exception ex) {
+ crawlLog.logSevere("ERROR ",ex.getLocalizedMessage());
+ ex.printStackTrace();
+ throw ex;
}
}
@@ -280,9 +277,10 @@ public void reset() {
public void go() throws Exception {
// before go, always reset
+ reset();
versionControlCheck();
- this.reset();
- this.crawl();
+ dumpToFile(this.body);
+ this.simpleCrawl();
this.solrIndex();
this.map();
@@ -488,36 +486,52 @@ private synchronized void wipeProductType(String
productTypeName) {
}
- private String parseAsVersionControlledRepo()
- throws IOException {
- String projectName = null;
- boolean git = urlLoc.endsWith(".git");
- String line = null;
- if (git) {
- line = "git clone --depth 1 --branch master " + urlLoc;
- } else {
- projectName = urlLoc.substring(urlLoc.lastIndexOf("/") + 1);
- line = "svn export " + urlLoc;
- }
- String clonePath = this.path;
- File cloneDir = new File(clonePath);
- LOG.info("Cloning Git / SVN project: [" + projectName + "] remote repo: ["
- + this.urlLoc + "] into " + this.path);
-
- CommandLine cmdLine = CommandLine.parse(line);
- DefaultExecutor executor = new DefaultExecutor();
- executor.setWorkingDirectory(new File(this.path));
- int exitValue = executor.execute(cmdLine);
-
- if (git) {
- String gitHiddenDirPath = clonePath + File.separator + ".git";
- File gitHiddenDir = new File(gitHiddenDirPath);
- LOG.info("Removing .git directory from " + gitHiddenDirPath);
- org.apache.cxf.helpers.FileUtils.removeDir(gitHiddenDir);
+
+ private void versionControlCheck() throws Exception {
+ if (path.startsWith("http://";) || path.startsWith("https://";)) {
+ this.body.loc_url = this.path;
+ String projectName = null;
+ boolean git = path.endsWith(".git");
+ File tmpDir = new File(FileConstants.DRAT_CLONES);
+ String tmpDirPath = tmpDir.getCanonicalPath();
+ String line = null;
+ if (git) {
+ projectName = path.substring(path.lastIndexOf("/") + 1,
+ path.lastIndexOf("."));
+ line = "git clone --depth 1 --branch master " + path;
+ } else {
+ projectName = path.substring(path.lastIndexOf("/") + 1);
+ line = "svn export " + path;
+ }
+ String clonePath = tmpDirPath + File.separator + projectName;
+ File cloneDir = new File(clonePath);
+ if (cloneDir.isDirectory() && cloneDir.exists()) {
+ LOG.info(
+ "Git / SVN clone: [" + clonePath + "] already exists, removing
it.");
+ org.apache.cxf.helpers.FileUtils.removeDir(cloneDir);
+ }
+ LOG.info("Cloning Git / SVN project: [" + projectName + "] remote repo:
["
+ + path + "] into " + tmpDirPath);
+
+ CommandLine cmdLine = CommandLine.parse(line);
+ DefaultExecutor executor = new DefaultExecutor();
+ executor.setWorkingDirectory(tmpDir);
+ int exitValue = executor.execute(cmdLine);
+
+ if (git) {
+ String gitHiddenDirPath = clonePath + File.separator + ".git";
+ File gitHiddenDir = new File(gitHiddenDirPath);
+ LOG.info("Removing .git directory from " + gitHiddenDirPath);
+ org.apache.cxf.helpers.FileUtils.removeDir(gitHiddenDir);
+ }
+
+ this.path = clonePath;
+ }else{
+ this.body.loc_url = "http://drat.apache.org/#";;
}
-
- return clonePath;
-
+
+ this.body.id = "id:"+this.path;
+ this.body.repo = this.path;
}
private synchronized void wipeInstanceRepo(String wmUrl) {
@@ -547,6 +561,12 @@ private synchronized void wipeSolrCore(String coreName) {
}
}
+
+ public void dumpToFile(DratRequestWrapper body) throws IOException {
+ File repo = new File(FileConstants.CURRENT_REPO_DETAILS_FILE);
+ Files.write(repo.toPath(),new Gson().toJson(body).getBytes());
+ }
+
private class DratLog{
private static final String MODULE = "DRAT_LOG";
long startTime =0;
diff --git a/proteus/src/main/java/drat/proteus/WicketApplication.java
b/proteus/src/main/java/drat/proteus/WicketApplication.java
index d84d1641..4d65c791 100644
--- a/proteus/src/main/java/drat/proteus/WicketApplication.java
+++ b/proteus/src/main/java/drat/proteus/WicketApplication.java
@@ -25,6 +25,8 @@
import java.util.logging.Logger;
import java.util.regex.Pattern;
+import drat.proteus.filemgr.rest.FileManagerProgressResponse;
+import drat.proteus.filemgr.rest.FileManagerRestResource;
import drat.proteus.rest.DratRestResource;
import drat.proteus.rest.ServicesRestResource;
import drat.proteus.workflow.rest.WorkflowRestResource;
@@ -82,6 +84,12 @@ public IResource getResource() {
return resource;
}
});
+ mountResource("/filemanager", new ResourceReference("restReference") {
+ @Override
+ public IResource getResource() {
+ return new FileManagerRestResource();
+ }
+ });
mountPage("/workflow", DratWorkflow.class);
doImageMounts(
diff --git
a/proteus/src/main/java/drat/proteus/filemgr/rest/FileManagerProgressResponse.java
b/proteus/src/main/java/drat/proteus/filemgr/rest/FileManagerProgressResponse.java
new file mode 100644
index 00000000..7b564c35
--- /dev/null
+++
b/proteus/src/main/java/drat/proteus/filemgr/rest/FileManagerProgressResponse.java
@@ -0,0 +1,5 @@
+package drat.proteus.filemgr.rest;
+
+public class FileManagerProgressResponse {
+ public int crawledFiles;
+}
diff --git
a/proteus/src/main/java/drat/proteus/filemgr/rest/FileManagerRestResource.java
b/proteus/src/main/java/drat/proteus/filemgr/rest/FileManagerRestResource.java
new file mode 100644
index 00000000..805d5203
--- /dev/null
+++
b/proteus/src/main/java/drat/proteus/filemgr/rest/FileManagerRestResource.java
@@ -0,0 +1,43 @@
+package drat.proteus.filemgr.rest;
+
+import backend.FileConstants;
+import org.apache.oodt.cas.filemgr.structs.ProductType;
+import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
+import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
+import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.util.PathUtils;
+import org.wicketstuff.rest.annotations.MethodMapping;
+import
org.wicketstuff.rest.contenthandling.json.webserialdeserial.GsonWebSerialDeserial;
+import org.wicketstuff.rest.resource.AbstractRestResource;
+import org.wicketstuff.rest.utils.http.HttpMethod;
+
+import java.net.URL;
+import java.util.logging.Logger;
+
+public class FileManagerRestResource extends
AbstractRestResource<GsonWebSerialDeserial> {
+
+ private static final long serialVersionUID = -588588505908995065L;
+ XmlRpcFileManagerClient fileManagerClient;
+
+ private static final Logger LOG =
Logger.getLogger(FileManagerRestResource.class.getName());
+
+ public FileManagerRestResource() {
+ super(new GsonWebSerialDeserial());
+ try{
+ fileManagerClient = new XmlRpcFileManagerClient(new
URL(PathUtils.replaceEnvVariables(FileConstants.FILEMGR_URL)));
+ }catch (Exception ex){
+ LOG.severe(ex.getMessage());
+ }
+
+ }
+
+ @MethodMapping(value = "/progress",httpMethod = HttpMethod.GET)
+ public FileManagerProgressResponse getProgress() throws Exception {
+ FileManagerProgressResponse response = new FileManagerProgressResponse();
+ response.crawledFiles =
fileManagerClient.getNumProducts(fileManagerClient.getProductTypeByName("GenericFile"));
+ return response;
+ }
+
+
+}
diff --git a/proteus/src/main/java/drat/proteus/rest/DratRestResource.java
b/proteus/src/main/java/drat/proteus/rest/DratRestResource.java
index 1dff94b8..35c402ba 100644
--- a/proteus/src/main/java/drat/proteus/rest/DratRestResource.java
+++ b/proteus/src/main/java/drat/proteus/rest/DratRestResource.java
@@ -48,24 +48,23 @@ public DratRestResource() {
@MethodMapping(value = "/go", httpMethod = HttpMethod.POST)
public void go(@RequestBody DratRequestWrapper body) throws Exception {
- dumpToFile(body);
+
+ dratWrapper.setData(body);
dratWrapper.setIndexablePath(body.repo);
- dratWrapper.setUrlLoc(body.loc_url);
dratWrapper.go();
}
@MethodMapping(value = "/index", httpMethod = HttpMethod.POST)
public void index(@RequestBody DratRequestWrapper body) throws Exception {
- dumpToFile(body);
+ dratWrapper.setData(body);
dratWrapper.setIndexablePath(body.repo);
dratWrapper.index();
}
@MethodMapping(value = "/crawl", httpMethod = HttpMethod.POST)
public void crawl(@RequestBody DratRequestWrapper body) throws Exception {
- dumpToFile(body);
+ dratWrapper.setData(body);
dratWrapper.setIndexablePath(body.repo);
- dratWrapper.setUrlLoc(body.loc_url);
dratWrapper.crawl();
}
@@ -83,6 +82,11 @@ public void reduce() throws Exception {
public void reset() throws Exception {
dratWrapper.reset();
}
+
+ @MethodMapping(value = "/currentrepo",httpMethod = HttpMethod.GET)
+ public String currentRepo() throws Exception{
+ return dratWrapper.getIndexablePath();
+ }
@MethodMapping(value = "/log", httpMethod = HttpMethod.GET)
public String getProcessLog() {
@@ -98,9 +102,5 @@ public String getProcessLog() {
return "Log is empty!";
}
}
-
- public void dumpToFile(DratRequestWrapper body) throws IOException {
- File repo = new File(FileConstants.CURRENT_REPO_DETAILS_FILE);
- Files.write(repo.toPath(),new Gson().toJson(body).getBytes());
- }
+
}
diff --git
a/webapps/proteus-new/src/main/webapp/resources/src/components/auditsummarycomp.vue
b/webapps/proteus-new/src/main/webapp/resources/src/components/auditsummarycomp.vue
index e8593734..345dfdcf 100644
---
a/webapps/proteus-new/src/main/webapp/resources/src/components/auditsummarycomp.vue
+++
b/webapps/proteus-new/src/main/webapp/resources/src/components/auditsummarycomp.vue
@@ -59,7 +59,7 @@ the License.
var docs = response.data.response.docs;
-
+ // var docs =
[{"license_Generated":0,"license_Archives":1,"license_Standards":108,"license_Unknown":12,"license_Binaries":139,"license_Apache":96,"id":"/media/Workings/GSOC-2018/drat/deploy/data/clones/android-UniversalMusicPlayer","license_Notes":1}];
for(var i = 0; i < docs.length; i++) {
var doc = docs[i];
var repo = doc.id.split("/");
@@ -174,11 +174,18 @@ the License.
bar.append("rect")
.attr("style", function(d,i) { return "fill:"+color(i %
data.series.length); })
.attr("class", "bar")
- .attr("width", x)
+ .attr("width", function(x){return x>0?x:0})
.attr("height", barHeight - 1);
bar.append('rect')
- .attr("x", function(d) { if(x(d) < 10) return x(d) + 10;
else if(x(d) < 75) return x(d) - 77; else return x(d) - 83; })
+ .attr("x", function(d) {
+ var xtoreturn = x(d);
+ if(x(d) < 10) xtoreturn = x(d) + 10;
+ else if(x(d) < 75) xtoreturn = x(d) - 77;
+ else xtoreturn = x(d) - 83;
+ if(xtoreturn<0) return 0;
+ else return xtoreturn;
+ })
.attr("y", 7)
.attr('width', 80)
.attr('height', barHeight - 15)
@@ -186,8 +193,13 @@ the License.
// Add text label in bar
bar.append("text")
- .attr("x", function(d) { if(x(d) < 80) return 80; else
return x(d) - 63; })
- .attr("y", (barHeight / 2)+1)
+ .attr("x", function(d) {
+ var xtoreturn = x(d)
+ if(x(d) < 80) xtoreturn = 5;
+ else xtoreturn = x(d) - 63;
+ if(xtoreturn<0) return 0;
+ else return xtoreturn })
+ .attr("y", (barHeight / 2)+1.5)
.attr("style", function(d,i) { return "fill:"+color(i %
data.series.length); })
.attr("style", "font-size:.55em")
.text(function(d, i) { return d + " " + mylabels[i %
data.series.length]; });
@@ -196,9 +208,15 @@ the License.
// Draw labels
bar.append("text")
.attr("class", "label")
- .attr("x", function(d) { return - 25; })
- .attr("y", groupHeight / 2)
+ .attr("x", function(d,i) {
+ console.log(d);
+ var x = -groupHeight/2;
+ if (i % data.series.length === 0)
+ x-=
(data.labels[Math.floor(i/data.series.length)].length)*3;
+ return x; })
+ .attr("y", -20)
.attr("dy", ".35em")
+ .attr("transform","rotate(-90)")
.text(function(d,i) {
if (i % data.series.length === 0)
return data.labels[Math.floor(i/data.series.length)];
diff --git
a/webapps/proteus-new/src/main/webapp/resources/src/components/barchartcomp.vue
b/webapps/proteus-new/src/main/webapp/resources/src/components/barchartcomp.vue
index 57a028d5..235daa02 100644
---
a/webapps/proteus-new/src/main/webapp/resources/src/components/barchartcomp.vue
+++
b/webapps/proteus-new/src/main/webapp/resources/src/components/barchartcomp.vue
@@ -40,7 +40,6 @@ import store from './../store/store'
props: [],
store,
mounted() {
- this.loadData();
this.timerClearvar = setInterval(function () {
if(this.currentState=="MAP" ||
this.currentState=="REDUCE")this.loadData();
}.bind(this), 1000);
diff --git
a/webapps/proteus-new/src/main/webapp/resources/src/components/controll_bar.vue
b/webapps/proteus-new/src/main/webapp/resources/src/components/controll_bar.vue
index 5a538634..09ad7af1 100644
---
a/webapps/proteus-new/src/main/webapp/resources/src/components/controll_bar.vue
+++
b/webapps/proteus-new/src/main/webapp/resources/src/components/controll_bar.vue
@@ -45,13 +45,6 @@ the License.
<v-spacer/>
<hr/>
<v-text-field
- solo
- name="location"
- label="Location of the repository"
- single-line
- v-model="repoloc"
- />
- <hr/><v-text-field
solo
name="description"
label="Description about the repository"
@@ -166,7 +159,7 @@ the License.
}
- if(this.url.length==0 || this.repoloc==0){
+ if(this.url.length==0 ){
this.$dialog.alert({title:"Invalid input",body:'Please
enter valid path and location, then continue'},options)
.then(function () {
diff --git
a/webapps/proteus-new/src/main/webapp/resources/src/components/filelistcomp.vue
b/webapps/proteus-new/src/main/webapp/resources/src/components/filelistcomp.vue
index c30348ee..a4dc90b1 100644
---
a/webapps/proteus-new/src/main/webapp/resources/src/components/filelistcomp.vue
+++
b/webapps/proteus-new/src/main/webapp/resources/src/components/filelistcomp.vue
@@ -49,7 +49,6 @@ import store from './../store/store';
store,
props: [],
mounted() {
- this.loadData();
this.timerClearVar = setInterval(function () {
if(this.currentState=="CRAWL")this.loadData();
}.bind(this), 1000);
@@ -67,6 +66,12 @@ import store from './../store/store';
},
methods: {
loadData(){
+ if(this.currentRepo.indexOf("http")>=0){
+ axios.get(this.origin+"/proteus/drat/currentrepo")
+ .then(response=>{
+ store.commit("setCurrentRepo",response.data);
+ });
+ }
if(this.currentRepo!=''){
axios.get(this.origin+"/proteus/service/products?topn=10")
.then(response=>{
diff --git
a/webapps/proteus-new/src/main/webapp/resources/src/components/piechart.vue
b/webapps/proteus-new/src/main/webapp/resources/src/components/piechart.vue
index 19d1fa89..03e9df03 100644
--- a/webapps/proteus-new/src/main/webapp/resources/src/components/piechart.vue
+++ b/webapps/proteus-new/src/main/webapp/resources/src/components/piechart.vue
@@ -38,10 +38,9 @@ import store from './../store/store';
store,
props: [],
mounted() {
- this.loadData();
this.timerClearVar = setInterval(function () {
if(this.currentState=="INDEX")this.loadData();
- }.bind(this), 10000);
+ }.bind(this), 1000);
},
beforeDestroy(){
clearInterval(this.timerClearVar);
diff --git
a/webapps/proteus-new/src/main/webapp/resources/src/components/progresscomp.vue
b/webapps/proteus-new/src/main/webapp/resources/src/components/progresscomp.vue
index 04cd578f..8c03a6af 100644
---
a/webapps/proteus-new/src/main/webapp/resources/src/components/progresscomp.vue
+++
b/webapps/proteus-new/src/main/webapp/resources/src/components/progresscomp.vue
@@ -17,7 +17,10 @@ the License.
<section class="progresscomp">
<v-card id="progresscard">
<h1>Progress</h1>
+ <hr>
+
<v-progress-circular
+ id="progresscircle"
:rotate="-90"
:size="100"
:width="15"
@@ -148,6 +151,12 @@ the License.
<style scoped >
#progresscard {
margin-left: 10%;
- margin-top: 10%
+ margin-top: 10%;
+ padding:4%;
+ background: lightgray
+ }
+
+ #progresscircle{
+ margin-top: 4%;
}
</style>
diff --git
a/webapps/proteus-new/src/main/webapp/resources/src/components/statisticscomp.vue
b/webapps/proteus-new/src/main/webapp/resources/src/components/statisticscomp.vue
index 98aa8d76..1280e9ce 100644
---
a/webapps/proteus-new/src/main/webapp/resources/src/components/statisticscomp.vue
+++
b/webapps/proteus-new/src/main/webapp/resources/src/components/statisticscomp.vue
@@ -16,10 +16,23 @@ the License.
<section class="statisticscomp">
<v-card id="statisticscard">
+
+ <v-card id="crawlingprogress">
+ <v-progress-linear height="10"
v-model="crawlingprogress"></v-progress-linear>
+ <strong>{{stat.crawledfiles}}</strong> of
<strong>{{stat.numOfFiles}}</strong> files Crawled
+
+ </v-card>
+ <v-card id="indexingprogress">
+ <v-progress-linear height="10"
v-model="indexingprogress"></v-progress-linear>
+ <strong>{{stat.indexedfiles}}</strong> of
<strong>{{stat.numOfFiles}}</strong> files Indexed
+
+ </v-card>
+
<v-toolbar height="50" color="primary" dark>
<v-toolbar-title>Statistics</v-toolbar-title>
<v-spacer></v-spacer>
</v-toolbar>
+ <hr>
<v-expansion-panel
v-model="panel"
expand
@@ -64,6 +77,8 @@ the License.
this.timerClearVar = setInterval(function () {
if(this.currentState!="IDLE") this.loadSizeData();
if(this.currentState=="MAP" ||
this.currentState=="REDUCE")this.loadInstanceCount()
+ if(this.currentState=="CRAWL") this.loadCrawledFiles();
+ if(this.currentState=="INDEX" ||
this.currentState=="MAP")this.loadIndexedFiles();
}.bind(this), 1000);
},
beforeDestroy(){
@@ -77,6 +92,8 @@ the License.
numOfFiles:0,
runningRatInstances:0,
finishedRatInstances:0,
+ crawledfiles:0,
+ indexedfiles:0,
},
panel:[false,true,true]
@@ -110,7 +127,20 @@ the License.
}
}
});
+ },
+ loadCrawledFiles(){
+ axios.get(this.origin+"/proteus/filemanager/progress")
+ .then(response=> {
+ this.stat.crawledfiles = response.data.crawledFiles;
+ });
+ },
+ loadIndexedFiles(){
+
axios.get(this.origin+"/solr/drat/select?q=producttype:GenericFile&fl=numFound&wt=json&indent=true")
+ .then(response=>{
+ this.stat.indexedfiles = response.data.response.numFound;
+ });
}
+
},
computed: {
currentRepo (){
@@ -121,6 +151,12 @@ the License.
},
currentState(){
return store.state.currentActionStep;
+ },
+ crawlingprogress(){
+ return this.stat.crawledfiles/this.stat.numOfFiles *100;
+ },
+ indexingprogress(){
+ return this.stat.indexedfiles/this.stat.numberOfFiles * 100;
}
}
}
@@ -134,4 +170,7 @@ the License.
#header{
background-color:lightgray
}
+ #indexingprogress{
+ margin-bottom: 5%;
+ }
</style>
diff --git
a/webapps/proteus-new/src/main/webapp/resources/src/components/topmimepiecomp.vue
b/webapps/proteus-new/src/main/webapp/resources/src/components/topmimepiecomp.vue
index 8ab75358..325849c3 100644
---
a/webapps/proteus-new/src/main/webapp/resources/src/components/topmimepiecomp.vue
+++
b/webapps/proteus-new/src/main/webapp/resources/src/components/topmimepiecomp.vue
@@ -64,44 +64,44 @@ the License.
methods: {
init(rows){
axios.get(this.origin +
'/solr/statistics/select?q=type:software&rows=220&fl=mime_*&wt=json')
- .then(function(response) {
-
- console.log(response.data);
- var docs = response.data.response.docs;
- var resultingData = [];
- var result = [];
- var mime = {};
-
- for(var i = 0; i < docs.length; i++) {
- var doc = docs[i];
- for(var x in doc) {
- var key = x.split("mime_")[1];
- var value = doc[x];
- if(typeof mime[key] === 'undefined') {
- mime[key] = value;
- }
- else {
- mime[key] += value;
- }
- }
- }
-
- for(x in mime) {
- var jsonObject = {};
- jsonObject["key"] = x;
- jsonObject["y"] = mime[x];
- resultingData.push(jsonObject);
- }
-
- resultingData.sort(function(a, b) {
- return b.y - a.y;
- });
-
- for( i = 1; i <= rows; i++) {
- result[i-1] = resultingData[i-1];
- }
-
- console.log(result);
+ .then(function(response) {
+
+ console.log(response.data);
+ var docs = response.data.response.docs;
+ var resultingData = [];
+ var result = [];
+ var mime = {};
+
+ for(var i = 0; i < docs.length; i++) {
+ var doc = docs[i];
+ for(var x in doc) {
+ var key = x.split("mime_")[1];
+ var value = doc[x];
+ if(typeof mime[key] === 'undefined') {
+ mime[key] = value;
+ }
+ else {
+ mime[key] += value;
+ }
+ }
+ }
+
+ for(x in mime) {
+ var jsonObject = {};
+ jsonObject["key"] = x;
+ jsonObject["y"] = mime[x];
+ resultingData.push(jsonObject);
+ }
+
+ resultingData.sort(function(a, b) {
+ return b.y - a.y;
+ });
+ if(rows > resultingData.length)rows=resultingData.length;
+ for( i = 1; i <= rows; i++) {
+ result[i-1] = resultingData[i-1];
+ }
+
+ console.log(result);
var svg = d3.select("#pietopmimesvg");
svg.selectAll("*").remove();
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
