This is an automated email from the ASF dual-hosted git repository. riemer pushed a commit to branch dev in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit b762769fbc64f1c4b45eba41fbc23de33807d794 Author: Dominik Riemer <[email protected]> AuthorDate: Thu Sep 24 22:45:41 2020 +0200 [STREAMPIPES-246] Remove obsolete module streampipes-app-file-export --- pom.xml | 1 - streampipes-app-file-export/pom.xml | 79 ----- .../app/file/export/ElasticsearchAppData.java | 83 ------ .../app/file/export/ElasticsearchConfig.java | 60 ---- .../app/file/export/api/IElasticsearch.java | 36 --- .../application/AppFileExportApplication.java | 30 -- .../app/file/export/converter/JsonConverter.java | 65 ----- .../app/file/export/impl/Elasticsearch.java | 320 --------------------- .../app/file/export/model/IndexInfo.java | 45 --- streampipes-backend/pom.xml | 5 - .../backend/StreamPipesBackendApplication.java | 9 - 11 files changed, 733 deletions(-) diff --git a/pom.xml b/pom.xml index 5853729..7ca3711 100644 --- a/pom.xml +++ b/pom.xml @@ -837,7 +837,6 @@ <module>archetypes/streampipes-archetype-pe-processors-jvm</module> <module>archetypes/streampipes-archetype-pe-sinks-flink</module> <module>archetypes/streampipes-archetype-pe-processors-flink</module> - <module>streampipes-app-file-export</module> <module>streampipes-backend</module> <module>streampipes-code-generation</module> <module>streampipes-commons</module> diff --git a/streampipes-app-file-export/pom.xml b/streampipes-app-file-export/pom.xml deleted file mode 100644 index 211640b..0000000 --- a/streampipes-app-file-export/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one or more - ~ contributor license agreements. See the NOTICE file distributed with - ~ this work for additional information regarding copyright ownership. - ~ The ASF licenses this file to You under the Apache License, Version 2.0 - ~ (the "License"); you may not use this file except in compliance with - ~ the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - ~ - --> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <artifactId>streampipes-parent</artifactId> - <groupId>org.apache.streampipes</groupId> - <version>0.68.0-SNAPSHOT</version> - </parent> - - <name>StreamPipes App File Export</name> - <artifactId>streampipes-app-file-export</artifactId> - <packaging>jar</packaging> - - <dependencies> - <!-- StreamPipes dependencies --> - <dependency> - <groupId>org.apache.streampipes</groupId> - <artifactId>streampipes-storage-couchdb</artifactId> - <version>0.68.0-SNAPSHOT</version> - </dependency> - - <!-- External dependencies --> - <dependency> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </dependency> - <dependency> - <groupId>javax.ws.rs</groupId> - <artifactId>javax.ws.rs-api</artifactId> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.containers</groupId> - <artifactId>jersey-container-servlet</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>fluent-hc</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - <dependency> - <groupId>org.elasticsearch.client</groupId> - <artifactId>elasticsearch-rest-high-level-client</artifactId> - </dependency> - <dependency> - <groupId>org.lightcouch</groupId> - <artifactId>lightcouch</artifactId> - </dependency> - - <!-- dependency convergence --> - <dependency> - <groupId>org.yaml</groupId> - <artifactId>snakeyaml</artifactId> - </dependency> - - <!-- Test dependencies --> - - </dependencies> -</project> \ No newline at end of file diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchAppData.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchAppData.java deleted file mode 100644 index 794f50b..0000000 --- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchAppData.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.app.file.export; - -public class ElasticsearchAppData { - - private String index; - - private long timestampFrom; - - private long timestampTo; - - private String output; - - private boolean allData; - - public ElasticsearchAppData() { - } - - public ElasticsearchAppData(String index, long timestampFrom, long timeStampTo, String output, boolean allData) { - this.index = index; - this.timestampFrom = timestampFrom; - this.timestampTo = timeStampTo; - this.output = output; - this.allData = allData; - } - - public String getIndex() { - return index; - } - - public void setIndex(String index) { - this.index = index; - } - - public long getTimestampFrom() { - return timestampFrom; - } - - public void setTimestampFrom(long timestampFrom) { - this.timestampFrom = timestampFrom; - } - - public long getTimestampTo() { - return timestampTo; - } - - public void setTimeStampTo(long timestampTo) { - this.timestampTo = timestampTo; - } - - public String getOutput() { - return output; - } - - public void setOutput(String output) { - this.output = output; - } - - public boolean isAllData() { - return allData; - } - - public void setAllData(boolean allData) { - this.allData = allData; - } -} diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchConfig.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchConfig.java deleted file mode 100644 index 0460e18..0000000 --- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchConfig.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.app.file.export; - -import org.apache.streampipes.config.SpConfig; - -public enum ElasticsearchConfig { - INSTANCE; - - private SpConfig config; - private final static String HOST = "host"; - private final static String PORT = "port"; - private final static String PROTOCOL = "protocol"; - private final static String DATA_LOCATION = "data_location"; - - ElasticsearchConfig() { - config = SpConfig.getSpConfig("storage/elasticsearch"); - - config.register(HOST, "elasticsearch", "Hostname for the elasticsearch service"); - config.register(PORT, 9200, "Port for the elasticsearch service"); - config.register(PROTOCOL, "http", "Protocol the elasticsearch service"); - config.register(DATA_LOCATION,"/home/user/", "Folder that stores all the created data blobs"); - } - - public String getElasticsearchHost() { - return config.getString(HOST); - } - - public Integer getElasticsearchPort() { - return config.getInteger(PORT); - } - - public String getElasticsearchURL() { - return getElasticsearchProtocol()+ "://" + getElasticsearchHost() + ":" + getElasticsearchPort(); - } - - public String getElasticsearchProtocol() { - return config.getString(PROTOCOL); - } - - public String getDataLocation() { - return config.getString(DATA_LOCATION); - } -} diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/api/IElasticsearch.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/api/IElasticsearch.java deleted file mode 100644 index c61f670..0000000 --- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/api/IElasticsearch.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.app.file.export.api; - -import org.apache.streampipes.app.file.export.ElasticsearchAppData; - -import javax.ws.rs.core.Response; - -public interface IElasticsearch { - - Response createFiles(ElasticsearchAppData data); - - Response getFile(String fileName); - - Response deleteFile(String fileName); - - Response getEndpoints(); - - -} diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/application/AppFileExportApplication.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/application/AppFileExportApplication.java deleted file mode 100644 index 3168892..0000000 --- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/application/AppFileExportApplication.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.app.file.export.application; - -import org.glassfish.jersey.server.ResourceConfig; -import org.apache.streampipes.app.file.export.impl.Elasticsearch; - - -public class AppFileExportApplication extends ResourceConfig { - - public AppFileExportApplication() { - register(Elasticsearch.class); - } -} diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/converter/JsonConverter.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/converter/JsonConverter.java deleted file mode 100644 index 5909d42..0000000 --- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/converter/JsonConverter.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.app.file.export.converter; - -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - -import java.util.Map; -import java.util.Set; -import java.util.StringJoiner; - -public class JsonConverter { - - private JsonObject elasticJsonRepresentation; - private JsonParser jsonParser; - - public JsonConverter() { - this.jsonParser = new JsonParser(); - } - - public String getCsvHeader(String elasticJsonRepresentation) { - JsonObject inContent = jsonParser.parse(elasticJsonRepresentation).getAsJsonObject(); - - Set<Map.Entry<String, JsonElement>> elements = inContent.entrySet(); - StringJoiner sj = new StringJoiner(";"); - - for (Map.Entry<String, JsonElement> entry : elements) { - sj.add(entry.getKey().toString()); - } - - return sj.toString() + "\n"; - - } - - public String convertToCsv(String elasticJsonRepresentation) { - JsonObject inContent = jsonParser.parse(elasticJsonRepresentation).getAsJsonObject(); - - Set<Map.Entry<String, JsonElement>> elements = inContent.entrySet(); - StringJoiner sj = new StringJoiner(";"); - - for (Map.Entry<String, JsonElement> entry : elements) { - sj.add(entry.getValue().toString()); - } - - return sj.toString() + "\n"; - - } - -} diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/impl/Elasticsearch.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/impl/Elasticsearch.java deleted file mode 100644 index c3fb684..0000000 --- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/impl/Elasticsearch.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.app.file.export.impl; - -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import org.apache.http.HttpHost; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.fluent.Request; -import org.elasticsearch.action.search.ClearScrollRequest; -import org.elasticsearch.action.search.ClearScrollResponse; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.Scroll; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.lightcouch.CouchDbClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.streampipes.app.file.export.ElasticsearchAppData; -import org.apache.streampipes.app.file.export.ElasticsearchConfig; -import org.apache.streampipes.app.file.export.api.IElasticsearch; -import org.apache.streampipes.app.file.export.converter.JsonConverter; -import org.apache.streampipes.app.file.export.model.IndexInfo; -import org.apache.streampipes.storage.couchdb.utils.Utils; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -@Path("/v1/elasticsearch") -public class Elasticsearch implements IElasticsearch { - - private static String mainFilePath = ElasticsearchConfig.INSTANCE.getDataLocation(); - private static final List<String> excludedIndices = Collections.singletonList(".kibana"); - - Logger LOG = LoggerFactory.getLogger(Elasticsearch.class); - - @POST - @Produces(MediaType.APPLICATION_JSON) - @Consumes(MediaType.APPLICATION_JSON) - @Path("/file") - @Override - public Response createFiles(ElasticsearchAppData data) { - String index = data.getIndex(); - long timestampFrom = data.getTimestampFrom(); - long timeStampTo = data.getTimestampTo(); - String output = data.getOutput(); - boolean allData = data.isAllData(); - - try { - RestHighLevelClient client = getRestHighLevelClient(); - - final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L)); - SearchRequest searchRequest = new SearchRequest(index); - searchRequest.scroll(scroll); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - - if (!allData) { - searchSourceBuilder.query(QueryBuilders.rangeQuery("timestamp").from(timestampFrom).to(timeStampTo)); - } - - searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - String scrollId = searchResponse.getScrollId(); - SearchHit[] searchHits = searchResponse.getHits().getHits(); - - //Time created in milli sec, index, from, to - long timestamp = System.currentTimeMillis(); - String fileName = System.currentTimeMillis() + "-" + index + "-" + timestampFrom + "-" + timeStampTo + "." + output; - String filePath = mainFilePath + fileName; - FileOutputStream fileStream = this.getFileStream(filePath); - - if(("csv").equals(output)) { - processCSV(client, fileStream, scrollId, scroll, searchHits); - } else { - processJSON(client, fileStream, scrollId, scroll, searchHits); - } - - fileStream.close(); - - CouchDbClient couchDbClient = getCouchDbClient(); - Map<String, Object> map = new HashMap<>(); - map.put("_id", fileName); - map.put("fileName", fileName); - map.put("filePath", filePath); - map.put("createAt", timestamp); - map.put("from", timestampFrom); - map.put("to", timeStampTo); - couchDbClient.save(map); - - LOG.info("Created file: " + fileName); - - return Response.ok().build(); - - } catch (IOException e) { - e.printStackTrace(); - LOG.error(e.getMessage()); - return Response.status(500).entity(e).build(); - } - } - - @GET - @Produces(MediaType.APPLICATION_JSON) - @Path("/file/{fileName}") - public Response getFile(@PathParam("fileName") String fileName) { - File file = new File(mainFilePath + fileName); - if (file.exists()) { - LOG.info("Downloaded file: " + fileName); - return Response.ok(file, MediaType.APPLICATION_OCTET_STREAM) - .header("Content-Disposition", "attachment; filename=\"" + fileName + "\"") - .build(); - } else { - LOG.info("Download - File not found"); - return Response.status(404).entity("File not found").build(); - } - } - - @GET - @Produces(MediaType.APPLICATION_JSON) - @Path("/indices") - public Response getIndices() { - String url = ElasticsearchConfig.INSTANCE.getElasticsearchURL() + "/_cat/indices?v"; - try { - JsonElement jsonResponse = get(url); - - JsonArray response = jsonResponse.getAsJsonArray(); - List<IndexInfo> availableIndices = new ArrayList<>(); - for(int i = 0; i < response.size(); i++) { - JsonObject object = response.get(i).getAsJsonObject(); - String index = object.get("index").getAsString(); - if (!shouldExclude(index)) { - Integer documentCount = Integer.parseInt(object.get("docs.count").getAsString()); - availableIndices.add(new IndexInfo(index, documentCount)); - } - } - return Response.ok(availableIndices).build(); - } catch (IOException e) { - e.printStackTrace(); - return Response.serverError().build(); - } - } - - private boolean shouldExclude(String index) { - return excludedIndices.stream().anyMatch(i -> i.equals(index)); - } - - @DELETE - @Path("/file/{fileName}") - @Override - public Response deleteFile(@PathParam("fileName") String fileName) { - CouchDbClient couchDbClient = getCouchDbClient(); - JsonObject found = couchDbClient.find(JsonObject.class, fileName); - couchDbClient.remove(found.get("_id").getAsString(), found.get("_rev").getAsString()); - File file = new File(mainFilePath + fileName); - file.delete(); - LOG.info("Deleted: " + fileName); - - return Response.ok().build(); - } - - @GET - @Path("/files") - @Override - public Response getEndpoints() { - CouchDbClient couchDbClient = getCouchDbClient(); - List<JsonObject> endpoints = couchDbClient.view("_all_docs").includeDocs(true).query(JsonObject.class); - String json = new Gson().toJson(endpoints); - - return Response.ok(json).build(); - } - - private CouchDbClient getCouchDbClient() { - return Utils.getCouchDbElasticsearchFilesEndppointClient(); - } - - private FileOutputStream getFileStream(String filePath) throws IOException { - File file = new File(filePath); - file.getParentFile().mkdirs(); - return new FileOutputStream(filePath); - } - - private JsonElement get(String url) throws IOException { - String jsonResponse = Request.Get(url) - .addHeader("accept", "application/json") - .addHeader("Content-Type", "application/json") - .execute() - .returnContent().asString(); - return new JsonParser().parse(jsonResponse); - } - - private RestHighLevelClient getRestHighLevelClient() { - String host = ElasticsearchConfig.INSTANCE.getElasticsearchHost(); - int port = ElasticsearchConfig.INSTANCE.getElasticsearchPort(); - - return new RestHighLevelClient( - RestClient.builder( - new HttpHost(host, port, "http")) - .setRequestConfigCallback( - new RestClientBuilder.RequestConfigCallback() { - @Override - public RequestConfig.Builder customizeRequestConfig( - RequestConfig.Builder requestConfigBuilder) { - return requestConfigBuilder - .setConnectTimeout(5000) - .setSocketTimeout(60000); - } - }) - ); - - } - - private void processJSON(RestHighLevelClient client, FileOutputStream fileStream, String scrollId, Scroll scroll, SearchHit[] searchHits) throws IOException { - fileStream.write("[".getBytes()); - boolean isFirstElement = true; - for (SearchHit hit : searchHits) { - if(!isFirstElement) - fileStream.write(",".getBytes()); - fileStream.write(hit.getSourceAsString().getBytes()); - isFirstElement = false; - } - - while (searchHits != null && searchHits.length > 0) { - - SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); - scrollRequest.scroll(scroll); - SearchResponse searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); - scrollId = searchResponse.getScrollId(); - searchHits = searchResponse.getHits().getHits(); - for (SearchHit hit : searchHits) { - fileStream.write(",".getBytes()); - fileStream.write(hit.getSourceAsString().getBytes()); - } - } - fileStream.write("]".getBytes()); - - ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); - clearScrollRequest.addScrollId(scrollId); - ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); - - - } - - private void processCSV(RestHighLevelClient client, FileOutputStream fileStream, String scrollId, Scroll scroll, - SearchHit[] searchHits) throws IOException { - JsonConverter jsonConverter = new JsonConverter(); - - boolean isFirstElement = true; - for (SearchHit hit : searchHits) { - if (isFirstElement) - fileStream.write(jsonConverter.getCsvHeader(hit.getSourceAsString()).getBytes()); - String response = jsonConverter.convertToCsv(hit.getSourceAsString()); - fileStream.write(response.getBytes()); - isFirstElement = false; - - } - - while (searchHits != null && searchHits.length > 0) { - - SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); - scrollRequest.scroll(scroll); - SearchResponse searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); - scrollId = searchResponse.getScrollId(); - searchHits = searchResponse.getHits().getHits(); - for (SearchHit hit : searchHits) { - fileStream.write(jsonConverter.convertToCsv(hit.getSourceAsString()).getBytes()); - } - - } - - ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); - clearScrollRequest.addScrollId(scrollId); - ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); - - } - - - - } diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/model/IndexInfo.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/model/IndexInfo.java deleted file mode 100644 index b23e82b..0000000 --- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/model/IndexInfo.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.app.file.export.model; - -public class IndexInfo { - - private String indexName; - private Integer documentCount; - - public IndexInfo(String indexName, Integer documentCount) { - this.indexName = indexName; - this.documentCount = documentCount; - } - - public String getIndexName() { - return indexName; - } - - public void setIndexName(String indexName) { - this.indexName = indexName; - } - - public Integer getDocumentCount() { - return documentCount; - } - - public void setDocumentCount(Integer documentCount) { - this.documentCount = documentCount; - } -} diff --git a/streampipes-backend/pom.xml b/streampipes-backend/pom.xml index e035bd5..7bd3afd 100644 --- a/streampipes-backend/pom.xml +++ b/streampipes-backend/pom.xml @@ -32,11 +32,6 @@ <!-- StreamPipes dependencies --> <dependency> <groupId>org.apache.streampipes</groupId> - <artifactId>streampipes-app-file-export</artifactId> - <version>0.68.0-SNAPSHOT</version> - </dependency> - <dependency> - <groupId>org.apache.streampipes</groupId> <artifactId>streampipes-container-base</artifactId> <version>0.68.0-SNAPSHOT</version> <exclusions> diff --git a/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesBackendApplication.java b/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesBackendApplication.java index 5c6ddbc..b94c9e1 100644 --- a/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesBackendApplication.java +++ b/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesBackendApplication.java @@ -20,18 +20,15 @@ package org.apache.streampipes.backend; import org.apache.shiro.web.env.EnvironmentLoaderListener; import org.apache.shiro.web.servlet.OncePerRequestFilter; import org.apache.shiro.web.servlet.ShiroFilter; -import org.apache.streampipes.app.file.export.application.AppFileExportApplication; import org.apache.streampipes.manager.operations.Operations; import org.apache.streampipes.model.pipeline.PipelineOperationStatus; import org.apache.streampipes.rest.notifications.NotificationListener; -import org.glassfish.jersey.servlet.ServletContainer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.boot.web.servlet.ServletListenerRegistrationBean; -import org.springframework.boot.web.servlet.ServletRegistrationBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; @@ -66,12 +63,6 @@ public class StreamPipesBackendApplication { } @Bean - public ServletRegistrationBean appFileExportRegistrationBean() { - ServletContainer jerseyContainer = new ServletContainer(new AppFileExportApplication()); - return new ServletRegistrationBean<>(jerseyContainer, "/api/apps/*"); - } - - @Bean public FilterRegistrationBean shiroFilterBean() { FilterRegistrationBean<OncePerRequestFilter> bean = new FilterRegistrationBean<>(); bean.setFilter(new ShiroFilter());
