Repository: incubator-pirk Updated Branches: refs/heads/master 3957be411 -> 53f548a87
Enhancements to Wideskies encrypt/decrypt -- closes apache/incubator-pirk#81 Project: http://git-wip-us.apache.org/repos/asf/incubator-pirk/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-pirk/commit/53f548a8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-pirk/tree/53f548a8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-pirk/diff/53f548a8 Branch: refs/heads/master Commit: 53f548a87b48015c820ab63c92d64840abf6f33b Parents: 3957be4 Author: tellison <telli...@apache.org> Authored: Sat Aug 27 11:25:19 2016 -0400 Committer: eawilliams <eawilli...@apache.org> Committed: Sat Aug 27 11:25:19 2016 -0400 ---------------------------------------------------------------------- .../apache/pirk/querier/wideskies/Querier.java | 8 +- .../pirk/querier/wideskies/QuerierDriver.java | 10 +- .../wideskies/decrypt/DecryptResponse.java | 60 ++------ .../wideskies/decrypt/DecryptResponseTask.java | 2 +- .../querier/wideskies/encrypt/EncryptQuery.java | 145 +++++++++---------- .../partitioner/PrimitiveTypePartitioner.java | 2 +- .../distributed/testsuite/DistTestSuite.java | 14 +- .../apache/pirk/test/utils/StandaloneQuery.java | 17 +-- .../org/apache/pirk/test/utils/TestUtils.java | 8 +- .../java/org/apache/pirk/utils/FileIOUtils.java | 24 +-- .../apache/pirk/utils/QueryResultsWriter.java | 73 ++++++++++ 11 files changed, 188 insertions(+), 175 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/53f548a8/src/main/java/org/apache/pirk/querier/wideskies/Querier.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/Querier.java b/src/main/java/org/apache/pirk/querier/wideskies/Querier.java index 172e1e5..b63e06e 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/Querier.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/Querier.java @@ -19,8 +19,8 @@ package org.apache.pirk.querier.wideskies; import java.io.Serializable; -import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.pirk.encryption.Paillier; import org.apache.pirk.query.wideskies.Query; @@ -42,9 +42,9 @@ public class Querier implements Serializable, Storable // map to check the embedded selectors in the results for false positives; // if the selector is a fixed size < 32 bits, it is included as is // if the selector is of variable lengths - private HashMap<Integer,String> embedSelectorMap = null; + private Map<Integer,String> embedSelectorMap = null; - public Querier(List<String> selectorsInput, Paillier paillierInput, Query queryInput, HashMap<Integer,String> embedSelectorMapInput) + public Querier(List<String> selectorsInput, Paillier paillierInput, Query queryInput, Map<Integer,String> embedSelectorMapInput) { selectors = selectorsInput; @@ -70,7 +70,7 @@ public class Querier implements Serializable, Storable return selectors; } - public HashMap<Integer,String> getEmbedSelectorMap() + public Map<Integer,String> getEmbedSelectorMap() { return embedSelectorMap; } http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/53f548a8/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java index 4f26a71..c1a4e4a 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java @@ -33,6 +33,7 @@ import org.apache.pirk.schema.query.QuerySchemaRegistry; import org.apache.pirk.serialization.LocalFileSystemStore; import org.apache.pirk.utils.FileIOUtils; import org.apache.pirk.utils.PIRException; +import org.apache.pirk.utils.QueryResultsWriter; import org.apache.pirk.utils.SystemConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -188,13 +189,13 @@ public class QuerierDriver implements Serializable // Perform the encryption EncryptQuery encryptQuery = new EncryptQuery(queryInfo, selectors, paillier); - encryptQuery.encrypt(numThreads); + Querier querier = encryptQuery.encrypt(numThreads); // Write necessary output files - two files written - // (1) Querier object to <outputFile>-QuerierConst.QUERIER_FILETAG // (2) Query object to <outputFile>-QuerierConst.QUERY_FILETAG - storage.store(outputFile + "-" + QuerierConst.QUERIER_FILETAG, encryptQuery.getQuerier()); - storage.store(outputFile + "-" + QuerierConst.QUERY_FILETAG, encryptQuery.getQuery()); + storage.store(outputFile + "-" + QuerierConst.QUERIER_FILETAG, querier); + storage.store(outputFile + "-" + QuerierConst.QUERY_FILETAG, querier.getQuery()); } else // Decryption @@ -214,8 +215,7 @@ public class QuerierDriver implements Serializable // Perform decryption and output the result file DecryptResponse decryptResponse = new DecryptResponse(response, querier); - decryptResponse.decrypt(numThreads); - decryptResponse.writeResultFile(outputFile); + QueryResultsWriter.writeResultFile(outputFile, decryptResponse.decrypt(numThreads)); } } } http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/53f548a8/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java b/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java index dbf2381..eae288f 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java @@ -18,16 +18,11 @@ */ package org.apache.pirk.querier.wideskies.decrypt; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.TreeMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -42,6 +37,7 @@ import org.apache.pirk.query.wideskies.QueryInfo; import org.apache.pirk.response.wideskies.Response; import org.apache.pirk.schema.response.QueryResponseJSON; import org.apache.pirk.utils.PIRException; +import org.apache.pirk.utils.SystemConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,14 +54,18 @@ public class DecryptResponse private final Querier querier; - private final Map<String,List<QueryResponseJSON>> resultMap = new HashMap<>(); // selector -> ArrayList of hits - public DecryptResponse(Response responseInput, Querier querierInput) { response = responseInput; querier = querierInput; } + public Map<String,List<QueryResponseJSON>> decrypt() throws InterruptedException, PIRException + { + int numThreads = SystemConfiguration.getIntProperty("numThreads", 1); + return decrypt(numThreads); + } + /** * Method to decrypt the response elements and reconstructs the data elements * <p> @@ -86,8 +86,10 @@ public class DecryptResponse * where D^k_r,l = Y_{r*numPartitionsPerDataElement + l} & (2^{r*numPartitionsPerDataElement} * (2^numBitsPerDataElement - 1)) * */ - public void decrypt(int numThreads) throws InterruptedException, PIRException + public Map<String,List<QueryResponseJSON>> decrypt(int numThreads) throws InterruptedException, PIRException { + Map<String,List<QueryResponseJSON>> resultMap = new HashMap<>(); // selector -> ArrayList of hits + QueryInfo queryInfo = response.getQueryInfo(); Paillier paillier = querier.getPaillier(); @@ -159,8 +161,10 @@ public class DecryptResponse } es.shutdown(); + + return resultMap; } - + // Method to perform basic decryption of each raw response element - does not // extract and reconstruct the data elements private List<BigInteger> decryptElements(TreeMap<Integer,BigInteger> elements, Paillier paillier) @@ -174,42 +178,4 @@ public class DecryptResponse return decryptedElements; } - - /** - * Writes elements of the resultMap to output file, one line for each element, where each line is a string representation of the corresponding - * QueryResponseJSON object - */ - public void writeResultFile(String filename) throws IOException - { - try (BufferedWriter bw = new BufferedWriter(new FileWriter(new File(filename)))) - { - for (Entry<String,List<QueryResponseJSON>> entry : resultMap.entrySet()) - { - for (QueryResponseJSON hitJSON : entry.getValue()) - { - bw.write(hitJSON.getJSONString()); - bw.newLine(); - } - } - } - } - - /** - * Writes elements of the resultMap to output file, one line for each element, where each line is a string representation of the corresponding - * QueryResponseJSON object - */ - public void writeResultFile(File file) throws IOException - { - try (BufferedWriter bw = new BufferedWriter(new FileWriter(file))) - { - for (Entry<String,List<QueryResponseJSON>> entry : resultMap.entrySet()) - { - for (QueryResponseJSON hitJSON : entry.getValue()) - { - bw.write(hitJSON.getJSONString()); - bw.newLine(); - } - } - } - } } http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/53f548a8/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponseTask.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponseTask.java b/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponseTask.java index 3f5a982..ab02094 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponseTask.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponseTask.java @@ -73,7 +73,7 @@ class DecryptResponseTask<V> implements Callable<Map<String,List<QueryResponseJS String selectorName = qSchema.getSelectorName(); // Result is a map of (selector -> List of hits). - Map<String,List<QueryResponseJSON>> resultMap = new HashMap<>(); + Map<String,List<QueryResponseJSON>> resultMap = new HashMap<>(selectors.size()); for (String selector : selectors.values()) { resultMap.put(selector, new ArrayList<QueryResponseJSON>()); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/53f548a8/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java b/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java index 4dd90f7..60ba859 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java @@ -56,103 +56,90 @@ public class EncryptQuery { private static final Logger logger = LoggerFactory.getLogger(EncryptQuery.class); - private final QueryInfo queryInfo; // contains basic query information and functionality + // Contains basic query information. + private final QueryInfo queryInfo; - private Query query = null; // contains the query vectors + // Selectors for this query. + private final List<String> selectors; - private Querier querier = null; // contains the query vectors and encryption object + // Paillier encryption functionality. + private final Paillier paillier; - private final Paillier paillier; // Paillier encryption functionality - - private List<String> selectors = null; // selectors for the query - - // Map to check the embedded selectors in the results for false positives; - // if the selector is a fixed size < 32 bits, it is included as is - // if the selector is of variable lengths - private HashMap<Integer,String> embedSelectorMap = null; - - public EncryptQuery(QueryInfo queryInfoInput, List<String> selectorsInput, Paillier paillierInput) - { - queryInfo = queryInfoInput; - - selectors = selectorsInput; - - paillier = paillierInput; - - embedSelectorMap = new HashMap<>(); - } - - public Paillier getPaillier() - { - return paillier; - } - - public QueryInfo getQueryInfo() - { - return queryInfo; - } - - public Query getQuery() - { - return query; - } - - public Querier getQuerier() - { - return querier; - } - - public List<String> getSelectors() - { - return selectors; - } - - public HashMap<Integer,String> getEmbedSelectorMap() + /** + * Constructs a query encryptor using the given query information, selectors, and Paillier cryptosystem. + * + * @param queryInfo + * Fundamental information about the query. + * @param selectors + * the list of selectors for this query. + * @param paillier + * the Paillier cryptosystem to use. + */ + public EncryptQuery(QueryInfo queryInfo, List<String> selectors, Paillier paillier) { - return embedSelectorMap; + this.queryInfo = queryInfo; + this.selectors = selectors; + this.paillier = paillier; } /** - * Encrypt, building the Query object, calculating and setting the query vectors. + * Encrypts the query described by the query information using Paillier encryption. + * <p> + * The encryption builds a <code>Querier</code> object, calculating and setting the query vectors. * <p> * Uses the system configured number of threads to conduct the encryption, or a single thread if the configuration has not been set. * * @throws InterruptedException - * if the task was interrupted during encryption. + * If the task was interrupted during encryption. * @throws PIRException + * If a problem occurs performing the encryption. + * @return The querier containing the query, and all information required to perform decryption. */ - public void encrypt() throws InterruptedException, PIRException + public Querier encrypt() throws InterruptedException, PIRException { int numThreads = SystemConfiguration.getIntProperty("numThreads", 1); - encrypt(numThreads); + return encrypt(numThreads); } /** - * Encrypt, building the Query object, calculating and setting the query vectors + * Encrypts the query described by the query information using Paillier encryption using the given number of threads. + * <p> + * The encryption builds a <code>Querier</code> object, calculating and setting the query vectors. * <p> * If we have hash collisions over our selector set, we will append integers to the key starting with 0 until we no longer have collisions. * <p> * For encrypted query vector E = <E_0, ..., E_{(2^hashBitSize)-1}>: * <p> * E_i = 2^{j*dataPartitionBitSize} if i = H_k(selector_j) 0 otherwise + * + * @param numThreads + * the number of threads to use when performing the encryption. + * @throws InterruptedException + * If the task was interrupted during encryption. + * @throws PIRException + * If a problem occurs performing the encryption. + * @return The querier containing the query, and all information required to perform decryption. */ - public void encrypt(int numThreads) throws InterruptedException, PIRException + public Querier encrypt(int numThreads) throws InterruptedException, PIRException { - query = new Query(queryInfo, paillier.getN()); + Query query = new Query(queryInfo, paillier.getN()); // Determine the query vector mappings for the selectors; vecPosition -> selectorNum Map<Integer,Integer> selectorQueryVecMapping = computeSelectorQueryVecMap(); // Form the embedSelectorMap - populateEmbeddedSelectorMap(); + // Map to check the embedded selectors in the results for false positives; + // if the selector is a fixed size < 32 bits, it is included as is + // if the selector is of variable lengths + Map<Integer,String> embedSelectorMap = computeEmbeddedSelectorMap(); if (numThreads == 1) { - serialEncrypt(selectorQueryVecMapping); + serialEncrypt(query, selectorQueryVecMapping); } else { - parallelEncrypt(Math.max(2, numThreads), selectorQueryVecMapping); + parallelEncrypt(Math.max(2, numThreads), query, selectorQueryVecMapping); } // Generate the expTable in Query, if we are using it and if @@ -163,8 +150,8 @@ public class EncryptQuery query.generateExpTable(); } - // Set the Querier object - querier = new Querier(selectors, paillier, query, embedSelectorMap); + // Return the Querier object. + return new Querier(selectors, paillier, query, embedSelectorMap); } private Map<Integer,Integer> computeSelectorQueryVecMap() @@ -200,31 +187,30 @@ public class EncryptQuery return selectorQueryVecMapping; } - private void populateEmbeddedSelectorMap() + private Map<Integer,String> computeEmbeddedSelectorMap() throws PIRException { QuerySchema qSchema = QuerySchemaRegistry.get(queryInfo.getQueryType()); + String selectorName = qSchema.getSelectorName(); DataSchema dSchema = DataSchemaRegistry.get(qSchema.getDataSchemaName()); - String type = dSchema.getElementType(qSchema.getSelectorName()); + String type = dSchema.getElementType(selectorName); + + Map<Integer,String> embedSelectorMap = new HashMap<>(selectors.size()); + int sNum = 0; for (String selector : selectors) { - String embeddedSelector = null; - try - { - embeddedSelector = QueryUtils.getEmbeddedSelector(selector, type, dSchema.getPartitionerForElement(qSchema.getSelectorName())); - } catch (Exception e) - { - logger.info("Caught exception for selector = " + selector); - e.printStackTrace(); - // TODO: Check: should continue? - } - + String embeddedSelector = QueryUtils.getEmbeddedSelector(selector, type, dSchema.getPartitionerForElement(selectorName)); embedSelectorMap.put(sNum, embeddedSelector); - ++sNum; + sNum += 1; } + + return embedSelectorMap; } - private void serialEncrypt(Map<Integer,Integer> selectorQueryVecMapping) throws PIRException + /* + * Perform the encryption using a single thread, and avoiding the overhead of thread management. + */ + private void serialEncrypt(Query query, Map<Integer,Integer> selectorQueryVecMapping) throws PIRException { int numElements = 1 << queryInfo.getHashBitSize(); // 2^hashBitSize @@ -234,7 +220,10 @@ public class EncryptQuery logger.info("Completed serial creation of encrypted query vectors"); } - private void parallelEncrypt(int numThreads, Map<Integer,Integer> selectorQueryVecMapping) throws InterruptedException, PIRException + /* + * Performs the encryption with numThreads. + */ + private void parallelEncrypt(int numThreads, Query query, Map<Integer,Integer> selectorQueryVecMapping) throws InterruptedException, PIRException { // Encrypt and form the query vector ExecutorService es = Executors.newCachedThreadPool(); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/53f548a8/src/main/java/org/apache/pirk/schema/data/partitioner/PrimitiveTypePartitioner.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/schema/data/partitioner/PrimitiveTypePartitioner.java b/src/main/java/org/apache/pirk/schema/data/partitioner/PrimitiveTypePartitioner.java index 7559441..d222ec0 100644 --- a/src/main/java/org/apache/pirk/schema/data/partitioner/PrimitiveTypePartitioner.java +++ b/src/main/java/org/apache/pirk/schema/data/partitioner/PrimitiveTypePartitioner.java @@ -32,7 +32,7 @@ import org.slf4j.LoggerFactory; * Class for partitioning objects with primitive Java types * */ -public class PrimitiveTypePartitioner implements DataPartitioner +public final class PrimitiveTypePartitioner implements DataPartitioner { private static final long serialVersionUID = 1L; http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/53f548a8/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java b/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java index f44815a..d02bb87 100644 --- a/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java +++ b/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java @@ -42,6 +42,7 @@ import org.apache.pirk.test.distributed.DistributedTestDriver; import org.apache.pirk.test.utils.BaseTests; import org.apache.pirk.test.utils.Inputs; import org.apache.pirk.test.utils.TestUtils; +import org.apache.pirk.utils.QueryResultsWriter; import org.apache.pirk.utils.SystemConfiguration; import org.apache.spark.launcher.SparkLauncher; import org.json.simple.JSONObject; @@ -434,16 +435,12 @@ public class DistTestSuite // Perform the encryption logger.info("Performing encryption of the selectors - forming encrypted query vectors:"); EncryptQuery encryptQuery = new EncryptQuery(queryInfo, selectors, paillier); - encryptQuery.encrypt(numThreads); + Querier querier = encryptQuery.encrypt(numThreads); logger.info("Completed encryption of the selectors - completed formation of the encrypted query vectors:"); - // Grab the necessary objects - Querier querier = encryptQuery.getQuerier(); - Query query = encryptQuery.getQuery(); - - // Write the Querier object to a file + // Write the Query object to a file Path queryInputDirPath = new Path(queryInputDir); - new HadoopFileSystemStore(fs).store(queryInputDirPath, query); + new HadoopFileSystemStore(fs).store(queryInputDirPath, querier.getQuery()); fs.deleteOnExit(queryInputDirPath); // Grab the original data and query schema properties to reset upon completion @@ -542,8 +539,7 @@ public class DistTestSuite // Perform decryption and output the result file DecryptResponse decryptResponse = new DecryptResponse(response, querier); - decryptResponse.decrypt(numThreads); - decryptResponse.writeResultFile(fileFinalResults); + QueryResultsWriter.writeResultFile(fileFinalResults, decryptResponse.decrypt(numThreads)); logger.info("Completed performing decryption and writing final results file"); // Read in results http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/53f548a8/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java b/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java index afac2b0..b86d78c 100644 --- a/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java +++ b/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java @@ -22,8 +22,8 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; -import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.pirk.encryption.Paillier; import org.apache.pirk.querier.wideskies.Querier; @@ -40,6 +40,7 @@ import org.apache.pirk.schema.query.QuerySchemaRegistry; import org.apache.pirk.schema.response.QueryResponseJSON; import org.apache.pirk.serialization.LocalFileSystemStore; import org.apache.pirk.utils.PIRException; +import org.apache.pirk.utils.QueryResultsWriter; import org.apache.pirk.utils.SystemConfiguration; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -94,22 +95,21 @@ public class StandaloneQuery // Perform the encryption logger.info("Performing encryption of the selectors - forming encrypted query vectors:"); EncryptQuery encryptQuery = new EncryptQuery(queryInfo, selectors, paillier); - encryptQuery.encrypt(numThreads); + Querier querier = encryptQuery.encrypt(numThreads); logger.info("Completed encryption of the selectors - completed formation of the encrypted query vectors:"); // Dork with the embedSelectorMap to generate a false positive for the last valid selector in selectors if (testFalsePositive) { - Querier querier = encryptQuery.getQuerier(); - HashMap<Integer,String> embedSelectorMap = querier.getEmbedSelectorMap(); + Map<Integer,String> embedSelectorMap = querier.getEmbedSelectorMap(); logger.info("embedSelectorMap((embedSelectorMap.size()-2)) = " + embedSelectorMap.get((embedSelectorMap.size() - 2)) + " selector = " + selectors.get((embedSelectorMap.size() - 2))); embedSelectorMap.put((embedSelectorMap.size() - 2), "fakeEmbeddedSelector"); } // Write necessary output files - storage.store(fileQuerier, encryptQuery.getQuerier()); - storage.store(fileQuery, encryptQuery.getQuery()); + storage.store(fileQuerier, querier); + storage.store(fileQuery, querier.getQuery()); // Perform the PIR query and build the response elements logger.info("Performing the PIR Query and constructing the response elements:"); @@ -141,12 +141,11 @@ public class StandaloneQuery // Reconstruct the necessary objects from the files logger.info("Performing decryption; writing final results file"); Response responseIn = storage.recall(fileResponse, Response.class); - Querier querier = storage.recall(fileQuerier, Querier.class); + querier = storage.recall(fileQuerier, Querier.class); // Perform decryption and output the result file DecryptResponse decryptResponse = new DecryptResponse(responseIn, querier); - decryptResponse.decrypt(numThreads); - decryptResponse.writeResultFile(fileFinalResults); + QueryResultsWriter.writeResultFile(fileFinalResults, decryptResponse.decrypt(numThreads)); logger.info("Completed performing decryption and writing final results file"); // Read in results http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/53f548a8/src/main/java/org/apache/pirk/test/utils/TestUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/utils/TestUtils.java b/src/main/java/org/apache/pirk/test/utils/TestUtils.java index 1ea01fb..6573a12 100644 --- a/src/main/java/org/apache/pirk/test/utils/TestUtils.java +++ b/src/main/java/org/apache/pirk/test/utils/TestUtils.java @@ -21,6 +21,7 @@ package org.apache.pirk.test.utils; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; @@ -267,8 +268,10 @@ public class TestUtils /** * Converts the result file into an ArrayList of QueryResponseJSON objects + * @throws IOException + * @throws FileNotFoundException */ - public static List<QueryResponseJSON> readResultsFile(File file) + public static List<QueryResponseJSON> readResultsFile(File file) throws FileNotFoundException, IOException { List<QueryResponseJSON> results = new ArrayList<>(); try (BufferedReader br = new BufferedReader(new FileReader(file))) @@ -279,9 +282,6 @@ public class TestUtils QueryResponseJSON jsonResult = new QueryResponseJSON(line); results.add(jsonResult); } - } catch (Exception e) - { - logger.error(e.toString()); } return results; http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/53f548a8/src/main/java/org/apache/pirk/utils/FileIOUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/utils/FileIOUtils.java b/src/main/java/org/apache/pirk/utils/FileIOUtils.java index 8924a86..9fd0dec 100644 --- a/src/main/java/org/apache/pirk/utils/FileIOUtils.java +++ b/src/main/java/org/apache/pirk/utils/FileIOUtils.java @@ -119,28 +119,18 @@ public class FileIOUtils public static void writeArrayList(ArrayList<String> aList, File file) throws IOException { - FileOutputStream fos = new FileOutputStream(file); - BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fos)); - - for (String s : aList) + try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file)))) { - bw.write(s); - bw.newLine(); + for (String s : aList) + { + bw.write(s); + bw.newLine(); + } } - bw.close(); } public static void writeArrayList(ArrayList<String> aList, String filename) throws IOException { - File file = new File(filename); - FileOutputStream fos = new FileOutputStream(file); - BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fos)); - - for (String s : aList) - { - bw.write(s); - bw.newLine(); - } - bw.close(); + writeArrayList(aList, new File(filename)); } } http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/53f548a8/src/main/java/org/apache/pirk/utils/QueryResultsWriter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/utils/QueryResultsWriter.java b/src/main/java/org/apache/pirk/utils/QueryResultsWriter.java new file mode 100644 index 0000000..65470e4 --- /dev/null +++ b/src/main/java/org/apache/pirk/utils/QueryResultsWriter.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pirk.utils; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.pirk.schema.response.QueryResponseJSON; + +public class QueryResultsWriter +{ + + + /** + * Writes elements of the resultMap to output file, one line for each element, where each line is a string representation of the corresponding + * QueryResponseJSON object + */ + public static void writeResultFile(String filename, Map<String,List<QueryResponseJSON>> resultMap) throws IOException + { + try (BufferedWriter bw = new BufferedWriter(new FileWriter(new File(filename)))) + { + for (Entry<String,List<QueryResponseJSON>> entry : resultMap.entrySet()) + { + for (QueryResponseJSON hitJSON : entry.getValue()) + { + bw.write(hitJSON.getJSONString()); + bw.newLine(); + } + } + } + } + + /** + * Writes elements of the resultMap to output file, one line for each element, where each line is a string representation of the corresponding + * QueryResponseJSON object + */ + public static void writeResultFile(File file, Map<String,List<QueryResponseJSON>> resultMap) throws IOException + { + try (BufferedWriter bw = new BufferedWriter(new FileWriter(file))) + { + for (Entry<String,List<QueryResponseJSON>> entry : resultMap.entrySet()) + { + for (QueryResponseJSON hitJSON : entry.getValue()) + { + bw.write(hitJSON.getJSONString()); + bw.newLine(); + } + } + } + } + +}
