Repository: systemml Updated Branches: refs/heads/master 3c7a6eb70 -> 139ddfeed
http://git-wip-us.apache.org/repos/asf/systemml/blob/139ddfee/src/assembly/lite/LICENSE ---------------------------------------------------------------------- diff --git a/src/assembly/lite/LICENSE b/src/assembly/lite/LICENSE new file mode 100644 index 0000000..9c3793f --- /dev/null +++ b/src/assembly/lite/LICENSE @@ -0,0 +1,293 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +=============================================================================== + +The following included dependencies come under the Apache Software License 2.0. + +avro-1.7.4.jar +commons-cli-1.2.jar +commons-collections-3.2.1.jar +commons-compress-1.4.1.jar +commons-configuration-1.6.jar +commons-io-2.4.jar +commons-lang-2.6.jar +commons-lang3-3.5.jar +commons-math3-3.4.1.jar +guava-11.0.2.jar +hadoop-annotations-2.6.0.jar +hadoop-auth-2.6.0.jar +hadoop-common-2.6.0.jar +hadoop-hdfs-2.6.0.jar +hadoop-mapreduce-client-common-2.6.0.jar +hadoop-mapreduce-client-core-2.6.0.jar +jcl-over-slf4j-1.7.16.jar +log4j-1.2.15.jar +wink-json4j-1.4.jar +xercesImpl-2.9.1.jar + +=============================================================================== + +The following ANTLR dependency is distributed under the BSD license: +antlr4-runtime-4.5.3.jar + +Copyright (c) 2012 Terence Parr and Sam Harwell +All rights reserved. + +BSD license: + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + - Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + - Neither the name of the author nor the names of + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=============================================================================== + +The following SLF4J dependencies are distributed under the MIT license: +slf4j-api-1.7.5.jar +slf4j-log4j12-1.7.5.jar + +Copyright (c) 2004-2007 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/139ddfee/src/assembly/lite/NOTICE ---------------------------------------------------------------------- diff --git a/src/assembly/lite/NOTICE b/src/assembly/lite/NOTICE new file mode 100644 index 0000000..23d11e4 --- /dev/null +++ b/src/assembly/lite/NOTICE @@ -0,0 +1,5 @@ +Apache SystemML +Copyright [2015-2017] The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). http://git-wip-us.apache.org/repos/asf/systemml/blob/139ddfee/src/assembly/lite/log4j.properties ---------------------------------------------------------------------- diff --git a/src/assembly/lite/log4j.properties b/src/assembly/lite/log4j.properties new file mode 100644 index 0000000..250b7e9 --- /dev/null +++ b/src/assembly/lite/log4j.properties @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootCategory=WARN, console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n + http://git-wip-us.apache.org/repos/asf/systemml/blob/139ddfee/src/assembly/lite/systemml-lite.txt ---------------------------------------------------------------------- diff --git a/src/assembly/lite/systemml-lite.txt b/src/assembly/lite/systemml-lite.txt new file mode 100644 index 0000000..9d8191b --- /dev/null +++ b/src/assembly/lite/systemml-lite.txt @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +Identifier file for lite jar file. http://git-wip-us.apache.org/repos/asf/systemml/blob/139ddfee/src/main/java/org/apache/sysml/conf/ConfigurationManager.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/conf/ConfigurationManager.java b/src/main/java/org/apache/sysml/conf/ConfigurationManager.java index 903466a..27ab8b4 100644 --- a/src/main/java/org/apache/sysml/conf/ConfigurationManager.java +++ b/src/main/java/org/apache/sysml/conf/ConfigurationManager.java @@ -21,6 +21,9 @@ package org.apache.sysml.conf; import org.apache.hadoop.mapred.JobConf; import org.apache.sysml.conf.CompilerConfig.ConfigType; +import org.apache.sysml.runtime.matrix.mapred.MRConfigurationNames; +import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration; +import org.apache.sysml.utils.lite.LiteCheck; @@ -55,6 +58,14 @@ public class ConfigurationManager //ConfigManager -> OptimizerUtils -> InfrastructureAnalyer -> ConfigManager _dmlconf = new DMLConfig(); _cconf = new CompilerConfig(); + + if (LiteCheck.isLite() && MRJobConfiguration.USE_BINARYBLOCK_SERIALIZATION) { + // to be able to write using binary format + // WritableSerialization -> MatrixIndexes + // BinaryBlockSerialization -> MatrixBlock + _rJob.set(MRConfigurationNames.IO_SERIALIZATIONS, + "org.apache.hadoop.io.serializer.WritableSerialization,org.apache.sysml.runtime.io.BinaryBlockSerialization"); + } } http://git-wip-us.apache.org/repos/asf/systemml/blob/139ddfee/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java index d04d69c..1f1fce2 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/stat/InfrastructureAnalyzer.java @@ -30,6 +30,7 @@ import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext; import org.apache.sysml.runtime.matrix.mapred.MRConfigurationNames; import org.apache.sysml.runtime.util.UtilFunctions; +import org.apache.sysml.utils.lite.LiteCheck; /** * Central place for analyzing and obtaining static infrastructure properties @@ -93,24 +94,20 @@ public class InfrastructureAnalyzer * * @return number of cluster nodes */ - public static int getRemoteParallelNodes() - { - if( _remotePar == -1 ) + public static int getRemoteParallelNodes() { + if ((_remotePar == -1) && (!LiteCheck.isLite())) analyzeHadoopCluster(); - return _remotePar; - } - + } + /** * Gets the number of remote parallel map slots. * * @return number of remote parallel map tasks */ - public static int getRemoteParallelMapTasks() - { - if( _remoteParMap == -1 ) + public static int getRemoteParallelMapTasks() { + if ((_remoteParMap == -1) && (!LiteCheck.isLite())) analyzeHadoopCluster(); - return _remoteParMap; } @@ -124,11 +121,9 @@ public class InfrastructureAnalyzer * * @return number of remote parallel reduce tasks */ - public static int getRemoteParallelReduceTasks() - { - if( _remoteParReduce == -1 ) + public static int getRemoteParallelReduceTasks() { + if ((_remoteParReduce == -1) && (!LiteCheck.isLite())) analyzeHadoopCluster(); - return _remoteParReduce; } @@ -160,11 +155,9 @@ public class InfrastructureAnalyzer * * @return maximum memory of remote hadoop map task jvm */ - public static long getRemoteMaxMemoryMap() - { - if( _remoteJVMMaxMemMap == -1 ) + public static long getRemoteMaxMemoryMap() { + if ((_remoteJVMMaxMemMap == -1) && (!LiteCheck.isLite())) analyzeHadoopConfiguration(); - return _remoteJVMMaxMemMap; } @@ -178,11 +171,9 @@ public class InfrastructureAnalyzer * * @return maximum memory of remote hadoop reduce task jvm */ - public static long getRemoteMaxMemoryReduce() - { - if( _remoteJVMMaxMemReduce == -1 ) + public static long getRemoteMaxMemoryReduce() { + if ((_remoteJVMMaxMemReduce == -1) && (!LiteCheck.isLite())) analyzeHadoopConfiguration(); - return _remoteJVMMaxMemReduce; } @@ -196,22 +187,18 @@ public class InfrastructureAnalyzer * * @return maximum sort buffer memory of hadoop task */ - public static long getRemoteMaxMemorySortBuffer( ) - { - if( _remoteMRSortMem == -1 ) + public static long getRemoteMaxMemorySortBuffer() { + if ((_remoteMRSortMem == -1) && (!LiteCheck.isLite())) analyzeHadoopConfiguration(); - - return _remoteMRSortMem; + return _remoteMRSortMem; } - - public static boolean isLocalMode() - { - if( _remoteJVMMaxMemMap == -1 ) + + public static boolean isLocalMode() { + if ((_remoteJVMMaxMemMap == -1) && (!LiteCheck.isLite())) analyzeHadoopConfiguration(); - - return _localJT; + return _localJT; } - + public static boolean isLocalMode(JobConf job) { // Due to a bug in HDP related to fetching the "mode" of execution within mappers, @@ -267,19 +254,15 @@ public class InfrastructureAnalyzer * * @return HDFS block size */ - public static long getHDFSBlockSize() - { - if( _blocksize == -1 ) + public static long getHDFSBlockSize() { + if ((_blocksize == -1) && (!LiteCheck.isLite())) analyzeHadoopConfiguration(); - - return _blocksize; + return _blocksize; } - public static boolean isYarnEnabled() - { - if( _remoteJVMMaxMemMap == -1 ) + public static boolean isYarnEnabled() { + if ((_remoteJVMMaxMemMap == -1) && (!LiteCheck.isLite())) analyzeHadoopConfiguration(); - return _yarnEnabled; } http://git-wip-us.apache.org/repos/asf/systemml/blob/139ddfee/src/main/java/org/apache/sysml/utils/lite/BuildLite.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/lite/BuildLite.java b/src/main/java/org/apache/sysml/utils/lite/BuildLite.java new file mode 100644 index 0000000..9b7af29 --- /dev/null +++ b/src/main/java/org/apache/sysml/utils/lite/BuildLite.java @@ -0,0 +1,822 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sysml.utils.lite; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.Vector; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.compress.archivers.jar.JarArchiveEntry; +import org.apache.commons.compress.archivers.jar.JarArchiveInputStream; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.math3.random.Well1024a; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.ThrowableInformation; + +/** + * Builds a light-weight SystemML jar file based on loaded classes and + * additional resources. Additionally generates maven assembly dependency sets + * that are used by the lite.xml assembly. Note that the jar file automatically + * built by createLiteJar will only contain required SystemML classes, whereas + * the assembly jar file (built by lite.xml) includes all SystemML classes. All + * log4j classes are included in both the automatic jar and the assembly jar. + * All commons-math3 classes are included by default in both the automatic jar + * and the assembly jar, but this can be switched using createLiteJar to only + * include the detected required commons-math3 classes. + * + */ +public class BuildLite { + + /** + * Default lite jar path and name. + */ + public static final String DEFAULT_LITE_JAR_LOCATION = "systemml-lite.jar"; + + /** + * File within the lite jar that can be used to identify execution from the + * lite jar. + */ + public static final String LITE_JAR_IDENTIFIER_FILE = "META-INF/systemml-lite.txt"; + + /** + * The lite jar path and name. + */ + private static String liteJarLocation = DEFAULT_LITE_JAR_LOCATION; + + /** + * Additional resources that should be added to the lite jar file. This can + * include resources such as service files and shutdown hooks that aren't + * detected by query to the classloader. + */ + public static List<String> additionalResources = new ArrayList<String>(); + static { + // avoid "No FileSystem for scheme: file" error in JMLC + additionalResources.add("META-INF/services/org.apache.hadoop.fs.FileSystem"); + // shutdown hook class + additionalResources.add("org/apache/hadoop/util/ShutdownHookManager$2.class"); + + additionalResources.add("org/apache/hadoop/log/metrics/EventCounter.class"); + }; + + /** + * Map jars to the additional resources files in order to build the + * dependency sets required by lite.xml. + */ + public static SortedMap<String, SortedSet<String>> additionalJarToFileMappingsForDependencySets = new TreeMap<String, SortedSet<String>>(); + static { + SortedSet<String> hadoopCommonResources = new TreeSet<String>(); + hadoopCommonResources.add("META-INF/services/org.apache.hadoop.fs.FileSystem"); + hadoopCommonResources.add("org/apache/hadoop/util/ShutdownHookManager$2.class"); + hadoopCommonResources.add("org/apache/hadoop/log/metrics/EventCounter.class"); + additionalJarToFileMappingsForDependencySets.put("hadoop-common", hadoopCommonResources); + } + + /** + * Scan project *.java files for these packages/classes that should + * definitely be included in the lite jar. + */ + public static List<String> additionalPackages = new ArrayList<String>(); + static { + // math3, lang3, io, etc. + additionalPackages.add("org.apache.commons"); + } + + /** + * Exclude classes of the following packages from the lite jar. + */ + public static List<String> packagesToExclude = new ArrayList<String>(); + static { + packagesToExclude.add("com.sun.proxy"); + // these can be added if test suite code is run + packagesToExclude.add("org.junit"); + packagesToExclude.add("org.apache.spark"); + packagesToExclude.add("org.apache.sysml.test"); + packagesToExclude.add("scala"); + } + + /** + * The base source directory to scan for classes to potentially load. + */ + private static final String BASE_SRC_DIR = "src/main"; + + /** + * The detected classes and their jar files, where the jar names are keys in + * a sorted map and sorted sets of the class names are the values in the + * sorted map. + */ + public static SortedMap<String, SortedSet<String>> jarsAndClasses = new TreeMap<String, SortedSet<String>>(); + + /** + * The jar dependencies and their sizes for comparison purposes. + */ + public static SortedMap<String, Long> jarSizes = new TreeMap<String, Long>(); + + /** + * Dummy logger to fill in log4j info for things such as the jar sizes. + * Usually not needed. + */ + protected static Logger log = Logger.getLogger(BuildLite.class); + + private static boolean includeAllCommonsMath3 = true; + + /** + * Create lite jar file using the default path and file name as the + * destination. All commons-math3 classes will be included. + * + * @throws Exception + * if exception occurs building jar + */ + public static void createLiteJar() throws Exception { + createLiteJar(null, true); + } + + /** + * Create lite jar file using the default path and file name as the + * destination, specifying whether all commons-math3 classes should be + * included in the jar or only the detected required subset. + * + * @param allCommonsMath3 + * if true, include all commons-math3 classes. if false, include + * only required subset in jar built + * @throws Exception + * if exception occurs building jar + */ + public static void createLiteJar(boolean allCommonsMath3) throws Exception { + createLiteJar(null, allCommonsMath3); + } + + /** + * Create lite jar file specifying the destination path and file name as a + * string. + * + * @param jarFileDestination + * the path and file name for the lite jar + * @param allCommonsMath3 + * if true, include all commons-math3 classes. if false, include + * only required subset in jar built + * @throws Exception + * if exception occurs building jar + */ + public static void createLiteJar(String jarFileDestination, boolean allCommonsMath3) throws Exception { + if (jarFileDestination != null) { + liteJarLocation = jarFileDestination; + } + includeAllCommonsMath3 = allCommonsMath3; + scanJavaFilesForClassesToLoad(); + List<Class<?>> loadedClasses = getLoadedClasses(); + displayLoadedClasses(loadedClasses); + excludePackages(loadedClasses); + displayLoadedClasses(loadedClasses); + groupLoadedClassesByJarAndClass(loadedClasses); + List<String> log4jClassPathNames = getLog4jClassPathNames(); + displayLog4JClassPathNames(log4jClassPathNames); + List<String> commonsMath3ClassPathNames = null; + if (includeAllCommonsMath3) { + commonsMath3ClassPathNames = getCommonsMath3ClassPathNames(); + displayCommonsMath3ClassPathNames(commonsMath3ClassPathNames); + } + displayJarsAndClasses(); + Set<String> consolidatedClassPathNames = consolidateClassPathNames(loadedClasses, log4jClassPathNames, + commonsMath3ClassPathNames); + createJarFromConsolidatedClassPathNames(consolidatedClassPathNames); + createDependencySets(); + displayJarSizes(); + liteJarStats(); + } + + /** + * Exclude selected packages from the loaded classes. + * + * @param loadedClasses + * classes that have been loaded by the classloader + */ + private static void excludePackages(List<Class<?>> loadedClasses) { + System.out.println("\nExcluding selected packages"); + int count = 0; + for (int i = 0; i < loadedClasses.size(); i++) { + Class<?> clazz = loadedClasses.get(i); + String className = clazz.getName(); + for (String packageToExclude : packagesToExclude) { + if (className.startsWith(packageToExclude)) { + System.out.println(" #" + ++count + ": Excluding " + className); + loadedClasses.remove(i); + i--; + } + } + } + + } + + /** + * Statistics about the lite jar such as jar size. + */ + private static void liteJarStats() { + File f = new File(liteJarLocation); + if (f.exists()) { + Long jarSize = f.length(); + String jarSizeDisplay = FileUtils.byteCountToDisplaySize(jarSize); + System.out.println("\nFinished creating " + liteJarLocation + " (" + jarSizeDisplay + " [" + + NumberFormat.getInstance().format(jarSize) + " bytes])"); + } else { + System.out.println(liteJarLocation + " could not be found"); + } + } + + /** + * Consolidate the loaded classes and all the log4j classes and potentially + * all the commons-math3 classes. + * + * @param loadedClasses + * the loaded classes + * @param log4jClassPathNames + * the log4j class names + * @param commonsMath3ClassPathNames + * the commons-math3 class names + * @return the set of unique class names that combines the loaded classes + * and the log4j classes + */ + private static Set<String> consolidateClassPathNames(List<Class<?>> loadedClasses, List<String> log4jClassPathNames, + List<String> commonsMath3ClassPathNames) { + + SortedSet<String> allClassPathNames = new TreeSet<String>(log4jClassPathNames); + if (includeAllCommonsMath3) { + System.out.println("\nConsolidating loaded class names, log4j class names, and commons-math3 class names"); + allClassPathNames.addAll(commonsMath3ClassPathNames); + } else { + System.out.println("\nConsolidating loaded class names and log4j class names"); + } + for (Class<?> clazz : loadedClasses) { + String loadedClassPathName = clazz.getName(); + loadedClassPathName = loadedClassPathName.replace(".", "/"); + loadedClassPathName = loadedClassPathName + ".class"; + allClassPathNames.add(loadedClassPathName); + } + return allClassPathNames; + } + + /** + * Build a lite jar based on the consolidated class names. + * + * @param consolidateClassPathNames + * the consolidated class names + * @throws IOException + * if an IOException occurs + */ + private static void createJarFromConsolidatedClassPathNames(Set<String> consolidateClassPathNames) + throws IOException { + System.out.println("\nCreating " + liteJarLocation + " file"); + ClassLoader cl = BuildLite.class.getClassLoader(); + + Manifest mf = new Manifest(); + Attributes attr = mf.getMainAttributes(); + attr.putValue("" + Attributes.Name.MANIFEST_VERSION, "1.0"); + + File file = new File(liteJarLocation); + try (FileOutputStream fos = new FileOutputStream(file); JarOutputStream jos = new JarOutputStream(fos, mf)) { + int numFilesWritten = 0; + for (String classPathName : consolidateClassPathNames) { + writeMessage(classPathName, ++numFilesWritten); + InputStream is = cl.getResourceAsStream(classPathName); + byte[] bytes = IOUtils.toByteArray(is); + + JarEntry je = new JarEntry(classPathName); + jos.putNextEntry(je); + jos.write(bytes); + } + + writeIdentifierFileToLiteJar(jos, ++numFilesWritten); + writeAdditionalResourcesToJar(jos, numFilesWritten); + } + + } + + /** + * Write an identifier file to the lite jar that can be used to identify + * that the lite jar is being used. + * + * @param jos + * output stream to the jar being written + * @param numFilesWritten + * the number of files written to the jar so far + * @throws IOException + * if an IOException occurs + */ + private static void writeIdentifierFileToLiteJar(JarOutputStream jos, int numFilesWritten) throws IOException { + writeMessage(LITE_JAR_IDENTIFIER_FILE, numFilesWritten); + JarEntry je = new JarEntry(LITE_JAR_IDENTIFIER_FILE); + jos.putNextEntry(je); + String created = "Created " + (new Date()); + String userName = System.getProperty("user.name"); + if (userName != null) { + created = created + " by " + userName; + } + jos.write(created.getBytes()); + } + + /** + * Write the additional resources to the jar. + * + * @param jos + * output stream to the jar being written + * @param numFilesWritten + * the number of files written to the jar so far + * @throws IOException + * if an IOException occurs + */ + private static void writeAdditionalResourcesToJar(JarOutputStream jos, int numFilesWritten) throws IOException { + for (String resource : additionalResources) { + writeMessage(resource, ++numFilesWritten); + JarEntry je = new JarEntry(resource); + jos.putNextEntry(je); + ClassLoader cl = BuildLite.class.getClassLoader(); + InputStream is = cl.getResourceAsStream(resource); + byte[] bytes = IOUtils.toByteArray(is); + jos.write(bytes); + } + } + + /** + * Output message about the resource being written to the lite jar. + * + * @param resource + * the path to the resource + * @param numFilesWritten + * the number of files written to the jar so far + */ + private static void writeMessage(String resource, int numFilesWritten) { + System.out.println(" #" + numFilesWritten + ": Writing " + resource + " to " + liteJarLocation); + } + + /** + * Obtain a list of all log4j classes in the referenced log4j jar file. + * + * @return list of all the log4j classes in the referenced log4j jar file + * @throws IOException + * if an IOException occurs + * @throws ClassNotFoundException + * if a ClassNotFoundException occurs + */ + private static List<String> getLog4jClassPathNames() throws IOException, ClassNotFoundException { + return getAllClassesInJar(ThrowableInformation.class); + } + + /** + * Obtain a list of all commons-math3 classes in the referenced + * commons-math3 jar file. + * + * @return list of all the commons-math3 classes in the referenced + * commons-math3 jar file + * @throws IOException + * if an IOException occurs + * @throws ClassNotFoundException + * if a ClassNotFoundException occurs + */ + private static List<String> getCommonsMath3ClassPathNames() throws IOException, ClassNotFoundException { + return getAllClassesInJar(Well1024a.class); + } + + /** + * Obtain a list of all classes in a jar file corresponding to a referenced + * class. + * + * @param classInJarFile + * @return list of all the commons-math3 classes in the referenced + * commons-math3 jar file + * @throws IOException + * if an IOException occurs + * @throws ClassNotFoundException + * if a ClassNotFoundException occurs + */ + private static List<String> getAllClassesInJar(Class<?> classInJarFile) throws IOException, ClassNotFoundException { + List<String> classPathNames = new ArrayList<String>(); + String jarLocation = classInJarFile.getProtectionDomain().getCodeSource().getLocation().getPath(); + File f = new File(jarLocation); + try (FileInputStream fis = new FileInputStream(f); + JarArchiveInputStream jais = new JarArchiveInputStream(fis)) { + while (true) { + JarArchiveEntry jae = jais.getNextJarEntry(); + if (jae == null) { + break; + } + String name = jae.getName(); + if (name.endsWith(".class")) { + classPathNames.add(name); + } + } + } + + String jarName = jarLocation.substring(jarLocation.lastIndexOf("/") + 1); + addClassPathNamesToJarsAndClasses(jarName, classPathNames); + + return classPathNames; + } + + /** + * Add jar and classes to the map of jars and their classes. + * + * @param log4jJar + * the log4j jar file + * @param classPathNames + * the list of log4j classes + */ + private static void addClassPathNamesToJarsAndClasses(String jar, List<String> classPathNames) { + for (String classPathName : classPathNames) { + String className = classPathName.substring(0, classPathName.length() - 6); + className = className.replace("/", "."); + addJarAndClass(jar, className); + } + } + + /** + * Dislay all the log4j classes + * + * @param log4jClassPathNames + * the list of log4j classes + */ + private static void displayLog4JClassPathNames(List<String> log4jClassPathNames) { + int numClasses = 0; + System.out.println("\nAll log4j class files:"); + for (String classPathName : log4jClassPathNames) { + numClasses++; + System.out.println(" #" + numClasses + ": " + classPathName); + } + } + + /** + * Dislay all the commons-math3 classes + * + * @param commonsMath3ClassPathNames + * the list of commons-math3 classes + */ + private static void displayCommonsMath3ClassPathNames(List<String> commonsMath3ClassPathNames) { + int numClasses = 0; + System.out.println("\nAll commons-math3 class files:"); + for (String classPathName : commonsMath3ClassPathNames) { + numClasses++; + System.out.println(" #" + numClasses + ": " + classPathName); + } + } + + /** + * Obtain a list of all the classes that have been loaded by the + * classloader. + * + * @return a list of all the classes that have been loaded by the + * classloader + * @throws NoSuchFieldException + * if NoSuchFieldException occurs + * @throws SecurityException + * if SecurityException occurs + * @throws IllegalArgumentException + * if IllegalArgumentException occurs + * @throws IllegalAccessException + * if IllegalAccessException occurs + */ + private static List<Class<?>> getLoadedClasses() + throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException { + ClassLoader cl = BuildLite.class.getClassLoader(); + Class<?> clClazz = cl.getClass(); + while (clClazz != java.lang.ClassLoader.class) { + clClazz = clClazz.getSuperclass(); + } + Field f = clClazz.getDeclaredField("classes"); + f.setAccessible(true); + @SuppressWarnings("unchecked") + Vector<Class<?>> classes = (Vector<Class<?>>) f.get(cl); + List<Class<?>> list = new ArrayList<Class<?>>(classes); + + return list; + } + + /** + * Group the classes by jar file. Also, obtain the jar file sizes for + * comparison purposes. + * + * @param loadedClasses + * the list of loaded classes + */ + private static void groupLoadedClassesByJarAndClass(List<Class<?>> loadedClasses) { + for (Class<?> clazz : loadedClasses) { + String pathToClass = getPathToClass(clazz); + if (pathToClass == null) { + addJarAndClass("?", clazz.getName()); + } else if (pathToClass.endsWith(".jar")) { + String jarName = pathToClass.substring(pathToClass.lastIndexOf("/") + 1); + addJarAndClass(jarName, clazz.getName()); + + // for comparison purposes + if (!jarSizes.containsKey(jarName)) { + String jarPath = pathToClass; + File jarFile = new File(jarPath); + long fileLength = jarFile.length(); + jarSizes.put(jarName, fileLength); + } + + } else if (pathToClass.contains("systemml")) { + addJarAndClass("SystemML", clazz.getName()); + } else { + addJarAndClass("Other", clazz.getName()); + } + } + } + + /** + * Add a jar and class to the map of jars to their sets of classes. + * + * @param jarName + * the name of the jar file + * @param className + * the name of the class + */ + private static void addJarAndClass(String jarName, String className) { + if (jarsAndClasses.containsKey(jarName)) { + SortedSet<String> classNames = jarsAndClasses.get(jarName); + classNames.add(className); + } else { + SortedSet<String> classNames = new TreeSet<String>(); + classNames.add(className); + jarsAndClasses.put(jarName, classNames); + } + } + + /** + * Display the list of loaded classes. + * + * @param loadedClasses + * the list of loaded classes + */ + private static void displayLoadedClasses(List<Class<?>> loadedClasses) { + int numClasses = 0; + System.out.println("\nLoaded classes:"); + for (Class<?> clazz : loadedClasses) { + numClasses++; + System.out.println(" #" + numClasses + ": " + clazz + " (" + getPathToClass(clazz) + ")"); + } + } + + /** + * Obtain the file system path to the location of a class. + * + * @param clazz + * the class + * @return the file system path to the location of a class + */ + private static String getPathToClass(Class<?> clazz) { + try { + return clazz.getProtectionDomain().getCodeSource().getLocation().getPath(); + } catch (java.lang.NullPointerException e) { + return null; + } + } + + /** + * Display the required classes grouped by their jar files. + * + * @throws IOException + * if IOException occurs + */ + private static void displayJarsAndClasses() throws IOException { + ClassLoader cl = BuildLite.class.getClassLoader(); + System.out.println("\nRequired Classes Grouped by Jar:"); + Set<String> jarNames = jarsAndClasses.keySet(); + int numClasses = 0; + for (String jarName : jarNames) { + SortedSet<String> classNames = jarsAndClasses.get(jarName); + StringBuilder sb = new StringBuilder(); + int totalBytesUncompressed = 0; + for (String className : classNames) { + String classNamePath = className.replace(".", "/") + ".class"; + InputStream is = cl.getResourceAsStream(classNamePath); + byte[] bytes = IOUtils.toByteArray(is); + int numBytes = bytes.length; + numClasses++; + sb.append(" #" + numClasses + " " + className + " [" + NumberFormat.getInstance().format(numBytes) + + " bytes])\n"); + totalBytesUncompressed += numBytes; + } + System.out.println("Jar: " + jarName + " [" + NumberFormat.getInstance().format(totalBytesUncompressed) + + " bytes uncompressed]"); + System.out.println(sb.toString()); + } + } + + /** + * Examine all java source files for additional classes to load. A + * relatively easy though not perfect way to do this is to look for imports + * of specified packages. + * + * @throws IOException + * if IOException occurs + * @throws ClassNotFoundException + * if ClassNotFoundException occurs + */ + private static void scanJavaFilesForClassesToLoad() throws IOException, ClassNotFoundException { + System.out.println("\nScanning java files for additional classes to load"); + int totalMatches = 0; + SortedSet<String> uniqueMatches = new TreeSet<String>(); + File base = new File(BASE_SRC_DIR); + List<File> javaFiles = (List<File>) FileUtils.listFiles(base, new String[] { "java" }, true); + for (File javaFile : javaFiles) { + String content = FileUtils.readFileToString(javaFile); + for (String additionalPackage : additionalPackages) { + String s = "import " + additionalPackage + "(.*?);"; + Pattern p = Pattern.compile(s); + Matcher m = p.matcher(content); + while (m.find()) { + totalMatches++; + String match = m.group(1); + String matchClass = additionalPackage + match; + uniqueMatches.add(matchClass); + } + } + } + System.out.println("Total matches found from scan: " + totalMatches); + int uniqueMatchesSize = uniqueMatches.size(); + System.out.println("Unique matches found from scan: " + uniqueMatchesSize); + int numMatches = 0; + for (String matchClass : uniqueMatches) { + Class<?> clazz = Class.forName(matchClass); + String pathToClass = getPathToClass(clazz); + System.out.println(" #" + ++numMatches + ": Loaded " + clazz.getName() + " from " + pathToClass); + } + } + + /** + * Display a list of the jar dependencies twice. The first display lists the + * jars, which can be useful when assembling information for the LICENSE + * file. The second display lists the jars and their sizes, which is useful + * for gauging the size decrease offered by the lite jar file. + */ + private static void displayJarSizes() { + System.out.println("\nIndividual Jar Dependencies (for Comparison):"); + Set<String> jarNames = jarSizes.keySet(); + + for (String jarName : jarNames) { + System.out.println(jarName); + } + System.out.println(); + + Long totalSize = 0L; + int count = 0; + for (String jarName : jarNames) { + Long jarSize = jarSizes.get(jarName); + String jarSizeDisplay = FileUtils.byteCountToDisplaySize(jarSize); + System.out.println(" #" + ++count + ": " + jarName + " (" + jarSizeDisplay + " [" + + NumberFormat.getInstance().format(jarSize) + " bytes])"); + totalSize = totalSize + jarSize; + } + String totalSizeDisplay = FileUtils.byteCountToDisplaySize(totalSize); + System.out.println("Total Size of Jar Dependencies: " + totalSizeDisplay + " [" + + NumberFormat.getInstance().format(totalSize) + " bytes]"); + } + + /** + * Generate maven assembly dependency sets that can be used by the lite.xml + * assembly. + * + * @throws IOException + * if IOException occurs + */ + private static void createDependencySets() throws IOException { + System.out.println("\nCreating maven dependency sets"); + + StringBuilder sb = new StringBuilder(); + sb.append("\t<dependencySets>\n"); + Set<String> jarNames = jarsAndClasses.keySet(); + for (String jarName : jarNames) { + String s = generateDependencySet(jarName, jarsAndClasses.get(jarName)); + sb.append(s); + } + sb.append(generateSystemMLDependencySet()); + sb.append("\t</dependencySets>\n"); + System.out.println(sb.toString()); + + final String liteXml = "src/assembly/lite.xml"; + File f = new File(liteXml); + if (f.exists()) { + System.out.println("Found '" + liteXml + "', so updating dependencySets in the file."); + String s = FileUtils.readFileToString(f); + int start = s.indexOf("\t<dependencySets>"); + int end = s.indexOf("</dependencySets>") + "</dependencySets>".length() + 1; + String before = s.substring(0, start); + String after = s.substring(end); + String newS = before + sb.toString() + after; + FileUtils.writeStringToFile(f, newS); + } + } + + /** + * Generate a maven assembly dependency set that can be used by the lite.xml + * assembly. Note that additional resources can be added to the jar by the + * additionalJarToFileMappingsForDependencySets entries. + * + * @param jarName + * the name of the jar file + * @param classNames + * a set of the classes in the jar file + * @return a string representation of the dependency set consisting of the + * resources in the jar file + */ + private static String generateDependencySet(String jarName, SortedSet<String> classNames) { + StringBuilder sb = new StringBuilder(); + String jarNameNoVersion = null; + if ("SystemML".equalsIgnoreCase(jarName)) { + jarNameNoVersion = "systemml"; + return ""; // handle in generateSystemMLDependencySet() + } else { + jarNameNoVersion = jarName.substring(0, jarName.lastIndexOf("-")); + } + sb.append("\t\t<dependencySet>\n"); + sb.append("\t\t\t<includes>\n"); + sb.append("\t\t\t\t<include>*:" + jarNameNoVersion + "</include>\n"); + sb.append("\t\t\t</includes>\n"); + + sb.append("\t\t\t<unpackOptions>\n"); + sb.append("\t\t\t\t<includes>\n"); + + Set<String> jarsWithAdditionalFileMappings = additionalJarToFileMappingsForDependencySets.keySet(); + if (jarsWithAdditionalFileMappings.contains(jarNameNoVersion)) { + SortedSet<String> additionalResourceFiles = additionalJarToFileMappingsForDependencySets + .get(jarNameNoVersion); + for (String resourceFile : additionalResourceFiles) { + sb.append("\t\t\t\t\t<include>" + resourceFile + "</include>\n"); + } + } + + if (jarName.startsWith("log4j")) { + sb.append("\t\t\t\t\t<include>**/*.class</include>\n"); + } else if (includeAllCommonsMath3 && (jarName.startsWith("commons-math3"))) { + sb.append("\t\t\t\t\t<include>**/*.class</include>\n"); + } else { + for (String className : classNames) { + String classFileName = className.replace(".", "/") + ".class"; + sb.append("\t\t\t\t\t<include>" + classFileName + "</include>\n"); + } + } + sb.append("\t\t\t\t</includes>\n"); + sb.append("\t\t\t</unpackOptions>\n"); + + sb.append("\t\t\t<scope>compile</scope>\n"); + sb.append("\t\t\t<unpack>true</unpack>\n"); + sb.append("\t\t</dependencySet>\n"); + sb.append("\n"); + return sb.toString(); + } + + private static String generateSystemMLDependencySet() { + StringBuilder sb = new StringBuilder(); + sb.append("\t\t<dependencySet>\n"); + sb.append("\t\t\t<includes>\n"); + sb.append("\t\t\t\t<include>*:systemml*</include>\n"); + sb.append("\t\t\t</includes>\n"); + sb.append("\t\t\t<unpackOptions>\n"); + sb.append("\t\t\t\t<excludes>\n"); + sb.append("\t\t\t\t\t<exclude>META-INF/DEPENDENCIES</exclude>\n"); + sb.append("\t\t\t\t\t<exclude>META-INF/maven/**</exclude>\n"); + sb.append("\t\t\t\t\t<exclude>kernels/**</exclude>\n"); + sb.append("\t\t\t\t\t<exclude>lib/**</exclude>\n"); + sb.append("\t\t\t\t</excludes>\n"); + sb.append("\t\t\t</unpackOptions>\n"); + sb.append("\t\t\t<outputDirectory>.</outputDirectory>\n"); + sb.append("\t\t\t<scope>compile</scope>\n"); + sb.append("\t\t\t<unpack>true</unpack>\n"); + sb.append("\t\t</dependencySet>\n"); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/139ddfee/src/main/java/org/apache/sysml/utils/lite/BuildLiteExecution.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/lite/BuildLiteExecution.java b/src/main/java/org/apache/sysml/utils/lite/BuildLiteExecution.java new file mode 100644 index 0000000..1c858ef --- /dev/null +++ b/src/main/java/org/apache/sysml/utils/lite/BuildLiteExecution.java @@ -0,0 +1,709 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sysml.utils.lite; + +import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; + +import org.apache.commons.io.FileUtils; +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.sysml.api.jmlc.Connection; +import org.apache.sysml.api.jmlc.PreparedScript; +import org.apache.sysml.api.jmlc.ResultVariables; +import org.apache.sysml.conf.CompilerConfig.ConfigType; +import org.apache.sysml.runtime.DMLRuntimeException; +import org.apache.sysml.runtime.util.DataConverter; + +/** + * Execute code and conditionally build a lightweight jar file that can support + * the execution of that code. + * + */ +public class BuildLiteExecution { + + protected static Logger log = Logger.getLogger(BuildLiteExecution.class); + + public static void main(String[] args) throws Exception { + + BasicConfigurator.configure(); + Logger.getRootLogger().setLevel(Level.INFO); + log.setLevel(Level.DEBUG); + + jmlcHelloWorld(); + jmlcScoringExample(); + jmlcUnivariateStatistics(); + jmlcWriteMatrix(); + jmlcReadMatrix(); + jmlcBasics(); + jmlcL2SVM(); + jmlcLinReg(); + jmlcALS(); + jmlcKmeans(); + jmlcTests(); + + BuildLite.createLiteJar(true); + + } + + public static void jmlcHelloWorld() throws Exception { + Connection conn = getConfiguredConnection(); + PreparedScript script = conn.prepareScript("print('hello world');", new String[] {}, new String[] {}, false); + script.executeScript(); + conn.close(); + } + + public static void jmlcScoringExample() throws Exception { + /* @formatter:off */ + String scriptString = + "X = read(\"./tmp/X\", rows=-1, cols=-1);\n" + + "W = read(\"./tmp/W\", rows=-1, cols=-1);\n" + + "\n" + + "numRows = nrow(X);\n" + + "numCols = ncol(X);\n" + + "b = W[numCols+1,]\n" + + "scores = X %*% W[1:numCols,] + b;\n" + + "predicted_y = rowIndexMax(scores);\n" + + "\n" + + "print('pred:' + toString(predicted_y))\n" + + "\n" + + "write(predicted_y, \"./tmp\", format=\"text\");\n"; + /* @formatter:on */ + + File file = new File("temp/scoring-example.dml"); + FileUtils.writeStringToFile(file, scriptString); + + Connection conn = getConfiguredConnection(); + String dml = conn.readScript("temp/scoring-example.dml"); + PreparedScript script = conn.prepareScript(dml, new String[] { "W", "X" }, new String[] { "predicted_y" }, + false); + + double[][] mtx = matrix(4, 3, new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + double[][] result = null; + + script.setMatrix("W", mtx); + script.setMatrix("X", randomMatrix(3, 3, -1, 1, 0.7)); + result = script.executeScript().getMatrix("predicted_y"); + log.debug(displayMatrix(result)); + + script.setMatrix("W", mtx); + script.setMatrix("X", randomMatrix(3, 3, -1, 1, 0.7)); + result = script.executeScript().getMatrix("predicted_y"); + log.debug(displayMatrix(result)); + + script.setMatrix("W", mtx); + script.setMatrix("X", randomMatrix(3, 3, -1, 1, 0.7)); + result = script.executeScript().getMatrix("predicted_y"); + log.debug(displayMatrix(result)); + + conn.close(); + } + + public static void jmlcUnivariateStatistics() throws Exception { + + Connection conn = getConfiguredConnection(); + + String dml = conn.readScript("scripts/algorithms/Univar-Stats.dml"); + Map<String, String> m = new HashMap<String, String>(); + m.put("$CONSOLE_OUTPUT", "TRUE"); + + PreparedScript script = conn.prepareScript(dml, m, new String[] { "A", "K" }, new String[] { "baseStats" }, + false); + + double[][] data = new double[100][4]; + for (int i = 0; i < 100; i++) { + int one = ThreadLocalRandom.current().nextInt(0, 101); + int two = ThreadLocalRandom.current().nextInt(0, 101); + int three = ThreadLocalRandom.current().nextInt(0, 101); + int four = ThreadLocalRandom.current().nextInt(1, 3); + double[] row = new double[] { one, two, three, four }; + data[i] = row; + } + log.debug(displayMatrix(data)); + + double[][] types = matrix(1, 4, new double[] { 1, 1, 1, 2 }); + + script.setMatrix("A", data); + script.setMatrix("K", types); + double[][] baseStats = script.executeScript().getMatrix("baseStats"); + log.debug(displayMatrix(baseStats)); + + conn.close(); + } + + public static void jmlcWriteMatrix() throws Exception { + Connection conn = getConfiguredConnection(); + PreparedScript script = conn.prepareScript( + "x=matrix('1 2 3 4',rows=2,cols=2);write(x,'temp/x.csv',format='csv');", new String[] {}, + new String[] {}, false); + script.executeScript(); + + /* @formatter:off */ + String scriptString = + "m = matrix('1 2 3 0 0 0 7 8 9 0 0 0', rows=4, cols=3)\n" + + "write(m, 'temp/m.txt', format='text')\n" + + "write(m, 'temp/m.mm', format='mm')\n" + + "write(m, 'temp/m.csv', format='csv')\n" + + "write(m, 'temp/m.binary', format='binary')\n"; + /* @formatter:on */ + + script = conn.prepareScript(scriptString, new String[] {}, new String[] {}, false); + script.executeScript(); + + conn.close(); + } + + public static void jmlcReadMatrix() throws Exception { + Connection conn = getConfiguredConnection(); + PreparedScript script = conn.prepareScript("x=read('temp/x.csv',format='csv');y=x*2;print(toString(y));", + new String[] {}, new String[] {}, false); + script.executeScript(); + + /* @formatter:off */ + String scriptString = "m = read('temp/m.csv',format='csv')\n" + + "print(toString(m))\n" + + "print('min:' + min(m))\n" + + "print('max:' + max(m))\n" + + "print('sum:' + sum(m))\n" + + "mRowSums = rowSums(m)\n" + + "for (i in 1:nrow(mRowSums)) {\n" + + " print('row ' + i + ' sum:' + as.scalar(mRowSums[i,1]))\n" + + "}\n" + + "mColSums = colSums(m)\n" + + "for (i in 1:ncol(mColSums)) {\n" + + " print('col ' + i + ' sum:' + as.scalar(mColSums[1,i]))\n" + + "}\n"; + + // note: the following can be set to work using the following setting + // in the Connection class: cconf.set(ConfigType.IGNORE_READ_WRITE_METADATA, false); + + // "m2=read('temp/m.txt', format='text')\n" + + // "m3=read('temp/m.mm', format='mm')\n" + + // "m4=read('temp/m.binary', format='binary')\n" + + // "print('m2:'+toString(m2))\n" + + // "print('m3:'+toString(m3))\n" + + // "print('m4:'+toString(m4))\n"; + /* @formatter:on */ + + script = conn.prepareScript(scriptString, new String[] {}, new String[] {}, false); + script.executeScript(); + + conn.close(); + } + + public static void jmlcBasics() throws Exception { + + /* @formatter:off */ + String dml = + "A = matrix(\"1 2 3 4 5 6\", rows=3, cols=2)\n"+ + "print(toString(A))\n"+ + "B = A + 4\n"+ + "B = t(B)\n"+ + "print(toString(B))\n"+ + "C = A %*% B\n"+ + "print(toString(C))\n"+ + "D = matrix(5, rows=nrow(C), cols=ncol(C))\n"+ + "D = (C - D) / 2\n"+ + "print(toString(D))\n"+ + "\n"+ + "A = matrix(\"1 2 3 4 5 6 7 8 9\", rows=3, cols=3)\n"+ + "print(toString(A))\n"+ + "B = A[3,3]\n"+ + "print(toString(B))\n"+ + "C = A[2,]\n"+ + "print(toString(C))\n"+ + "D = A[,3]\n"+ + "print(toString(D))\n"+ + "E = A[2:3,1:2]\n"+ + "print(toString(E))\n"+ + "\n"+ + "i = 1\n"+ + "while (i <= 3) {\n"+ + " if (i == 1) {\n"+ + " print(\'hello\')\n"+ + " } else if (i == 2) {\n"+ + " print(\'world\')\n"+ + " } else {\n"+ + " print(\'!!!\')\n"+ + " }\n"+ + " i = i + 1\n"+ + "}\n"+ + "\n"+ + "A = matrix(\"1 2 3 4 5 6\", rows=3, cols=2)\n"+ + "\n"+ + "for (i in 1:nrow(A)) {\n"+ + " print(\"for A[\" + i + \",1]:\" + as.scalar(A[i,1]))\n"+ + "}\n"+ + "\n"+ + "parfor(i in 1:nrow(A)) {\n"+ + " print(\"parfor A[\" + i + \",1]:\" + as.scalar(A[i,1]))\n"+ + "}\n"+ + "\n"+ + "doSomething = function(matrix[double] mat) return (matrix[double] ret) {\n"+ + " additionalCol = matrix(1, rows=nrow(mat), cols=1) # 1x3 matrix with 1 values\n"+ + " ret = cbind(mat, additionalCol) # concatenate column to matrix\n"+ + " ret = cbind(ret, seq(0, 2, 1)) # concatenate column (0,1,2) to matrix\n"+ + " ret = cbind(ret, rowMaxs(ret)) # concatenate column of max row values to matrix\n"+ + " ret = cbind(ret, rowSums(ret)) # concatenate column of row sums to matrix\n"+ + "}\n"+ + "\n"+ + "A = rand(rows=3, cols=2, min=0, max=2) # random 3x2 matrix with values 0 to 2\n"+ + "B = doSomething(A)\n"+ + "write(A, \"A.csv\", format=\"csv\")\n"+ + "write(B, \"B.csv\", format=\"csv\")\n"; + /* @formatter:on */ + + Connection conn = getConfiguredConnection(); + PreparedScript script = conn.prepareScript(dml, new String[] {}, new String[] {}, false); + script.executeScript(); + conn.close(); + } + + public static void jmlcL2SVM() throws Exception { + + Connection conn = getConfiguredConnection(); + + String dml = conn.readScript("scripts/algorithms/l2-svm.dml"); + PreparedScript l2svm = conn.prepareScript(dml, new String[] { "X", "Y", "fmt", "Log" }, + new String[] { "w", "debug_str" }, false); + + double[][] trainData = new double[150][3]; + for (int i = 0; i < 150; i++) { + int one = ThreadLocalRandom.current().nextInt(0, 101); + int two = ThreadLocalRandom.current().nextInt(0, 101); + int three = ThreadLocalRandom.current().nextInt(0, 101); + double[] row = new double[] { one, two, three }; + trainData[i] = row; + } + + l2svm.setMatrix("X", trainData); + log.debug(displayMatrix(trainData)); + + double[][] trainLabels = new double[150][1]; + for (int i = 0; i < 150; i++) { + int one = ThreadLocalRandom.current().nextInt(1, 3); + double[] row = new double[] { one }; + trainLabels[i] = row; + } + l2svm.setMatrix("Y", trainLabels); + log.debug(displayMatrix(trainLabels)); + + l2svm.setScalar("fmt", "csv"); + + l2svm.setScalar("Log", "temp/l2-svm-log.csv"); + + ResultVariables l2svmResults = l2svm.executeScript(); + double[][] model = l2svmResults.getMatrix("w"); + log.debug("MODEL:"); + log.debug(displayMatrix(model)); + String debugString = l2svmResults.getString("debug_str"); + log.debug("DEBUG STRING:"); + log.debug(debugString); + + String s = conn.readScript("scripts/algorithms/l2-svm-predict.dml"); + Map<String, String> m = new HashMap<String, String>(); + m.put("$Y", "temp/1.csv"); + m.put("$confusion", "temp/2.csv"); + m.put("$scores", "temp/3.csv"); + + PreparedScript l2svmPredict = conn.prepareScript(s, m, new String[] { "X", "y", "w", "fmt" }, + new String[] { "scores", "confusion_mat" }, false); + + double[][] testData = new double[150][3]; + for (int i = 0; i < 150; i++) { + int one = ThreadLocalRandom.current().nextInt(0, 101); + int two = ThreadLocalRandom.current().nextInt(0, 101); + int three = ThreadLocalRandom.current().nextInt(0, 101); + double[] row = new double[] { one, two, three }; + testData[i] = row; + } + l2svmPredict.setMatrix("X", testData); + + double[][] testLabels = new double[150][1]; + for (int i = 0; i < 150; i++) { + int one = ThreadLocalRandom.current().nextInt(1, 3); + double[] row = new double[] { one }; + testLabels[i] = row; + } + l2svmPredict.setMatrix("y", testLabels); + + l2svmPredict.setMatrix("w", model); + + l2svmPredict.setScalar("fmt", "csv"); + + ResultVariables l2svmPredictResults = l2svmPredict.executeScript(); + double[][] scores = l2svmPredictResults.getMatrix("scores"); + log.debug("SCORES:"); + log.debug(displayMatrix(scores)); + + double[][] confusionMatrix = l2svmPredictResults.getMatrix("confusion_mat"); + log.debug("CONFUSION MATRIX:"); + log.debug(displayMatrix(confusionMatrix)); + + conn.close(); + } + + public static void jmlcLinReg() throws Exception { + + Connection conn = getConfiguredConnection(); + + String linRegDS = conn.readScript("scripts/algorithms/LinearRegDS.dml"); + PreparedScript linRegDSScript = conn.prepareScript(linRegDS, new String[] { "X", "y" }, + new String[] { "beta_out" }, false); + + double[][] trainData = new double[500][3]; + for (int i = 0; i < 500; i++) { + double one = ThreadLocalRandom.current().nextDouble(0, 100); + double two = ThreadLocalRandom.current().nextDouble(0, 100); + double three = ThreadLocalRandom.current().nextDouble(0, 100); + double[] row = new double[] { one, two, three }; + trainData[i] = row; + } + linRegDSScript.setMatrix("X", trainData); + log.debug(displayMatrix(trainData)); + + double[][] trainLabels = new double[500][1]; + for (int i = 0; i < 500; i++) { + double one = ThreadLocalRandom.current().nextDouble(0, 100); + double[] row = new double[] { one }; + trainLabels[i] = row; + } + linRegDSScript.setMatrix("y", trainLabels); + log.debug(displayMatrix(trainLabels)); + + ResultVariables linRegDSResults = linRegDSScript.executeScript(); + double[][] dsBetas = linRegDSResults.getMatrix("beta_out"); + log.debug("DS BETAS:"); + log.debug(displayMatrix(dsBetas)); + + String linRegCG = conn.readScript("scripts/algorithms/LinearRegCG.dml"); + PreparedScript linRegCGScript = conn.prepareScript(linRegCG, new String[] { "X", "y" }, + new String[] { "beta_out" }, false); + linRegCGScript.setMatrix("X", trainData); + linRegCGScript.setMatrix("y", trainLabels); + ResultVariables linRegCGResults = linRegCGScript.executeScript(); + double[][] cgBetas = linRegCGResults.getMatrix("beta_out"); + log.debug("CG BETAS:"); + log.debug(displayMatrix(cgBetas)); + + String glmPredict = conn.readScript("scripts/algorithms/GLM-predict.dml"); + PreparedScript glmPredictScript = conn.prepareScript(glmPredict, new String[] { "X", "Y", "B_full" }, + new String[] { "means" }, false); + double[][] testData = new double[500][3]; + for (int i = 0; i < 500; i++) { + double one = ThreadLocalRandom.current().nextDouble(0, 100); + double two = ThreadLocalRandom.current().nextDouble(0, 100); + double three = ThreadLocalRandom.current().nextDouble(0, 100); + double[] row = new double[] { one, two, three }; + testData[i] = row; + } + glmPredictScript.setMatrix("X", testData); + double[][] testLabels = new double[500][1]; + for (int i = 0; i < 500; i++) { + double one = ThreadLocalRandom.current().nextDouble(0, 100); + double[] row = new double[] { one }; + testLabels[i] = row; + } + glmPredictScript.setMatrix("Y", testLabels); + glmPredictScript.setMatrix("B_full", cgBetas); + ResultVariables glmPredictResults = glmPredictScript.executeScript(); + double[][] means = glmPredictResults.getMatrix("means"); + log.debug("GLM PREDICT MEANS:"); + log.debug(displayMatrix(means)); + + conn.close(); + } + + public static void jmlcALS() throws Exception { + Connection conn = getConfiguredConnection(); + + String dataGen = conn.readScript("scripts/datagen/genRandData4ALS.dml"); + + Map<String, String> m = new HashMap<String, String>(); + m.put("$rows", "1000"); + m.put("$cols", "1000"); + m.put("$rank", "100"); + m.put("$nnz", "10000"); + + PreparedScript dataGenScript = conn.prepareScript(dataGen, m, new String[] {}, new String[] { "X", "W", "H" }, + false); + + ResultVariables dataGenResults = dataGenScript.executeScript(); + double[][] x = dataGenResults.getMatrix("X"); + log.debug(displayMatrix(x)); + + Map<String, String> m2 = new HashMap<String, String>(); + m2.put("$rank", "100"); + String alsCg = conn.readScript("scripts/algorithms/ALS-CG.dml"); + PreparedScript alsCgScript = conn.prepareScript(alsCg, m2, new String[] { "X" }, new String[] { "U", "V" }, + false); + alsCgScript.setMatrix("X", x); + ResultVariables alsCgResults = alsCgScript.executeScript(); + double[][] u = alsCgResults.getMatrix("U"); + log.debug("u:" + u); + log.debug(displayMatrix(u)); + double[][] v = alsCgResults.getMatrix("V"); + log.debug("v:" + v); + log.debug(displayMatrix(v)); + + String alsDs = conn.readScript("scripts/algorithms/ALS-DS.dml"); + PreparedScript alsDsScript = conn.prepareScript(alsDs, m2, new String[] { "V" }, new String[] { "L", "Rt" }, + false); + alsDsScript.setMatrix("V", x); + ResultVariables alsDsResults = alsDsScript.executeScript(); + double[][] l = alsDsResults.getMatrix("L"); + log.debug("l:" + l); + log.debug(displayMatrix(l)); + double[][] rt = alsDsResults.getMatrix("Rt"); + log.debug("rt:" + rt); + log.debug(displayMatrix(rt)); + + conn.close(); + } + + public static void jmlcKmeans() throws Exception { + + Connection conn = getConfiguredConnection(); + + Map<String, String> m = new HashMap<String, String>(); + m.put("$k", "5"); + m.put("$isY", "TRUE"); + m.put("$verb", "TRUE"); + + String kMeans = conn.readScript("scripts/algorithms/Kmeans.dml"); + PreparedScript kMeansScript = conn.prepareScript(kMeans, m, new String[] { "X" }, new String[] { "C", "Y" }, + false); + + double[][] x = randomMatrix(50, 50, -1, 1, 0.1); + kMeansScript.setMatrix("X", x); + log.debug("X:"); + log.debug(displayMatrix(x)); + + ResultVariables kMeansResults = kMeansScript.executeScript(); + double[][] c = kMeansResults.getMatrix("C"); + log.debug("C:"); + log.debug(displayMatrix(c)); + + double[][] y = kMeansResults.getMatrix("Y"); + log.debug("Y:"); + log.debug(displayMatrix(y)); + + conn.close(); + } + + @SuppressWarnings({ "rawtypes", "resource" }) + public static void jmlcTests() { + try { + File jmlcTestDir = new File("target/test-classes"); + if (!jmlcTestDir.exists()) { + log.error("Test class directory could not be found"); + return; + } + URL url = jmlcTestDir.toURI().toURL(); + URL[] urls = new URL[] { url }; + ClassLoader cl = new URLClassLoader(urls); + + Class clazz = cl.loadClass("org.apache.sysml.test.integration.functions.jmlc.FrameCastingTest"); + Object obj = clazz.newInstance(); + invokeMethod(clazz, obj, "setUpBase"); + invokeMethod(clazz, obj, "setUp"); + invokeMethod(clazz, obj, "testJMLCTransformDense"); + invokeMethod(clazz, obj, "testJMLCTransformSparse"); + invokeMethod(clazz, obj, "testJMLCTransformDenseReuse"); + invokeMethod(clazz, obj, "testJMLCTransformSparseReuse"); + invokeMethod(clazz, obj, "tearDown"); + + clazz = cl.loadClass("org.apache.sysml.test.integration.functions.jmlc.FrameDecodeTest"); + obj = clazz.newInstance(); + invokeMethod(clazz, obj, "setUpBase"); + invokeMethod(clazz, obj, "setUp"); + invokeMethod(clazz, obj, "testJMLCTransformDense"); + invokeMethod(clazz, obj, "testJMLCTransformSparse"); + invokeMethod(clazz, obj, "testJMLCTransformDenseReuse"); + invokeMethod(clazz, obj, "testJMLCTransformSparseReuse"); + invokeMethod(clazz, obj, "tearDown"); + + clazz = cl.loadClass("org.apache.sysml.test.integration.functions.jmlc.FrameEncodeTest"); + obj = clazz.newInstance(); + invokeMethod(clazz, obj, "setUpBase"); + invokeMethod(clazz, obj, "setUp"); + invokeMethod(clazz, obj, "testJMLCTransformDense"); + invokeMethod(clazz, obj, "testJMLCTransformSparse"); + invokeMethod(clazz, obj, "testJMLCTransformDenseReuse"); + invokeMethod(clazz, obj, "testJMLCTransformSparseReuse"); + invokeMethod(clazz, obj, "tearDown"); + + clazz = cl.loadClass("org.apache.sysml.test.integration.functions.jmlc.FrameIndexingAppendTest"); + obj = clazz.newInstance(); + invokeMethod(clazz, obj, "setUpBase"); + invokeMethod(clazz, obj, "setUp"); + invokeMethod(clazz, obj, "testJMLCTransformDense"); + invokeMethod(clazz, obj, "testJMLCTransformSparse"); + invokeMethod(clazz, obj, "testJMLCTransformDenseReuse"); + invokeMethod(clazz, obj, "testJMLCTransformSparseReuse"); + invokeMethod(clazz, obj, "tearDown"); + + clazz = cl.loadClass("org.apache.sysml.test.integration.functions.jmlc.FrameLeftIndexingTest"); + obj = clazz.newInstance(); + invokeMethod(clazz, obj, "setUpBase"); + invokeMethod(clazz, obj, "setUp"); + invokeMethod(clazz, obj, "testJMLCTransformDense"); + invokeMethod(clazz, obj, "testJMLCTransformSparse"); + invokeMethod(clazz, obj, "testJMLCTransformDenseReuse"); + invokeMethod(clazz, obj, "testJMLCTransformSparseReuse"); + invokeMethod(clazz, obj, "tearDown"); + + clazz = cl.loadClass("org.apache.sysml.test.integration.functions.jmlc.FrameReadMetaTest"); + obj = clazz.newInstance(); + invokeMethod(clazz, obj, "setUpBase"); + invokeMethod(clazz, obj, "setUp"); + invokeMethod(clazz, obj, "testJMLCTransformDenseSpec"); + invokeMethod(clazz, obj, "testJMLCTransformDenseReuseSpec"); + invokeMethod(clazz, obj, "testJMLCTransformDense"); + invokeMethod(clazz, obj, "testJMLCTransformDenseReuse"); + invokeMethod(clazz, obj, "testJMLCTransformDenseReadFrame"); + invokeMethod(clazz, obj, "testJMLCTransformDenseReuseReadFrame"); + invokeMethod(clazz, obj, "tearDown"); + + clazz = cl.loadClass("org.apache.sysml.test.integration.functions.jmlc.FrameTransformTest"); + obj = clazz.newInstance(); + invokeMethod(clazz, obj, "setUpBase"); + invokeMethod(clazz, obj, "setUp"); + invokeMethod(clazz, obj, "testJMLCTransformDense"); + invokeMethod(clazz, obj, "testJMLCTransformSparse"); + invokeMethod(clazz, obj, "testJMLCTransformDenseReuse"); + invokeMethod(clazz, obj, "testJMLCTransformSparseReuse"); + invokeMethod(clazz, obj, "tearDown"); + + clazz = cl.loadClass("org.apache.sysml.test.integration.functions.jmlc.JMLCInputOutputTest"); + obj = clazz.newInstance(); + invokeMethod(clazz, obj, "setUpBase"); + invokeMethod(clazz, obj, "setUp"); + invokeMethod(clazz, obj, "testScalarInputInt"); + invokeMethod(clazz, obj, "testScalarInputDouble"); + invokeMethod(clazz, obj, "testScalarInputBoolean"); + invokeMethod(clazz, obj, "testScalarInputLong"); + invokeMethod(clazz, obj, "testScalarInputStringExplicitValueType"); + invokeMethod(clazz, obj, "testScalarOutputLong"); + invokeMethod(clazz, obj, "testScalarOutputDouble"); + invokeMethod(clazz, obj, "testScalarOutputString"); + invokeMethod(clazz, obj, "testScalarOutputBoolean"); + invokeMethod(clazz, obj, "testScalarOutputScalarObject"); + invokeMethod(clazz, obj, "tearDown"); + + clazz = cl.loadClass("org.apache.sysml.test.integration.functions.jmlc.JMLCInputStreamReadTest"); + obj = clazz.newInstance(); + invokeMethod(clazz, obj, "setUpBase"); + invokeMethod(clazz, obj, "setUp"); + invokeMethod(clazz, obj, "testInputStreamReadMatrixDenseCSV"); + invokeMethod(clazz, obj, "testInputStreamReadMatrixDenseText"); + invokeMethod(clazz, obj, "testInputStreamReadMatrixSparseCSV"); + invokeMethod(clazz, obj, "testInputStreamReadMatrixSparseText"); + invokeMethod(clazz, obj, "testInputStreamReadFrameDenseCSV"); + invokeMethod(clazz, obj, "testInputStreamReadFrameDenseText"); + invokeMethod(clazz, obj, "testInputStreamReadFrameSparseCSV"); + invokeMethod(clazz, obj, "testInputStreamReadFrameSparseText"); + invokeMethod(clazz, obj, "testInputStreamReadFrameDenseCSVMeta"); + invokeMethod(clazz, obj, "testInputStreamReadFrameDenseTextMeta"); + invokeMethod(clazz, obj, "testInputStreamReadFrameSparseCSVMeta"); + invokeMethod(clazz, obj, "testInputStreamReadFrameSparseTextMeta"); + invokeMethod(clazz, obj, "tearDown"); + + clazz = cl.loadClass("org.apache.sysml.test.integration.functions.jmlc.MulticlassSVMScoreTest"); + obj = clazz.newInstance(); + invokeMethod(clazz, obj, "setUpBase"); + invokeMethod(clazz, obj, "setUp"); + invokeMethod(clazz, obj, "testJMLCMulticlassScoreDense"); + invokeMethod(clazz, obj, "testJMLCMulticlassScoreSparse"); + invokeMethod(clazz, obj, "tearDown"); + + clazz = cl.loadClass("org.apache.sysml.test.integration.functions.jmlc.ReuseModelVariablesTest"); + obj = clazz.newInstance(); + invokeMethod(clazz, obj, "setUpBase"); + invokeMethod(clazz, obj, "setUp"); + invokeMethod(clazz, obj, "testJMLCScoreGLMDense"); + invokeMethod(clazz, obj, "testJMLCScoreGLMSparse"); + invokeMethod(clazz, obj, "testJMLCScoreGLMDenseReuse"); + invokeMethod(clazz, obj, "testJMLCScoreGLMSparseReuse"); + invokeMethod(clazz, obj, "testJMLCScoreMSVMDense"); + invokeMethod(clazz, obj, "testJMLCScoreMSVMSparse"); + invokeMethod(clazz, obj, "testJMLCScoreMSVMDenseReuse"); + invokeMethod(clazz, obj, "testJMLCScoreMSVMSparseReuse"); + invokeMethod(clazz, obj, "tearDown"); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + private static void invokeMethod(Class clazz, Object obj, String methodName) throws NoSuchMethodException, + SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + Method m = clazz.getMethod(methodName); + m.invoke(obj); + } + + public static double[][] matrix(int rows, int cols, double[] vals) { + double[][] matrix = new double[rows][cols]; + if ((vals == null) || (vals.length == 0)) { + return matrix; + } + for (int i = 0; i < vals.length; i++) { + matrix[i / cols][i % cols] = vals[i]; + } + return matrix; + } + + public static double[][] randomMatrix(int rows, int cols, double min, double max, double sparsity) { + double[][] matrix = new double[rows][cols]; + Random random = new Random(System.currentTimeMillis()); + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + if (random.nextDouble() > sparsity) { + continue; + } + matrix[i][j] = (random.nextDouble() * (max - min) + min); + } + } + return matrix; + } + + public static String displayMatrix(double[][] matrix) { + try { + return DataConverter.toString(DataConverter.convertToMatrixBlock(matrix)); + } catch (DMLRuntimeException e) { + return "N/A"; + } + } + + private static Connection getConfiguredConnection() { + Connection conn = new Connection(ConfigType.ALLOW_DYN_RECOMPILATION, ConfigType.PARALLEL_LOCAL_OR_REMOTE_PARFOR, + ConfigType.PARALLEL_CP_MATRIX_OPERATIONS); + return conn; + } + +} http://git-wip-us.apache.org/repos/asf/systemml/blob/139ddfee/src/main/java/org/apache/sysml/utils/lite/LiteCheck.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/lite/LiteCheck.java b/src/main/java/org/apache/sysml/utils/lite/LiteCheck.java new file mode 100644 index 0000000..64426ea --- /dev/null +++ b/src/main/java/org/apache/sysml/utils/lite/LiteCheck.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sysml.utils.lite; + +import java.io.IOException; +import java.io.InputStream; + +public class LiteCheck { + + private static Boolean lite = null; + + /** + * Return {@code true} if using lightweight SystemML jar, {@code false} + * otherwise. This is done by looking for an identifier file that is written + * to the lite jar. + * + * @return {@code true} if using lightweight SystemML jar, {@code false} + * otherwise + */ + public static boolean isLite() { + if (lite != null) { + return lite; + } + try (InputStream is = LiteCheck.class.getResourceAsStream("/" + BuildLite.LITE_JAR_IDENTIFIER_FILE)) { + lite = (is != null); + return lite; + } catch (IOException e) { + lite = false; + return lite; + } + } + +} http://git-wip-us.apache.org/repos/asf/systemml/blob/139ddfee/src/main/python/pre_setup.py ---------------------------------------------------------------------- diff --git a/src/main/python/pre_setup.py b/src/main/python/pre_setup.py index 12367e6..694b0c2 100644 --- a/src/main/python/pre_setup.py +++ b/src/main/python/pre_setup.py @@ -33,7 +33,8 @@ for file in os.listdir(os.path.join(root_dir, 'target')): if (fnmatch.fnmatch(file, 'systemml-*-SNAPSHOT.jar') or fnmatch.fnmatch(file, 'systemml-*.jar') and not (fnmatch.fnmatch(file, 'systemml-*javadoc.jar') or fnmatch.fnmatch(file, 'systemml-*sources.jar') - or fnmatch.fnmatch(file, 'systemml-*standalone.jar'))): + or fnmatch.fnmatch(file, 'systemml-*standalone.jar') + or fnmatch.fnmatch(file, 'systemml-*lite.jar'))): shutil.copyfile(os.path.join(root_dir, 'target', file), os.path.join(java_dir_full_path, file)) if fnmatch.fnmatch(file, 'systemml-*-SNAPSHOT-extra.jar') or fnmatch.fnmatch(file, 'systemml-*-extra.jar'):
