MAHOUT-1885: Inital commit of VCL bindings. closes apache/mahout#269 closes apache/mahout#261
Project: http://git-wip-us.apache.org/repos/asf/mahout/repo Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/034790cc Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/034790cc Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/034790cc Branch: refs/heads/master Commit: 034790cce40fcee2b7a875c482345d35f7c0fa8d Parents: 84e90ed Author: Andrew Palumbo <[email protected]> Authored: Wed Jan 25 20:23:28 2017 -0800 Committer: Andrew Palumbo <[email protected]> Committed: Wed Jan 25 20:23:28 2017 -0800 ---------------------------------------------------------------------- .travis.yml | 7 +- LICENSE.txt | 605 +++++++ bin/mahout | 26 + distribution/pom.xml | 109 ++ examples/bin/SparseSparseDrmTimer.mscala | 37 + flink/pom.xml | 6 + math-scala/pom.xml | 10 +- .../apache/mahout/math/backend/Backend.scala | 33 + .../mahout/math/backend/RootSolverFactory.scala | 87 + .../mahout/math/backend/SolverFactory.scala | 55 + .../mahout/math/backend/incore/package.scala | 17 + .../mahout/math/backend/jvm/JvmBackend.scala | 51 + .../apache/mahout/math/scalabindings/MMul.scala | 3 +- .../math/scalabindings/RLikeMatrixOps.scala | 14 +- .../mahout/math/scalabindings/package.scala | 3 + .../mahout/math/backend/BackendSuite.scala | 59 + .../scalabindings/RLikeMatrixOpsSuite.scala | 3 + pom.xml | 37 +- runtests.sh | 1 + spark-shell/pom.xml | 6 + .../sparkbindings/shell/MahoutSparkILoop.scala | 5 +- spark/pom.xml | 6 + spark/src/main/assembly/dependency-reduced.xml | 2 + .../apache/mahout/sparkbindings/blas/ABt.scala | 2 +- .../apache/mahout/sparkbindings/package.scala | 33 +- .../drivers/ItemSimilarityDriverSuite.scala | 1664 +++++++++--------- .../drivers/RowSimilarityDriverSuite.scala | 278 +-- .../TextDelimitedReaderWriterSuite.scala | 106 +- .../sparkbindings/SparkBindingsSuite.scala | 24 +- .../test/DistributedSparkSuite.scala | 2 +- viennacl-omp/linux-haswell.properties | 28 + viennacl-omp/linux-x86_64-viennacl.properties | 24 + viennacl-omp/pom.xml | 278 +++ viennacl-omp/runs | 32 + .../viennacl/openmp/javacpp/Functions.java | 103 ++ .../openmp/javacpp/LinalgFunctions.java | 86 + .../openmp/javacpp/MatrixTransExpression.scala | 34 + .../apache/mahout/viennacl/openmp/OMPMMul.scala | 449 +++++ .../openmp/javacpp/CompressedMatrix.scala | 125 ++ .../viennacl/openmp/javacpp/Context.scala | 58 + .../openmp/javacpp/DenseColumnMatrix.scala | 83 + .../openmp/javacpp/DenseRowMatrix.scala | 69 + .../openmp/javacpp/MatMatProdExpression.scala | 33 + .../openmp/javacpp/MatVecProdExpression.scala | 33 + .../viennacl/openmp/javacpp/MatrixBase.scala | 75 + .../viennacl/openmp/javacpp/MemHandle.scala | 34 + .../openmp/javacpp/ProdExpression.scala | 33 + .../javacpp/SrMatDnMatProdExpression.scala | 33 + .../viennacl/openmp/javacpp/VCLVector.scala | 115 ++ .../openmp/javacpp/VecMultExpression.scala | 32 + .../viennacl/openmp/javacpp/VectorBase.scala | 55 + .../apache/mahout/viennacl/openmp/package.scala | 434 +++++ .../mahout/viennacl/omp/ViennaCLSuiteOMP.scala | 249 +++ viennacl/linux-haswell.properties | 28 + viennacl/linux-x86_64-viennacl.properties | 24 + viennacl/pom.xml | 271 +++ .../viennacl/opencl/javacpp/Functions.java | 104 ++ .../opencl/javacpp/LinalgFunctions.java | 86 + .../opencl/javacpp/MatrixTransExpression.scala | 34 + .../apache/mahout/viennacl/opencl/GPUMMul.scala | 455 +++++ .../opencl/javacpp/CompressedMatrix.scala | 125 ++ .../viennacl/opencl/javacpp/Context.scala | 73 + .../opencl/javacpp/DenseColumnMatrix.scala | 83 + .../opencl/javacpp/DenseRowMatrix.scala | 86 + .../opencl/javacpp/MatMatProdExpression.scala | 33 + .../opencl/javacpp/MatVecProdExpression.scala | 33 + .../viennacl/opencl/javacpp/MatrixBase.scala | 75 + .../viennacl/opencl/javacpp/MemHandle.scala | 48 + .../opencl/javacpp/ProdExpression.scala | 33 + .../javacpp/SrMatDnMatProdExpression.scala | 33 + .../viennacl/opencl/javacpp/VCLVector.scala | 133 ++ .../opencl/javacpp/VecMultExpression.scala | 32 + .../viennacl/opencl/javacpp/VectorBase.scala | 57 + .../apache/mahout/viennacl/opencl/package.scala | 434 +++++ .../viennacl/opencl/ViennaCLSuiteVCL.scala | 427 +++++ 75 files changed, 7428 insertions(+), 1065 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 426d57e..36653d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: false +sudo: required cache: directories: @@ -33,8 +33,11 @@ before_install: - unzip -qq apache-maven-3.3.9-bin.zip - export M2_HOME=$PWD/apache-maven-3.3.9 - export PATH=$M2_HOME/bin:$PATH + - export MAHOUT_HOME=$PWD + - sudo apt-get -qq update + - sudo apt-get install -y libviennacl-dev script: - ./runtests.sh -- mvn javadoc:javadoc +#- mvn javadoc:javadoc http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/LICENSE.txt ---------------------------------------------------------------------- diff --git a/LICENSE.txt b/LICENSE.txt index 8ce7fff..110fb6e 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -799,3 +799,608 @@ The following license applies to the H2O package identification within third-party archives. Copyright 2012 0xdata, Inc + +================================================================ +The following applies to the ViennaCL library and files in the mahout-native-viennacl module +================================================================ + + Copyright (c) 2010-2016 Institute for Microelectronics, + Institute for Analysis and Scientific Computing, TU Wien. + Portions of this software are copyright by UChicago Argonne, LLC. + Argonne National Laboratory, with facilities in the state of Illinois, + is owned by The United States Government, and operated by UChicago Argonne, LLC + under provision of a contract with the Department of Energy. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + + =============================================================================== + The following applies to the bydeco/javacpp code and module + =============================================================================== + You may use this work under the terms of either the Apache License, + Version 2.0, or the GNU General Public License (GPL), either version 2, + or any later version, with "Classpath" exception (details below). + + You don't have to do anything special to choose one license or the other + and you don't have to notify anyone which license you are using. You are + free to use this work in any project (even commercial projects) as long + as the copyright header is left intact. + + =============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + =============================================================================== + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your + freedom to share and change it. By contrast, the GNU General Public + License is intended to guarantee your freedom to share and change free + software--to make sure the software is free for all its users. This + General Public License applies to most of the Free Software + Foundation's software and to any other program whose authors commit to + using it. (Some other Free Software Foundation software is covered by + the GNU Lesser General Public License instead.) You can apply it to + your programs, too. + + When we speak of free software, we are referring to freedom, not + price. Our General Public Licenses are designed to make sure that you + have the freedom to distribute copies of free software (and charge for + this service if you wish), that you receive source code or can get it + if you want it, that you can change the software or use pieces of it + in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid + anyone to deny you these rights or to ask you to surrender the rights. + These restrictions translate to certain responsibilities for you if you + distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether + gratis or for a fee, you must give the recipients all the rights that + you have. You must make sure that they, too, receive or can get the + source code. And you must show them these terms so they know their + rights. + + We protect your rights with two steps: (1) copyright the software, and + (2) offer you this license which gives you legal permission to copy, + distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain + that everyone understands that there is no warranty for this free + software. If the software is modified by someone else and passed on, we + want its recipients to know that what they have is not the original, so + that any problems introduced by others will not reflect on the original + authors' reputations. + + Finally, any free program is threatened constantly by software + patents. We wish to avoid the danger that redistributors of a free + program will individually obtain patent licenses, in effect making the + program proprietary. To prevent this, we have made it clear that any + patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and + modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains + a notice placed by the copyright holder saying it may be distributed + under the terms of this General Public License. The "Program", below, + refers to any such program or work, and a "work based on the Program" + means either the Program or any derivative work under copyright law: + that is to say, a work containing the Program or a portion of it, + either verbatim or with modifications and/or translated into another + language. (Hereinafter, translation is included without limitation in + the term "modification".) Each licensee is addressed as "you". + + Activities other than copying, distribution and modification are not + covered by this License; they are outside its scope. The act of + running the Program is not restricted, and the output from the Program + is covered only if its contents constitute a work based on the + Program (independent of having been made by running the Program). + Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's + source code as you receive it, in any medium, provided that you + conspicuously and appropriately publish on each copy an appropriate + copyright notice and disclaimer of warranty; keep intact all the + notices that refer to this License and to the absence of any warranty; + and give any other recipients of the Program a copy of this License + along with the Program. + + You may charge a fee for the physical act of transferring a copy, and + you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion + of it, thus forming a work based on the Program, and copy and + distribute such modifications or work under the terms of Section 1 + above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + + These requirements apply to the modified work as a whole. If + identifiable sections of that work are not derived from the Program, + and can be reasonably considered independent and separate works in + themselves, then this License, and its terms, do not apply to those + sections when you distribute them as separate works. But when you + distribute the same sections as part of a whole which is a work based + on the Program, the distribution of the whole must be on the terms of + this License, whose permissions for other licensees extend to the + entire whole, and thus to each and every part regardless of who wrote it. + + Thus, it is not the intent of this section to claim rights or contest + your rights to work written entirely by you; rather, the intent is to + exercise the right to control the distribution of derivative or + collective works based on the Program. + + In addition, mere aggregation of another work not based on the Program + with the Program (or with a work based on the Program) on a volume of + a storage or distribution medium does not bring the other work under + the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, + under Section 2) in object code or executable form under the terms of + Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + + The source code for a work means the preferred form of the work for + making modifications to it. For an executable work, complete source + code means all the source code for all modules it contains, plus any + associated interface definition files, plus the scripts used to + control compilation and installation of the executable. However, as a + special exception, the source code distributed need not include + anything that is normally distributed (in either source or binary + form) with the major components (compiler, kernel, and so on) of the + operating system on which the executable runs, unless that component + itself accompanies the executable. + + If distribution of executable or object code is made by offering + access to copy from a designated place, then offering equivalent + access to copy the source code from the same place counts as + distribution of the source code, even though third parties are not + compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program + except as expressly provided under this License. Any attempt + otherwise to copy, modify, sublicense or distribute the Program is + void, and will automatically terminate your rights under this License. + However, parties who have received copies, or rights, from you under + this License will not have their licenses terminated so long as such + parties remain in full compliance. + + 5. You are not required to accept this License, since you have not + signed it. However, nothing else grants you permission to modify or + distribute the Program or its derivative works. These actions are + prohibited by law if you do not accept this License. Therefore, by + modifying or distributing the Program (or any work based on the + Program), you indicate your acceptance of this License to do so, and + all its terms and conditions for copying, distributing or modifying + the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the + Program), the recipient automatically receives a license from the + original licensor to copy, distribute or modify the Program subject to + these terms and conditions. You may not impose any further + restrictions on the recipients' exercise of the rights granted herein. + You are not responsible for enforcing compliance by third parties to + this License. + + 7. If, as a consequence of a court judgment or allegation of patent + infringement or for any other reason (not limited to patent issues), + conditions are imposed on you (whether by court order, agreement or + otherwise) that contradict the conditions of this License, they do not + excuse you from the conditions of this License. If you cannot + distribute so as to satisfy simultaneously your obligations under this + License and any other pertinent obligations, then as a consequence you + may not distribute the Program at all. For example, if a patent + license would not permit royalty-free redistribution of the Program by + all those who receive copies directly or indirectly through you, then + the only way you could satisfy both it and this License would be to + refrain entirely from distribution of the Program. + + If any portion of this section is held invalid or unenforceable under + any particular circumstance, the balance of the section is intended to + apply and the section as a whole is intended to apply in other + circumstances. + + It is not the purpose of this section to induce you to infringe any + patents or other property right claims or to contest validity of any + such claims; this section has the sole purpose of protecting the + integrity of the free software distribution system, which is + implemented by public license practices. Many people have made + generous contributions to the wide range of software distributed + through that system in reliance on consistent application of that + system; it is up to the author/donor to decide if he or she is willing + to distribute software through any other system and a licensee cannot + impose that choice. + + This section is intended to make thoroughly clear what is believed to + be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in + certain countries either by patents or by copyrighted interfaces, the + original copyright holder who places the Program under this License + may add an explicit geographical distribution limitation excluding + those countries, so that distribution is permitted only in or among + countries not thus excluded. In such case, this License incorporates + the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions + of the General Public License from time to time. Such new versions will + be similar in spirit to the present version, but may differ in detail to + address new problems or concerns. + + Each version is given a distinguishing version number. If the Program + specifies a version number of this License which applies to it and "any + later version", you have the option of following the terms and conditions + either of that version or of any later version published by the Free + Software Foundation. If the Program does not specify a version number of + this License, you may choose any version ever published by the Free Software + Foundation. + + 10. If you wish to incorporate parts of the Program into other free + programs whose distribution conditions are different, write to the author + to ask for permission. For software which is copyrighted by the Free + Software Foundation, write to the Free Software Foundation; we sometimes + make exceptions for this. Our decision will be guided by the two goals + of preserving the free status of all derivatives of our free software and + of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY + FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN + OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES + PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED + OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS + TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE + PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, + REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING + WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR + REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, + INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING + OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED + TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY + YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER + PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest + possible use to the public, the best way to achieve this is to make it + free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest + to attach them to the start of each source file to most effectively + convey the exclusion of warranty; and each file should have at least + the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + Also add information on how to contact you by electronic and paper mail. + + If the program is interactive, make it output a short notice like this + when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + + The hypothetical commands `show w' and `show c' should show the appropriate + parts of the General Public License. Of course, the commands you use may + be called something other than `show w' and `show c'; they could even be + mouse-clicks or menu items--whatever suits your program. + + You should also get your employer (if you work as a programmer) or your + school, if any, to sign a "copyright disclaimer" for the program, if + necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + + This General Public License does not permit incorporating your program into + proprietary programs. If your program is a subroutine library, you may + consider it more useful to permit linking proprietary applications with the + library. If this is what you want to do, use the GNU Lesser General + Public License instead of this License. + + + "CLASSPATH" EXCEPTION TO THE GPL + + Linking this library statically or dynamically with other modules is making + a combined work based on this library. Thus, the terms and conditions of + the GNU General Public License cover the whole combination. + + As a special exception, the copyright holders of this library give you + permission to link this library with independent modules to produce an + executable, regardless of the license terms of these independent modules, + and to copy and distribute the resulting executable under terms of your + choice, provided that you also meet, for each linked independent module, + the terms and conditions of the license of that module. An independent + module is a module which is not derived from or based on this library. If + you modify this library, you may extend this exception to your version of + the library, but you are not obligated to do so. If you do not wish to do + so, delete this exception statement from your version. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/bin/mahout ---------------------------------------------------------------------- diff --git a/bin/mahout b/bin/mahout index c58e865..d88abe8 100755 --- a/bin/mahout +++ b/bin/mahout @@ -211,6 +211,21 @@ then CLASSPATH=${CLASSPATH}:$f; done + # viennacl jars- may or may not be available depending on build profile + for f in $MAHOUT_HOME/viennacl/target/mahout-native-viennacl_*.jar ; do + CLASSPATH=${CLASSPATH}:$f; + done + + # viennacl jars- may or may not be available depending on build profile + for f in $MAHOUT_HOME/viennacl-omp/target/mahout-native-viennacl-omp_*.jar ; do + CLASSPATH=${CLASSPATH}:$f; + done + + # viennacl jars- may or may not be available depending on build profile + for f in $MAHOUT_HOME/viennacl-omp/target/mahout-native-viennacl-omp_*.jar ; do + CLASSPATH=${CLASSPATH}:$f; + done + SPARK_CP_BIN="${MAHOUT_HOME}/bin/compute-classpath.sh" if [ -x "${SPARK_CP_BIN}" ]; then SPARK_CLASSPATH=$("${SPARK_CP_BIN}" 2>/dev/null) @@ -230,6 +245,17 @@ then fi fi + # add vcl jars at any point. + # viennacl jars- may or may not be available depending on build profile + for f in $MAHOUT_HOME/viennacl/target/mahout-native-viennacl_*.jar ; do + CLASSPATH=${CLASSPATH}:$f; + done + + # viennacl jars- may or may not be available depending on build profile + for f in $MAHOUT_HOME/viennacl-omp/target/mahout-native-viennacl-omp_*.jar ; do + CLASSPATH=${CLASSPATH}:$f; + done + # add release dependencies to CLASSPATH for f in $MAHOUT_HOME/lib/*.jar; do CLASSPATH=${CLASSPATH}:$f; http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/distribution/pom.xml ---------------------------------------------------------------------- diff --git a/distribution/pom.xml b/distribution/pom.xml index 536c76f..b8295ba 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -78,6 +78,106 @@ <mahout.skip.distribution>false</mahout.skip.distribution> </properties> </profile> + <profile> + <id>viennacl</id> + <dependencies> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-math</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-integration</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-hdfs</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-mr</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-examples</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-spark_${scala.compat.version}</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-flink_${scala.compat.version}</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-spark-shell_${scala.compat.version}</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-math-scala_${scala.compat.version}</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-native-viennacl_${scala.compat.version}</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-native-viennacl-omp_${scala.compat.version}</artifactId> + <version>0.13.0-SNAPSHOT</version> + </dependency> + </dependencies> + </profile> + <profile> + <id>viennacl-omp</id> + <dependencies> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-math</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-integration</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-hdfs</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-mr</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-examples</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-spark_${scala.compat.version}</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-flink_${scala.compat.version}</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-spark-shell_${scala.compat.version}</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-math-scala_${scala.compat.version}</artifactId> + </dependency> + <dependency> + <groupId>org.apache.mahout</groupId> + <artifactId>mahout-native-viennacl-omp_${scala.compat.version}</artifactId> + <version>0.13.0-SNAPSHOT</version> + </dependency> + </dependencies> + + </profile> + + + </profiles> <dependencies> @@ -117,5 +217,14 @@ <groupId>org.apache.mahout</groupId> <artifactId>mahout-math-scala_${scala.compat.version}</artifactId> </dependency> + <!--Viennacl is not part of the Default build currently.--> + <!--<dependency>--> + <!--<groupId>org.apache.mahout</groupId>--> + <!--<artifactId>mahout-native-viennacl_${scala.compat.version}</artifactId>--> + <!--</dependency>--> + <!--<dependency>--> + <!--<groupId>org.apache.mahout</groupId>--> + <!--<artifactId>mahout-native-viennacl-omp_${scala.compat.version}</artifactId>--> + <!--</dependency>--> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/examples/bin/SparseSparseDrmTimer.mscala ---------------------------------------------------------------------- diff --git a/examples/bin/SparseSparseDrmTimer.mscala b/examples/bin/SparseSparseDrmTimer.mscala new file mode 100644 index 0000000..d01bf3a --- /dev/null +++ b/examples/bin/SparseSparseDrmTimer.mscala @@ -0,0 +1,37 @@ + + +def timeSparseDRMMMul(m: Int, n: Int, s: Int, para: Int, pctDense: Double = .20, seed: Long = 1234L): Long = { + + + + val drmA = drmParallelizeEmpty(m , s, para).mapBlock(){ + case (keys,block:Matrix) => + val R = scala.util.Random + R.setSeed(seed) + val blockB = new SparseRowMatrix(block.nrow, block.ncol) + blockB := {x => if (R.nextDouble > pctDense) R.nextDouble else x } + (keys -> blockB) + } + val drmB = drmParallelizeEmpty(s , n, para).mapBlock(){ + case (keys,block:Matrix) => + val R = scala.util.Random + R.setSeed(seed + 1) + val blockB = new SparseRowMatrix(block.nrow, block.ncol) + blockB := {x => if (R.nextDouble > pctDense) R.nextDouble else x } + (keys -> blockB) + } + + var time = System.currentTimeMillis() + + val drmC = drmA %*% drmB + + // trigger computation + drmC.numRows() + + time = System.currentTimeMillis() - time + + time + +} + + http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/flink/pom.xml ---------------------------------------------------------------------- diff --git a/flink/pom.xml b/flink/pom.xml index 7857210..f9ec56b 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -153,6 +153,12 @@ <artifactId>mahout-math-scala_${scala.compat.version}</artifactId> </dependency> + <dependency> + <groupId>org.bytedeco</groupId> + <artifactId>javacpp</artifactId> + <version>1.2.2</version> + </dependency> + <!-- enforce current version of kryo as of 0.10.1--> <dependency> <groupId>com.esotericsoftware.kryo</groupId> http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/pom.xml ---------------------------------------------------------------------- diff --git a/math-scala/pom.xml b/math-scala/pom.xml index 9eb7e80..da1c05b 100644 --- a/math-scala/pom.xml +++ b/math-scala/pom.xml @@ -102,6 +102,9 @@ <plugin> <groupId>org.scalatest</groupId> <artifactId>scalatest-maven-plugin</artifactId> + <configuration> + <argLine>-Xmx4g</argLine> + </configuration> <executions> <execution> <id>test</id> @@ -147,10 +150,13 @@ <artifactId>scalatest_${scala.compat.version}</artifactId> </dependency> - + <dependency> + <groupId>org.scala-lang</groupId> + <artifactId>scala-reflect</artifactId> + <version>${scala.version}</version> + </dependency> </dependencies> - <profiles> <profile> <id>mahout-release</id> http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/backend/Backend.scala ---------------------------------------------------------------------- diff --git a/math-scala/src/main/scala/org/apache/mahout/math/backend/Backend.scala b/math-scala/src/main/scala/org/apache/mahout/math/backend/Backend.scala new file mode 100644 index 0000000..9dfb7f2 --- /dev/null +++ b/math-scala/src/main/scala/org/apache/mahout/math/backend/Backend.scala @@ -0,0 +1,33 @@ +package org.apache.mahout.math.backend + +import org.apache.mahout.math.backend.jvm.JvmBackend + +import collection._ +import scala.reflect.{ClassTag, classTag} +import jvm.JvmBackend + +/** + * == Overview == + * + * Backend representing collection of in-memory solvers or distributed operators. + * + * == Note to implementors == + * + * Backend is expected to initialize & verify its own viability lazily either upon first time the + * class is loaded, or upon the first invocation of any of its methods. After that, the value of + * [[Backend.isAvailable]] must be cached and defined. + * + * A Backend is also a [[SolverFactory]] of course in a sense that it enumerates solvers made + * available via the backend. + */ +trait Backend extends SolverFactory { + + /** + * If backend has loaded (lazily) ok and verified its availability/functionality, + * this must return `true`. + * + * @return `true` + */ + def isAvailable: Boolean + +} http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/backend/RootSolverFactory.scala ---------------------------------------------------------------------- diff --git a/math-scala/src/main/scala/org/apache/mahout/math/backend/RootSolverFactory.scala b/math-scala/src/main/scala/org/apache/mahout/math/backend/RootSolverFactory.scala new file mode 100644 index 0000000..253a435 --- /dev/null +++ b/math-scala/src/main/scala/org/apache/mahout/math/backend/RootSolverFactory.scala @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.mahout.math.backend + +import org.apache.mahout.logging._ +import org.apache.mahout.math.backend.jvm.JvmBackend +import org.apache.mahout.math.scalabindings.{MMul, _} + +import scala.collection._ +import scala.reflect.{ClassTag, classTag} + + +final object RootSolverFactory extends SolverFactory { + + import org.apache.mahout.math.backend.incore._ + + implicit val logger = getLog(RootSolverFactory.getClass) + + private val solverTagsToScan = + classTag[MMulSolver] :: + classTag[MMulSparseSolver] :: + classTag[MMulDenseSolver] :: + Nil + + private val defaultBackendPriority = + JvmBackend.getClass.getName :: Nil + + private def initBackends(): Unit = { + + } + + //////////////////////////////////////////////////////////// + + // TODO: MAHOUT-1909: lazy initialze the map. Query backends. Build resolution rules. + override protected[backend] val solverMap = new mutable.HashMap[ClassTag[_], Any]() + validateMap() + + + // default is JVM + var clazz: MMBinaryFunc = MMul + + // eventually match on implicit Classtag . for now. just take as is. + // this is a bit hacky, Shoud not be doing onlytry/catch here.. + def getOperator[C: ClassTag]: MMBinaryFunc = { + + try { + // TODO: fix logging properties so that we're not mimicing as we are here. + println("[INFO] Creating org.apache.mahout.viennacl.opencl.GPUMMul solver") + clazz = Class.forName("org.apache.mahout.viennacl.opencl.GPUMMul$").getField("MODULE$").get(null).asInstanceOf[MMBinaryFunc] + println("[INFO] Successfully created org.apache.mahout.viennacl.opencl.GPUMMul solver") + + } catch { + case x: Exception => + println("[WARN] Unable to create class GPUMMul: attempting OpenMP version") + // println(x.getMessage) + try { + // attempt to instantiate the OpenMP version, assuming weâve + // created a separate OpenMP-only module (none exist yet) + println("[INFO] Creating org.apache.mahout.viennacl.openmp.OMPMMul solver") + clazz = Class.forName("org.apache.mahout.viennacl.openmp.OMPMMul$").getField("MODULE$").get(null).asInstanceOf[MMBinaryFunc] + println("[INFO] Successfully created org.apache.mahout.viennacl.openmp.OMPMMul solver") + + } catch { + case xx: Exception => + println(xx.getMessage) + // fall back to JVM Dont need to Dynamicly assign MMul is in the same package. + println("[INFO] Unable to create class OMPMMul: falling back to java version") + clazz = MMul + } + } + clazz + } +} http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/backend/SolverFactory.scala ---------------------------------------------------------------------- diff --git a/math-scala/src/main/scala/org/apache/mahout/math/backend/SolverFactory.scala b/math-scala/src/main/scala/org/apache/mahout/math/backend/SolverFactory.scala new file mode 100644 index 0000000..756b971 --- /dev/null +++ b/math-scala/src/main/scala/org/apache/mahout/math/backend/SolverFactory.scala @@ -0,0 +1,55 @@ +package org.apache.mahout.math.backend + +import scala.collection.{Iterable, Map} +import scala.reflect.{ClassTag, classTag} + +/** + * == Overview == + * + * Solver factory is an essence a collection of lazily initialized strategy singletons solving some + * (any) problem in context of the Mahout project. + * + * We intend to use it _mainly_ for problems that are super-linear problems, and often involve more + * than one argument (operand). + * + * The main method to probe for an available solver is [[RootSolverFactory.getSolver]]. + */ +trait SolverFactory { + /** + * We take an implicit context binding, the classTag, of the trait of the solver desired. + * + * == Note to callers == + * + * Due to Scala semantics, it is usually not enough to request a solver via merely {{{ + * val s:SolverType = backend.getSolver + * }}} but instead requires an explicit solver tag, i.e.: {{{ + * val s = backend.getSolver[SolverType] + * }}} + * + * + */ + def getSolver[S: ClassTag]: Option[S] = { + solverMap.get(classTag[S]).flatMap { + _ match { + case s: S â Some(s) + case _ â None + } + } + } + + lazy val availableSolverTags: Iterable[ClassTag[_]] = solverMap.keySet + + + + protected[backend] val solverMap: Map[ClassTag[_], Any] + + protected[backend] def validateMap(): Unit = { + + for ((tag, instance) â solverMap) { + require(tag.runtimeClass.isAssignableFrom(instance.getClass), + s"Solver implementation class `${instance.getClass.getName}` is not a subclass of solver trait `${tag}`.") + + } + } + +} http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/backend/incore/package.scala ---------------------------------------------------------------------- diff --git a/math-scala/src/main/scala/org/apache/mahout/math/backend/incore/package.scala b/math-scala/src/main/scala/org/apache/mahout/math/backend/incore/package.scala new file mode 100644 index 0000000..1bb4480 --- /dev/null +++ b/math-scala/src/main/scala/org/apache/mahout/math/backend/incore/package.scala @@ -0,0 +1,17 @@ +package org.apache.mahout.math.backend + +import org.apache.mahout.math.scalabindings.{MMBinaryFunc, MMUnaryFunc} + +package object incore { + + trait MMulSolver extends MMBinaryFunc + trait MMulDenseSolver extends MMulSolver + trait MMulSparseSolver extends MMulSolver + trait AAtSolver extends MMUnaryFunc + trait AAtDenseSolver extends AAtSolver + trait AAtSparseSolver extends AAtSolver + trait AtASolver extends MMUnaryFunc + trait AtADenseSolver extends AtASolver + trait AtASparseSolver extends AtASolver + +} http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/backend/jvm/JvmBackend.scala ---------------------------------------------------------------------- diff --git a/math-scala/src/main/scala/org/apache/mahout/math/backend/jvm/JvmBackend.scala b/math-scala/src/main/scala/org/apache/mahout/math/backend/jvm/JvmBackend.scala new file mode 100644 index 0000000..6588243 --- /dev/null +++ b/math-scala/src/main/scala/org/apache/mahout/math/backend/jvm/JvmBackend.scala @@ -0,0 +1,51 @@ +package org.apache.mahout.math.backend.jvm + +import org.apache.mahout.math._ +import scalabindings._ +import RLikeOps._ +import org.apache.mahout.math.backend.Backend +import org.apache.mahout.math.scalabindings.MMul + +import scala.collection.Map +import scala.reflect._ + +object JvmBackend extends Backend { + + import org.apache.mahout.math.backend.incore._ + + /** + * If backend has loaded (lazily) ok and verified its availability/functionality, + * this must return `true`. + * + * @return `true` + */ + override def isAvailable: Boolean = true + + // TODO: In a future release, Refactor MMul optimizations into this object + override protected[backend] val solverMap: Map[ClassTag[_], Any] = Map( + classTag[MMulSolver] â MMul + // classTag[MMulDenseSolver] â MMul, + // classTag[MMulSparseSolver] â MMul, + // classTag[AtASolver] â new AtASolver { + // override def apply(a: Matrix, r: Option[Matrix]): Matrix = MMul(a.t, a, r) + // }// , + // classTag[AtADenseSolver] â { (a: Matrix, r: Option[Matrix]) â MMul(a.t, a, r) }, + // classTag[AtASparseSolver] â { (a: Matrix, r: Option[Matrix]) â MMul(a.t, a, r) }, + // classTag[AAtSolver] â { (a: Matrix, r: Option[Matrix]) â MMul(a, a.t, r) }, + // classTag[AAtDenseSolver] â { (a: Matrix, r: Option[Matrix]) â MMul(a, a.t, r) }, + // classTag[AAtSparseSolver] â { (a: Matrix, r: Option[Matrix]) â MMul(a, a.t, r) } + ) + validateMap() + + private val mmulSolver = new MMulSolver with MMulDenseSolver with MMulSparseSolver { + override def apply(a: Matrix, b: Matrix, r: Option[Matrix]): Matrix = MMul(a, b, r) + } + + private val ataSolver = new AtASolver with AtADenseSolver with AtASparseSolver { + override def apply(a: Matrix, r: Option[Matrix]): Matrix = MMul(a.t, a, r) + } + + private val aatSolver = new AAtSolver { + override def apply(a: Matrix, r: Option[Matrix]): Matrix = MMul(a, a.t, r) + } +} http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/MMul.scala ---------------------------------------------------------------------- diff --git a/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/MMul.scala b/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/MMul.scala index 2938e5d..f9bda8a 100644 --- a/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/MMul.scala +++ b/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/MMul.scala @@ -22,10 +22,11 @@ import org.apache.mahout.math.flavor.{BackEnum, TraversingStructureEnum} import org.apache.mahout.math.function.Functions import RLikeOps._ import org.apache.mahout.logging._ +import org.apache.mahout.math.backend.incore.MMulSolver import scala.collection.JavaConversions._ -object MMul extends MMBinaryFunc { +object MMul extends MMulSolver { private final implicit val log = getLog(MMul.getClass) http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOps.scala ---------------------------------------------------------------------- diff --git a/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOps.scala b/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOps.scala index e994e31..3ba6ce0 100644 --- a/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOps.scala +++ b/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOps.scala @@ -17,14 +17,24 @@ package org.apache.mahout.math.scalabindings import org.apache.mahout.math.function.Functions -import org.apache.mahout.math.{Vector, Matrix} +import org.apache.mahout.math.{Matrix, Vector} + import scala.collection.JavaConversions._ import RLikeOps._ +import org.apache.mahout.math.backend.RootSolverFactory +import org.apache.mahout.math.scalabindings._ + class RLikeMatrixOps(m: Matrix) extends MatrixOps(m) { /** Structure-optimized mmul */ - def %*%(that: Matrix) = MMul(m, that, None) + + implicit var solverOperator: opMMulSolver = _ + + // get the solver matching the implicit variable solverOperator + def mmulSolver = RootSolverFactory.getOperator + + def %*%(that: Matrix) = mmulSolver(m, that, None) def :%*%(that:Matrix) = %*%(that) http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/package.scala ---------------------------------------------------------------------- diff --git a/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/package.scala b/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/package.scala index 8b1ce65..4115091 100644 --- a/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/package.scala +++ b/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/package.scala @@ -365,6 +365,9 @@ package object scalabindings { type VMBinaryFunc = (Vector, Matrix, Option[Matrix]) â Matrix type MDBinaryFunc = (Matrix, Double, Option[Matrix]) â Matrix + trait opMMulSolver extends MMBinaryFunc { + + } ///////////////////////////////////// // Miscellaneous in-core utilities http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/test/scala/org/apache/mahout/math/backend/BackendSuite.scala ---------------------------------------------------------------------- diff --git a/math-scala/src/test/scala/org/apache/mahout/math/backend/BackendSuite.scala b/math-scala/src/test/scala/org/apache/mahout/math/backend/BackendSuite.scala new file mode 100644 index 0000000..ba6e145 --- /dev/null +++ b/math-scala/src/test/scala/org/apache/mahout/math/backend/BackendSuite.scala @@ -0,0 +1,59 @@ +package org.apache.mahout.math.backend + +import org.apache.mahout.math.backend.jvm.JvmBackend +import org.scalatest.{FunSuite, Matchers} + +import scala.collection.mutable +import scala.reflect.{ClassTag, classTag} + +class BackendSuite extends FunSuite with Matchers { + + test("GenericBackend") { + + trait MySolverTrait1 { def myMethod1 = Unit } + + + trait MySolverTrait2 + + class MySolverImpl1 extends MySolverTrait1 { + } + + class MySolverImpl2 extends MySolverTrait2 + + // My dummy backend supporting to trait solvers filled with 2 dummy implementations of these + // traits should be able to serve based on their solver traits. + val myBackend = new Backend { + + override def isAvailable: Boolean = true + + override val solverMap = new mutable.HashMap[ClassTag[_], Any]() + + solverMap ++= Map( + classTag[MySolverTrait1] â new MySolverImpl1, + classTag[MySolverTrait2] â new MySolverImpl2 + ) + + validateMap() + } + + myBackend.getSolver shouldBe None + + val mySolver1 = myBackend.getSolver[MySolverTrait1] + + // This is indeed solver1 trait type: + mySolver1.get.myMethod1 + mySolver1.get.isInstanceOf[MySolverImpl1] shouldBe true + + // Validator should not allow non-subclasses in implementation. + an [IllegalArgumentException] mustBe thrownBy { + myBackend.solverMap(classTag[MySolverTrait2]) = 0 + myBackend.validateMap() + } + } + + test("JvmBackend") { + // Just create JVM backend and validate. + JvmBackend.validateMap() + } + +} http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/test/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOpsSuite.scala ---------------------------------------------------------------------- diff --git a/math-scala/src/test/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOpsSuite.scala b/math-scala/src/test/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOpsSuite.scala index b44e295..6dc8207 100644 --- a/math-scala/src/test/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOpsSuite.scala +++ b/math-scala/src/test/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOpsSuite.scala @@ -120,6 +120,9 @@ class RLikeMatrixOpsSuite extends FunSuite with MahoutSuite { System.currentTimeMillis() - ms } + + // We're not using GPUMMul or OMPMMul in math-scala so dont need to worry about + // changing it in this method def getMmulAvgs(mxA: Matrix, mxB: Matrix, n: Int) = { var control: Matrix = null http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 8e7bfe8..72bd766 100644 --- a/pom.xml +++ b/pom.xml @@ -250,6 +250,13 @@ <version>${project.version}</version> </dependency> + <dependency> + <artifactId>mahout-native-viennacl_${scala.compat.version}</artifactId> + <groupId>${project.groupId}</groupId> + <version>${project.version}</version> + </dependency> + + <!-- 3rd party --> <dependency> <groupId>org.apache.hadoop</groupId> @@ -845,25 +852,27 @@ <module>spark-shell</module> <module>flink</module> <module>h2o</module> + <module>viennacl</module> + <module>viennacl-omp</module> </modules> <profiles> + <profile> - <id>hadoop1</id> - <properties> - <hadoop.classifier>hadoop1</hadoop.classifier> - <hadoop.version>1.2.1</hadoop.version> - </properties> - <dependencyManagement> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>${hadoop.version}</version> - </dependency> - </dependencies> - </dependencyManagement> + <id>viennacl</id> + <modules> + <module>viennacl</module> + <module>viennacl-omp</module> + </modules> </profile> + + <profile> + <id>viennacl-omp</id> + <modules> + <module>viennacl-omp</module> + </modules> + </profile> + <profile> <id>hadoop2</id> <activation> http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/runtests.sh ---------------------------------------------------------------------- diff --git a/runtests.sh b/runtests.sh index daa36ea..642ae54 100755 --- a/runtests.sh +++ b/runtests.sh @@ -33,6 +33,7 @@ cd hdfs && mvn test >> $BUILD_OUTPUT 2>&1 cd ../math && mvn test >> $BUILD_OUTPUT 2>&1 cd ../math-scala && mvn test >> $BUILD_OUTPUT 2>&1 cd ../spark && mvn test >> $BUILD_OUTPUT 2>&1 +#cd ../viennacl && mvn test >> $BUILD_OUTPUT 2>&1 #cd ../flink && mvn test >> $BUILD_OUTPUT 2>&1 #cd ../h2o && mvn test >> $BUILD_OUTPUT 2>&1 http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark-shell/pom.xml ---------------------------------------------------------------------- diff --git a/spark-shell/pom.xml b/spark-shell/pom.xml index 732c39b..8b40dc3 100644 --- a/spark-shell/pom.xml +++ b/spark-shell/pom.xml @@ -117,6 +117,12 @@ <scope>test</scope> </dependency> + <dependency> + <groupId>org.bytedeco</groupId> + <artifactId>javacpp</artifactId> + <version>1.2.2</version> + </dependency> + <!-- 3rd-party --> <dependency> http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala ---------------------------------------------------------------------- diff --git a/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala b/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala index 0c9163a..422af76 100644 --- a/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala +++ b/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala @@ -95,7 +95,8 @@ class MahoutSparkILoop extends SparkILoop { masterUrl = master, appName = "Mahout Spark Shell", customJars = jars, - sparkConf = conf + sparkConf = conf, + addMahoutJars = true ) _interp.sparkContext = sdc @@ -162,7 +163,7 @@ class MahoutSparkILoop extends SparkILoop { _ __ ___ __ _| |__ ___ _ _| |_ | '_ ` _ \ / _` | '_ \ / _ \| | | | __| | | | | | | (_| | | | | (_) | |_| | |_ - |_| |_| |_|\__,_|_| |_|\___/ \__,_|\__| version 0.12.2 + |_| |_| |_|\__,_|_| |_|\___/ \__,_|\__| version 0.13.0 """) import Properties._ http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark/pom.xml ---------------------------------------------------------------------- diff --git a/spark/pom.xml b/spark/pom.xml index f965d38..4e75a62 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -160,6 +160,12 @@ </dependency> <dependency> + <groupId>org.bytedeco</groupId> + <artifactId>javacpp</artifactId> + <version>1.2.2</version> + </dependency> + + <dependency> <groupId>org.apache.mahout</groupId> <artifactId>mahout-hdfs</artifactId> <exclusions> http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark/src/main/assembly/dependency-reduced.xml ---------------------------------------------------------------------- diff --git a/spark/src/main/assembly/dependency-reduced.xml b/spark/src/main/assembly/dependency-reduced.xml index a3044da..f981639 100644 --- a/spark/src/main/assembly/dependency-reduced.xml +++ b/spark/src/main/assembly/dependency-reduced.xml @@ -42,6 +42,8 @@ <include>com.tdunning:t-digest</include> <include>org.apache.commons:commons-math3</include> <include>it.unimi.dsi:fastutil</include> + <include>org.apache.mahout:mahout-native-viennacl_2.10</include> + <include>org.bytedeco:javacpp</include> </includes> </dependencySet> </dependencySets> http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/ABt.scala ---------------------------------------------------------------------- diff --git a/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/ABt.scala b/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/ABt.scala index b57d8ae..ba2adc9 100644 --- a/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/ABt.scala +++ b/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/ABt.scala @@ -119,7 +119,7 @@ object ABt { createCombiner = (t: (Array[K], Array[Int], Matrix)) => { val (rowKeys, colKeys, block) = t - val comb = if (block.getFlavor == MatrixFlavor.SPARSELIKE) { + val comb = if (!densityAnalysis(block)) { new SparseMatrix(prodNCol, block.nrow).t } else { new DenseMatrix(prodNCol, block.nrow).t http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala ---------------------------------------------------------------------- diff --git a/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala b/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala index 8fd77e6..7da4729 100644 --- a/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala +++ b/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala @@ -64,23 +64,28 @@ package object sparkbindings { try { - if (addMahoutJars) { + // when not including the artifact, eg. for viennacl , we always need + // to load all mahout jars + // will need to handle this somehow. + + // if (addMahoutJars) { // context specific jars val mcjars = findMahoutContextJars(closeables) - if (log.isDebugEnabled) { +// if (log.isDebugEnabled) { log.debug("Mahout jars:") mcjars.foreach(j => log.debug(j)) - } +// } sparkConf.setJars(jars = mcjars.toSeq ++ customJars) if (!(customJars.size > 0)) sparkConf.setJars(customJars.toSeq) - } else { +// } else { // In local mode we don't care about jars, do we? - sparkConf.setJars(customJars.toSeq) - } + // yes adding jars always now since we are not including the artifacts + // sparkConf.setJars(customJars.toSeq) +// } sparkConf.setAppName(appName).setMaster(masterUrl).set("spark.serializer", "org.apache.spark.serializer.KryoSerializer").set("spark.kryo.registrator", @@ -254,7 +259,11 @@ package object sparkbindings { j.matches(".*mahout-hdfs-\\d.*\\.jar") || // no need for mapreduce jar in Spark // j.matches(".*mahout-mr-\\d.*\\.jar") || - j.matches(".*mahout-spark_\\d.*\\.jar") + j.matches(".*mahout-spark_\\d.*\\.jar") || + // vcl jars: mahout-native-viennacl_2.10.jar, + // mahout-native-viennacl-omp_2.10.jar + j.matches(".*mahout-native-viennacl_\\d.*\\\\.jar") || + j.matches(".*mahout-native-viennacl-omp_\\d.*\\.jar") ) // Tune out "bad" classifiers .filter(n => @@ -264,11 +273,11 @@ package object sparkbindings { // During maven tests, the maven classpath also creeps in for some reason !n.matches(".*/.m2/.*") ) - /* verify jar passed to context - log.info("\n\n\n") - mcjars.foreach(j => log.info(j)) - log.info("\n\n\n") - */ + /* verify jar passed to context */ +// info("\n\n\n") +// mcjars.foreach(j => info(j)) +// info("\n\n\n") + /**/ mcjars }
