Author: lewismc Date: Thu May 21 17:14:24 2015 New Revision: 1680929 URL: http://svn.apache.org/r1680929 Log: NUTCH-1923 Nutch + Cassandra Docker
Added: nutch/branches/2.x/docker/cassandra/ nutch/branches/2.x/docker/cassandra/LICENSE nutch/branches/2.x/docker/cassandra/README.md nutch/branches/2.x/docker/cassandra/bin/ nutch/branches/2.x/docker/cassandra/bin/build.sh (with props) nutch/branches/2.x/docker/cassandra/bin/docker-data/ nutch/branches/2.x/docker/cassandra/bin/docker-data/cassandra/ nutch/branches/2.x/docker/cassandra/bin/docker-data/cassandra/.gitkeep (with props) nutch/branches/2.x/docker/cassandra/bin/docker-data/nutch/ nutch/branches/2.x/docker/cassandra/bin/docker-data/nutch/.gitkeep (with props) nutch/branches/2.x/docker/cassandra/bin/ipof.sh (with props) nutch/branches/2.x/docker/cassandra/bin/nodes.sh (with props) nutch/branches/2.x/docker/cassandra/bin/restart.sh (with props) nutch/branches/2.x/docker/cassandra/bin/start.sh (with props) nutch/branches/2.x/docker/cassandra/bin/stop.sh (with props) nutch/branches/2.x/docker/cassandra/cassandra/ nutch/branches/2.x/docker/cassandra/cassandra/Dockerfile (with props) nutch/branches/2.x/docker/cassandra/cassandra/LICENSE (with props) nutch/branches/2.x/docker/cassandra/cassandra/README.md (with props) nutch/branches/2.x/docker/cassandra/cassandra/bootstrap.sh (with props) nutch/branches/2.x/docker/cassandra/nutch/ nutch/branches/2.x/docker/cassandra/nutch/Dockerfile (with props) nutch/branches/2.x/docker/cassandra/nutch/LICENSE (with props) nutch/branches/2.x/docker/cassandra/nutch/README.md (with props) nutch/branches/2.x/docker/cassandra/nutch/bootstrap.sh (with props) nutch/branches/2.x/docker/cassandra/nutch/config/ nutch/branches/2.x/docker/cassandra/nutch/config/nutch-site.xml (with props) nutch/branches/2.x/docker/cassandra/nutch/plugin/ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/.gitignore (with props) nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/LICENSE (with props) nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/README.md (with props) nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/build.xml (with props) nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/ivy.xml (with props) nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/plugin.xml (with props) nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/src/ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/src/java/ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/src/java/org/ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/src/java/org/apache/ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/src/java/org/apache/nutch/ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/src/java/org/apache/nutch/indexer/ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/src/java/org/apache/nutch/indexer/html/ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/src/java/org/apache/nutch/indexer/html/HtmlIndexingFilter.java (with props) nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/src/java/org/apache/nutch/indexer/html/package.html (with props) nutch/branches/2.x/docker/cassandra/nutch/testUrls/ nutch/branches/2.x/docker/cassandra/nutch/testUrls/seed.txt (with props) Added: nutch/branches/2.x/docker/cassandra/LICENSE URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/LICENSE?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/LICENSE (added) +++ nutch/branches/2.x/docker/cassandra/LICENSE Thu May 21 17:14:24 2015 @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + Added: nutch/branches/2.x/docker/cassandra/README.md URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/README.md?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/README.md (added) +++ nutch/branches/2.x/docker/cassandra/README.md Thu May 21 17:14:24 2015 @@ -0,0 +1,73 @@ +#Apache Nutch 2.x with Cassandra on Docker +======================= + +This project is 3 Docker containers running Apache Nutch 2.x configured with Cassandra storage. + +Due to the lack of integration information between Nutch 2.x / Cassandra, Mohamed Meabed (@Meabed) developed these docker containers with configuration and integration between them. + +This is project is fully operational but its still experimental, any feedback, suggestions should be directed to d...@nutch.apache.org and contribution(s) will be highly appreciated! + +##Usage notes: + +1. Build the images and start the containers " NOTE: for Mac OS running boot2docker, Please read the Notes section Below ". + +``` +# Build the images ( this will build the application ) +./bin/build.sh + +# Start all containers with data folders from scripts +./bin/start.sh + +# stop all containers +./bin/stop.sh + +# restart containers +./bin/restart.sh + +``` +2. Start Crawling with Nutch 2.X. +``` +# Run the crawler, You can use docker exec command, or you can docker attach to the container and run the commands there, or use docker-enter if you are using Mac OS + +docker exec NUTCH01 /opt/nutch/bin/crawl /opt/nutch/testUrls test_crawl 3 +# OR + +docker-enter NUTCH01 +root@9ec43c388769:/# cd opt/nutch +root@9ec43c388769:/opt/nutch# ./bin/crawl +Usage: crawl <seedDir> <crawlID> [<solrUrl>] <numberOfRounds> +root@9ec43c388769:/opt/nutch# ./bin/crawl testUrls test_crawl 3 + + +``` +##NOTES: + +Nutch 2.x Container name : NUTCH01 + +Cassandra Container name : CASS01 + +Cassandra installed with OpsCenter + + +##MAC OSx notes +- you need to mount data folders to your VirtualMachine to be able to get persistent data every time you run this application. +- You might need to install docker-enter for easier access to the containers + +``` +mkdir ~/docker-data +mkdir ~/docker-data/cassandra +mkdir ~/docker-data/nutch + +chmod -R 777 ~/docker-data/ + +VBoxManage sharedfolder add boot2docker-vm -name home -hostpath ~/ + +boot2docker up +boot2docker ssh + +#mkdir /data +#mount -t vboxsf -o uid=1000,gid=50 data /data +#vi /etc/fstab +#data /data vboxsf rw,nodev,relatime 0 0 +#docker-enter +``` Added: nutch/branches/2.x/docker/cassandra/bin/build.sh URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/bin/build.sh?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/bin/build.sh (added) +++ nutch/branches/2.x/docker/cassandra/bin/build.sh Thu May 21 17:14:24 2015 @@ -0,0 +1,8 @@ +#!/bin/sh + +B_DIR="`pwd`/" +docker pull meabed/debian-jdk + +# +docker build -t "meabed/nutch:2.3" $B_DIR/nutch/ +docker build -t "meabed/cassandra" $B_DIR/cassandra/ Propchange: nutch/branches/2.x/docker/cassandra/bin/build.sh ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/bin/docker-data/cassandra/.gitkeep URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/bin/docker-data/cassandra/.gitkeep?rev=1680929&view=auto ============================================================================== (empty) Propchange: nutch/branches/2.x/docker/cassandra/bin/docker-data/cassandra/.gitkeep ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/bin/docker-data/nutch/.gitkeep URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/bin/docker-data/nutch/.gitkeep?rev=1680929&view=auto ============================================================================== (empty) Propchange: nutch/branches/2.x/docker/cassandra/bin/docker-data/nutch/.gitkeep ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/bin/ipof.sh URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/bin/ipof.sh?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/bin/ipof.sh (added) +++ nutch/branches/2.x/docker/cassandra/bin/ipof.sh Thu May 21 17:14:24 2015 @@ -0,0 +1,4 @@ +#!/bin/sh + +CONTAINER=$1 +docker inspect --format '{{ .NetworkSettings.IPAddress }}' $CONTAINER Propchange: nutch/branches/2.x/docker/cassandra/bin/ipof.sh ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/bin/nodes.sh URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/bin/nodes.sh?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/bin/nodes.sh (added) +++ nutch/branches/2.x/docker/cassandra/bin/nodes.sh Thu May 21 17:14:24 2015 @@ -0,0 +1,12 @@ +#!/bin/sh + +function isRunning { + id=$(docker ps -a | grep $1 | awk '{print $1}') + echo $id +} +# cassandra +cassandraNodeName=CASS01 +# elasticsearch +esNodeName=ES01 +# apache nutch +nutchNodeName=NUTCH01 \ No newline at end of file Propchange: nutch/branches/2.x/docker/cassandra/bin/nodes.sh ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/bin/restart.sh URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/bin/restart.sh?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/bin/restart.sh (added) +++ nutch/branches/2.x/docker/cassandra/bin/restart.sh Thu May 21 17:14:24 2015 @@ -0,0 +1,7 @@ +#!/bin/sh + +B_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + +source "$B_DIR/nodes.sh" +source "$B_DIR/stop.sh" +source "$B_DIR/start.sh" \ No newline at end of file Propchange: nutch/branches/2.x/docker/cassandra/bin/restart.sh ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/bin/start.sh URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/bin/start.sh?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/bin/start.sh (added) +++ nutch/branches/2.x/docker/cassandra/bin/start.sh Thu May 21 17:14:24 2015 @@ -0,0 +1,18 @@ +#!/bin/sh + +B_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +DOCKER_DATA_FOLDER=$B_DIR/docker-data + +chmod -R 777 $DOCKER_DATA_FOLDER + +source "$B_DIR/nodes.sh" +source "$B_DIR/stop.sh" + +cassandraId=$(docker run -d -P -v $DOCKER_DATA_FOLDER:/data:rw --name $cassandraNodeName meabed/cassandra) +cassandraIP=$("$B_DIR"/ipof.sh $cassandraId) + +# -p 9200:9200 +# http://dockerhost:9200/_plugin/kopf/ +# http://dockerhost:9200/_plugin/HQ/ + +docker run -d -p 8899:8899 -P -e CASSANDRA_NODE_NAME=$cassandraNodeName -it --link $cassandraNodeName:$cassandraNodeName -v $DOCKER_DATA_FOLDER:/data:rw --name $nutchNodeName meabed/nutch:2.3 Propchange: nutch/branches/2.x/docker/cassandra/bin/start.sh ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/bin/stop.sh URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/bin/stop.sh?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/bin/stop.sh (added) +++ nutch/branches/2.x/docker/cassandra/bin/stop.sh Thu May 21 17:14:24 2015 @@ -0,0 +1,20 @@ +#!/bin/sh + +B_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +source "$B_DIR/nodes.sh" + +cassandraRunning=$(isRunning $cassandraNodeName) +esRunning=$(isRunning $esNodeName) +nutchRunning=$(isRunning $nutchNodeName) + +if [ -n "${cassandraRunning}" ]; then + docker kill $cassandraNodeName && docker rm $cassandraNodeName +fi + +if [ -n "${esRunning}" ]; then + docker kill $esNodeName && docker rm $esNodeName +fi + +if [ -n "${nutchRunning}" ]; then + docker kill $nutchNodeName && docker rm $nutchNodeName +fi \ No newline at end of file Propchange: nutch/branches/2.x/docker/cassandra/bin/stop.sh ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/cassandra/Dockerfile URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/cassandra/Dockerfile?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/cassandra/Dockerfile (added) +++ nutch/branches/2.x/docker/cassandra/cassandra/Dockerfile Thu May 21 17:14:24 2015 @@ -0,0 +1,72 @@ +# +# Cassandra +# meabed/debian-jdk +# docker build -t meabed/cassandra:latest . +# +# sudo sysctl -w vm.max_map_count=2621444 +# sudo su +# echo "vm.max_map_count=262144" >> /etc/sysctl.conf + +# you might need to run this commands in DOCKER HOST +# ulimit -l unlimited +# ulimit -n 16240 +# ulimit -c unlimited + +FROM meabed/debian-jdk +MAINTAINER Mohamed Meabed "mo.mea...@gmail.com" + +USER root +ENV DEBIAN_FRONTEND noninteractive + + +# ADD DataStax sources +RUN echo "deb http://debian.datastax.com/community stable main" | tee -a /etc/apt/sources.list.d/cassandra.sources.list +RUN curl -L http://debian.datastax.com/debian/repo_key | apt-key add - + +RUN apt-get update + +# Install cassandra +RUN apt-get install -y dsc21 +RUN apt-get install -y opscenter + +# Comment the ulimit setters by cassandra deamon +RUN sed -i "/^[^#]*ulimit/ s/.*/#&/" /etc/init.d/cassandra + + +RUN service cassandra stop +RUN rm -rf /var/lib/cassandra/data/system/* + +RUN sed -ri 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config +RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +RUN sed -ri 's/#UsePAM no/UsePAM no/g' /etc/ssh/sshd_config + + +RUN sed -i "/^cluster_name:/ s|.*|cluster_name: 'Cassandra Cluster'\n|" /etc/cassandra/cassandra.yaml +RUN sed -i "/^rpc_address:/ s|.*|rpc_address: 0.0.0.0\n|" /etc/cassandra/cassandra.yaml + +RUN sed -ri 's/\/var\/lib\/cassandra\/data/\/var\/lib\/cassandra\/shared\/data\/cassandra/g' /etc/cassandra/cassandra.yaml + +# Run Cassandra as Root +RUN sed -ri 's/-c cassandra/-c root/g' /etc/init.d/cassandra + +#RUN echo "broadcast_rpc_address: localhost" >> /etc/cassandra/cassandra.yaml + +VOLUME ["/data"] + + +#RUN rm -rf /var/lib/cassandra +RUN mkdir -p /var/lib/cassandra/shared/data +RUN ln -svf /data/cassandra/ /var/lib/cassandra/shared/data + +RUN service ssh start && service opscenterd start && service cassandra start + +ADD bootstrap.sh /etc/bootstrap.sh +RUN chown root:root /etc/bootstrap.sh +RUN chmod 700 /etc/bootstrap.sh + +VOLUME ["/data"] + +CMD ["/etc/bootstrap.sh", "-d"] + +# http://www.datastax.com/documentation/cassandra/2.1/cassandra/security/secureFireWall_r.html +EXPOSE 22 8888 7000 7001 7199 9160 9042 61620 61621 Propchange: nutch/branches/2.x/docker/cassandra/cassandra/Dockerfile ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/cassandra/LICENSE URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/cassandra/LICENSE?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/cassandra/LICENSE (added) +++ nutch/branches/2.x/docker/cassandra/cassandra/LICENSE Thu May 21 17:14:24 2015 @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + Propchange: nutch/branches/2.x/docker/cassandra/cassandra/LICENSE ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/cassandra/README.md URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/cassandra/README.md?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/cassandra/README.md (added) +++ nutch/branches/2.x/docker/cassandra/cassandra/README.md Thu May 21 17:14:24 2015 @@ -0,0 +1,2 @@ +cassandra +========= Propchange: nutch/branches/2.x/docker/cassandra/cassandra/README.md ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/cassandra/bootstrap.sh URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/cassandra/bootstrap.sh?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/cassandra/bootstrap.sh (added) +++ nutch/branches/2.x/docker/cassandra/cassandra/bootstrap.sh Thu May 21 17:14:24 2015 @@ -0,0 +1,26 @@ +#!/bin/bash + +export PATH=$PATH:/usr/local/sbin/ +export PATH=$PATH:/usr/sbin/ +export PATH=$PATH:/sbin + +service ssh start && service opscenterd start && service cassandra start + +# Save Enviroemnt Variables incase ot attach to the container with new tty +env | awk '{split($0,a,"\n"); print "export " a[1]}' > /etc/env_profile +container_ip=$(hostname -I | cut -d' ' -f1) +#sed -i "/^rpc_address:/ s|.*|\n|" /etc/cassandra/cassandra.yaml +#echo "rpc_address: "$container_ip >> /etc/cassandra/cassandra.yaml +echo "broadcast_rpc_address: "$container_ip >> /etc/cassandra/cassandra.yaml + +echo "export PATH=$PATH" >> /etc/env_profile + +if [[ $1 == "-d" ]]; then + while true; do + sleep 1000; + done +fi + +if [[ $1 == "-bash" ]]; then + /bin/bash +fi Propchange: nutch/branches/2.x/docker/cassandra/cassandra/bootstrap.sh ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/nutch/Dockerfile URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/nutch/Dockerfile?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/nutch/Dockerfile (added) +++ nutch/branches/2.x/docker/cassandra/nutch/Dockerfile Thu May 21 17:14:24 2015 @@ -0,0 +1,81 @@ +# +# Nutch +# meabed/debian-jdk +# docker build -t meabed/nutch:latest . +# + +FROM meabed/debian-jdk +MAINTAINER Mohamed Meabed "mo.mea...@gmail.com" + +USER root +ENV DEBIAN_FRONTEND noninteractive + +ENV NUTCH_VERSION 2.3 + +#ant +RUN apt-get install -y ant + +#Download nutch + +RUN mkdir -p /opt/downloads && cd /opt/downloads && curl -SsfLO "http://archive.apache.org/dist/nutch/$NUTCH_VERSION/apache-nutch-$NUTCH_VERSION-src.tar.gz" +RUN cd /opt && tar xvfz /opt/downloads/apache-nutch-$NUTCH_VERSION-src.tar.gz +#WORKDIR /opt/apache-nutch-$NUTCH_VERSION +ENV NUTCH_ROOT /opt/apache-nutch-$NUTCH_VERSION +ENV HOME /root + +#Nutch-default +# RUN sed -i '/^ <name>http.agent.name<\/name>$/{$!{N;s/^ <name>http.agent.name<\/name>\n <value><\/value>$/ <name>http.agent.name<\/name>\n <value>iData Bot<\/value>/;ty;P;D;:y}}' $NUTCH_ROOT/conf/nutch-default.xml + +RUN vim -c 'g/name="gora-cassandra"/+1d' -c 'x' $NUTCH_ROOT/ivy/ivy.xml +RUN vim -c 'g/name="gora-cassandra"/-1d' -c 'x' $NUTCH_ROOT/ivy/ivy.xml + +RUN cd $NUTCH_ROOT && ant runtime + +#native libs +RUN rm $NUTCH_ROOT/lib/native/* +#RUN mkdir -p $NUTCH_ROOT/lib/native/Linux-amd64-64 +#RUN curl -Ls http://dl.bintray.com/meabed/hadoop-debian/hadoop-native-64-2.5.1.tar|tar -x -C $NUTCH_ROOT/lib/native/Linux-amd64-64/ + + +#Modification and compilation again + +ADD plugin/nutch2-index-html/src/plugin/ $NUTCH_ROOT/src/plugin/ +RUN sed -i '/dir="index-more" target="deploy".*/ s/.*/&\n <ant dir="index-html" target="deploy"\/>/' $NUTCH_ROOT/src/plugin/build.xml +RUN sed -i '/dir="index-more" target="clean".*/ s/.*/&\n <ant dir="index-html" target="clean"\/>/' $NUTCH_ROOT/src/plugin/build.xml + + +RUN cd $NUTCH_ROOT && ant runtime + +RUN ln -s /opt/apache-nutch-$NUTCH_VERSION/runtime/local /opt/nutch + +ENV NUTCH_HOME /opt/nutch + +# urls folder we will use in crawling $NUTCH_HOME/bin/crawl urls crawlId(test01) elasticsearch_node_name(iData) iteration(1) +RUN mkdir $NUTCH_HOME/urls +# Adding test urls to use in crawling +CMD mkdir -p $NUTCH_HOME/testUrls +ADD testUrls $NUTCH_HOME/testUrls + +# Adding rawcontent that hold html of the page field in index to elasticsearch +RUN sed -i '/field name="date" type.*/ s/.*/&\n\n <field name="rawcontent" type="text" sstored="true" indexed="true" multiValued="false"\/>\n/' $NUTCH_HOME/conf/schema.xml + +# remove nutche-site.xml default file to replace it by our configuration +RUN rm $NUTCH_HOME/conf/nutch-site.xml +ADD config/nutch-site.xml $NUTCH_HOME/conf/nutch-site.xml + +# Port that nutchserver will use +ENV NUTCHSERVER_PORT 8899 + +#RUN cd $NUTCH_HOME && ls -al + +#RUN mkdir -p /opt/nutch/urls && cd /opt/crawl + +ADD bootstrap.sh /etc/bootstrap.sh +RUN chown root:root /etc/bootstrap.sh +RUN chmod 700 /etc/bootstrap.sh + +VOLUME ["/data"] + +CMD ["/etc/bootstrap.sh", "-d"] + +EXPOSE 8899 Propchange: nutch/branches/2.x/docker/cassandra/nutch/Dockerfile ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/nutch/LICENSE URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/nutch/LICENSE?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/nutch/LICENSE (added) +++ nutch/branches/2.x/docker/cassandra/nutch/LICENSE Thu May 21 17:14:24 2015 @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + Propchange: nutch/branches/2.x/docker/cassandra/nutch/LICENSE ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/nutch/README.md URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/nutch/README.md?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/nutch/README.md (added) +++ nutch/branches/2.x/docker/cassandra/nutch/README.md Thu May 21 17:14:24 2015 @@ -0,0 +1,2 @@ +nutch +===== Propchange: nutch/branches/2.x/docker/cassandra/nutch/README.md ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/nutch/bootstrap.sh URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/nutch/bootstrap.sh?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/nutch/bootstrap.sh (added) +++ nutch/branches/2.x/docker/cassandra/nutch/bootstrap.sh Thu May 21 17:14:24 2015 @@ -0,0 +1,31 @@ +#!/bin/bash + +export PATH=$PATH:/usr/local/sbin/ +export PATH=$PATH:/usr/sbin/ +export PATH=$PATH:/sbin + +# Save Enviroemnt Variables incase ot attach to the container with new tty +env | awk '{split($0,a,"\n"); print "export " a[1]}' > /etc/env_profile + +cassandra_env="$CASSANDRA_NODE_NAME"_PORT_9160_TCP_ADDR +cassandra_ip=$(printenv $cassandra_env) + +sed -i "/^gora\.cassandrastore\.servers.*/ s/.*//" $NUTCH_HOME/conf/gora.properties +echo gora.cassandrastore.servers=$cassandra_ip:9160 >> $NUTCH_HOME/conf/gora.properties +vim -c '%s/localhost/'$cassandra_ip'/' -c 'x' $NUTCH_HOME/conf/gora-cassandra-mapping.xml + +nutchserver_port=$(printenv NUTCHSERVER_PORT) + +$NUTCH_HOME/bin/nutch nutchserver $nutchserver_port + +echo "export PATH=$PATH" >> /etc/env_profile + +if [[ $1 == "-d" ]]; then + while true; do + sleep 1000; + done +fi + +if [[ $1 == "-bash" ]]; then + /bin/bash +fi Propchange: nutch/branches/2.x/docker/cassandra/nutch/bootstrap.sh ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/nutch/config/nutch-site.xml URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/nutch/config/nutch-site.xml?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/nutch/config/nutch-site.xml (added) +++ nutch/branches/2.x/docker/cassandra/nutch/config/nutch-site.xml Thu May 21 17:14:24 2015 @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + <property> + <name>http.agent.name</name> + <value>Mozilla/5.0</value> + </property> + <property> + <name>http.robots.agents</name> + <value>Mozilla/5.0,*</value> + </property> + <property> + <name>http.agent.description</name> + <value>Mozilla/5</value> + </property> + <property> + <name>http.agent.version</name> + <value>0.0.1</value> + </property> + <property> + <name>http.agent.url</name> + <value>http://www.google.com</value> + </property> + <property> + <name>http.agent.email</name> + <value>mo.mea...@gmail.com</value> + </property> + <property> + <name>http.content.limit</name> + <value>1000000</value> + </property> + <property> + <name>storage.data.store.class</name> + <value>org.apache.gora.cassandra.store.CassandraStore</value> + <description>Default class for storing data</description> + </property> + <property> + <name>fetcher.server.delay</name> + <value>2.0</value> + <description>The number of seconds the fetcher will delay between + successive requests to the same server. + </description> + </property> + <property> + <name>indexer.max.title.length</name> + <value>300</value> + <description>The maximum number of characters of a title that are indexed. A value of -1 disables this check. + Used by index-basic. + </description> + </property> + <property> + <name>db.ignore.external.links</name> + <value>true</value> + <description>If true, outlinks leading from a page to external hosts + will be ignored. This is an effective way to limit the crawl to include + only initially injected hosts, without creating complex URLFilters. + </description> + </property> + <property> + <name>fetcher.parse</name> + <value>true</value> + <description>If true, fetcher will parse content. NOTE: previous releases would + default to true. Since 2.0 this is set to false as a safer default. + </description> + </property> + <property> + <name>plugin.includes</name> + <value>protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor|more|html)|urlnormalizer-(pass|regex|basic)|scoring-opic|protocol-httpclient|language-identifier|indexer-solr</value> + <description>Regular expression naming plugin directory names to + include. Any plugin not matching this expression is excluded. + In any case you need at least include the nutch-extensionpoints plugin. By + default Nutch includes crawling just HTML and plain text via HTTP, + and basic indexing and search plugins. In order to use HTTPS please enable + protocol-httpclient, but be aware of possible intermittent problems with the + underlying commons-httpclient library. + </description> + </property> +</configuration> Propchange: nutch/branches/2.x/docker/cassandra/nutch/config/nutch-site.xml ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/.gitignore URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/.gitignore?rev=1680929&view=auto ============================================================================== (empty) Propchange: nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/.gitignore ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/LICENSE URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/LICENSE?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/LICENSE (added) +++ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/LICENSE Thu May 21 17:14:24 2015 @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + Propchange: nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/LICENSE ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/README.md URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/README.md?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/README.md (added) +++ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/README.md Thu May 21 17:14:24 2015 @@ -0,0 +1,69 @@ +Index-html Plugin for apache nutch 2.x +================= + +Index HTML content of the pages in Apache Nutch 2.x ( 2.x ) + +##Instruction: + +- ###Compile from Source + + - Download the plugin folder "index-html" and copy it to you Apache nutch 2 plugin directory ( ex: apache-nutch-2.x/src/plugin ) + - Add the ( index-html ) plugin to The plugin folder build.xml ( apache-nutch-2.x/src/plugin/build.xml ) in target ( deploy and clean ) so the file will look like + ```xml + <target name="deploy"> + ....... + <ant dir="index-basic" target="deploy"/> + <ant dir="index-more" target="deploy"/> + <ant dir="index-html" target="deploy"/> + <ant dir="language-identifier" target="deploy"/> + ......... + </target> + + <target name="clean"> + ....... + <ant dir="index-basic" target="deploy"/> + <ant dir="index-more" target="deploy"/> + <ant dir="index-html" target="deploy"/> + <ant dir="language-identifier" target="deploy"/> + ......... + </target> + + ``` + + - Run ( ant runtime ) in apache nutch 2 root folder to start the build + - You should have index-html.jar in build folder + - Enable the plugin by adding it to nutch-sites.xml ( or nutch-default.xml ) like beloe : + ```xml + <configuration> + .......... + <property> + <name>plugin.includes</name> + <value>...........someplugins....|index-html</value> + </property> + .......... + </configuration> + ``` + - The plugin will add new Field "rawcontent" to the Nutch Doc, To index this field you need to add it to ( scheme.xml or schema-solr4.xml ) like + ```xml + <field name="rawcontent" type="text" sstored="true" indexed="true" multiValued="false"/> + ``` + - Run the crawler and you should see the new field rawcontent in index! + +- ###Use Pre-Compiled Library + - In The repo there is Build folder contain compiled .jar library ready for use. + - Copy the library to your runtime path local if you are running the plugin locally ( apache-nutch-2.x/runtime/local/plugins ) Then follow the above steps to configure nutch-sites.xml + + +###Screen Shot +![Screen Shot](http://i.imgur.com/rPWAcoQ.png) + +## Need Help ? + +I'm Always glad ot help and assist. so if you have an idea that could make this project better + +Submit git issue or contact me [www.meabed.net](http://meabed.net) + + +### How to contribute + +Make a fork, commit to develop branch and make a pull request Propchange: nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/README.md ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/build.xml URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/build.xml?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/build.xml (added) +++ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/build.xml Thu May 21 17:14:24 2015 @@ -0,0 +1,22 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project name="index-html" default="jar-core"> + + <import file="../build-plugin.xml"/> + +</project> Propchange: nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/build.xml ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/ivy.xml URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/ivy.xml?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/ivy.xml (added) +++ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/ivy.xml Thu May 21 17:14:24 2015 @@ -0,0 +1,41 @@ +<?xml version="1.0" ?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<ivy-module version="1.0"> + <info organisation="org.apache.nutch" module="${ant.project.name}"> + <license name="Apache 2.0"/> + <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/> + <description> + Apache Nutch + </description> + </info> + + <configurations> + <include file="../../..//ivy/ivy-configurations.xml"/> + </configurations> + + <publications> + <!--get the artifact from our module name--> + <artifact conf="master"/> + </publications> + + <dependencies> + </dependencies> + +</ivy-module> Propchange: nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/ivy.xml ------------------------------------------------------------------------------ svn:executable = * Added: nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/plugin.xml URL: http://svn.apache.org/viewvc/nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/plugin.xml?rev=1680929&view=auto ============================================================================== --- nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/plugin.xml (added) +++ nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/plugin.xml Thu May 21 17:14:24 2015 @@ -0,0 +1,42 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<plugin + id="index-html" + name="Html Indexing Filter" + version="1.0.0" + provider-name="nutch.org"> + + + <runtime> + <library name="index-html.jar"> + <export name="*"/> + </library> + </runtime> + + <requires> + <import plugin="nutch-extensionpoints"/> + </requires> + + <extension id="org.apache.nutch.indexer.html" + name="Nutch Html Indexing Filter" + point="org.apache.nutch.indexer.IndexingFilter"> + <implementation id="HtmlIndexingFilter" + class="org.apache.nutch.indexer.html.HtmlIndexingFilter"/> + </extension> + +</plugin> Propchange: nutch/branches/2.x/docker/cassandra/nutch/plugin/nutch2-index-html/src/plugin/index-html/plugin.xml ------------------------------------------------------------------------------ svn:executable = *