METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/5f7454e4 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/5f7454e4 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/5f7454e4 Branch: refs/heads/feature/METRON-1136-extensions-parsers Commit: 5f7454e4ad4574574a6bcf3cefb9a4fd95ef4c8d Parents: 8560d2d Author: ottobackwards <ottobackwa...@gmail.com> Authored: Wed Aug 30 11:03:32 2017 -0400 Committer: otto <o...@apache.org> Committed: Wed Aug 30 11:37:00 2017 -0400 ---------------------------------------------------------------------- .gitignore | 1 + .travis.yml | 1 + README.md | 11 +- bundles-lib/README.md | 171 +++ bundles-lib/pom.xml | 185 +++ .../metron/bundles/BundleClassLoaders.java | 353 ++++++ .../apache/metron/bundles/BundleCloseable.java | 86 ++ .../metron/bundles/BundleManifestEntry.java | 48 + .../org/apache/metron/bundles/BundleMapper.java | 191 +++ .../org/apache/metron/bundles/BundleSystem.java | 202 +++ .../bundles/BundleThreadContextClassLoader.java | 211 ++++ .../apache/metron/bundles/ExtensionManager.java | 534 ++++++++ .../apache/metron/bundles/ExtensionMapping.java | 156 +++ .../metron/bundles/InstanceClassLoader.java | 161 +++ .../metron/bundles/NotInitializedException.java | 38 + .../metron/bundles/VFSBundleClassLoader.java | 520 ++++++++ .../bundles/VFSBundleClassLoaderResource.java | 110 ++ .../behavior/RequiresInstanceClassLoading.java | 37 + .../apache/metron/bundles/bundle/Bundle.java | 48 + .../bundles/bundle/BundleCoordinates.java | 96 ++ .../metron/bundles/bundle/BundleDetails.java | 205 +++ .../metron/bundles/util/BundleProperties.java | 247 ++++ .../metron/bundles/util/BundleSelector.java | 48 + .../apache/metron/bundles/util/BundleUtil.java | 145 +++ .../metron/bundles/util/DummyFileObject.java | 230 ++++ .../bundles/util/FileSystemManagerFactory.java | 87 ++ .../apache/metron/bundles/util/FileUtils.java | 35 + .../bundles/util/ImmutableCollectionUtils.java | 65 + .../apache/metron/bundles/util/StringUtils.java | 118 ++ .../org/apache/metron/bundles/AbstractFoo.java | 24 + .../metron/bundles/BundleClassLoadersTest.java | 110 ++ .../apache/metron/bundles/BundleMapperTest.java | 180 +++ .../apache/metron/bundles/BundleSystemTest.java | 56 + .../BundleThreadContextClassLoaderTest.java | 142 +++ .../apache/metron/bundles/BundleUtilTest.java | 125 ++ .../bundles/ExtensionClassInitializerTest.java | 33 + .../metron/bundles/ExtensionManagerTest.java | 82 ++ .../BundleMapperIntegrationTest.java | 146 +++ .../util/ImmutableCollectionUtilsTest.java | 85 ++ .../metron/bundles/util/ResourceCopier.java | 60 + .../apache/metron/bundles/util/TestUtil.java | 55 + .../parsers/interfaces/MessageParser.java | 22 + .../BundleMapper/conf/bundle.properties | 21 + .../lib/metron-parser-bar-bundle-0.4.1.bundle | Bin 0 -> 21972 bytes .../lib2/metron-parser-foo-bundle-0.4.1.bundle | Bin 0 -> 21983 bytes .../org.apache.metron.bundles.AbstractFoo | 16 + .../src/test/resources/bundle.properties | 22 + .../bundle-with-versioning/META-INF/MANIFEST.MF | 15 + .../META-INF/MANIFEST.MF | 12 + .../META-INF/MANIFEST.MF | 7 + bundles-maven-plugin/LICENSE | 202 +++ bundles-maven-plugin/NOTICE | 5 + bundles-maven-plugin/README.md | 89 ++ bundles-maven-plugin/pom.xml | 328 +++++ .../maven/plugins/bundles/BundleMojo.java | 743 +++++++++++ .../bundles/BundleProvidedDependenciesMojo.java | 328 +++++ .../resources/META-INF/plexus/components.xml | 52 + metron-analytics/metron-maas-service/README.md | 2 +- .../METRON/CURRENT/configuration/metron-env.xml | 42 + .../common-services/METRON/CURRENT/metainfo.xml | 39 + .../CURRENT/package/scripts/metron_service.py | 43 +- .../package/scripts/params/params_linux.py | 8 + .../package/scripts/params/status_params.py | 9 + .../CURRENT/package/scripts/parser_commands.py | 62 +- .../package/templates/bundle.properties.j2 | 21 + .../CURRENT/package/templates/global.json.j2 | 3 +- .../METRON/CURRENT/package/templates/metron.j2 | 1 + .../docker/rpm-docker/SPECS/metron.spec | 498 +++++++- .../packaging/docker/rpm-docker/pom.xml | 79 ++ .../sensor-config-single-parser.e2e-spec.ts | 2 +- .../sensor-field-schema.component.spec.ts | 4 +- ...sensor-parser-config-readonly.component.html | 2 +- ...sor-parser-config-readonly.component.spec.ts | 4 +- .../sensor-parser-config.component.spec.ts | 51 +- .../sensor-parser-config.component.ts | 12 +- .../sensor-parser-list.component.spec.ts | 16 +- .../src/app/service/grok-validation.service.ts | 9 +- .../sensor-parser-config.service.spec.ts | 2 +- metron-interface/metron-rest/README.md | 33 + metron-interface/metron-rest/pom.xml | 34 + .../src/main/config/rest_application.yml | 4 + .../apache/metron/rest/MetronRestConstants.java | 4 +- .../metron/rest/config/BundleSystemConfig.java | 64 + .../apache/metron/rest/config/GrokConfig.java | 8 +- .../metron/rest/controller/GrokController.java | 13 +- .../controller/ParserExtensionController.java | 114 ++ .../metron/rest/service/ExtensionService.java | 58 + .../apache/metron/rest/service/GrokService.java | 4 +- .../apache/metron/rest/service/HdfsService.java | 4 + .../rest/service/SensorParserConfigService.java | 4 +- .../rest/service/impl/ExtensionServiceImpl.java | 562 +++++++++ .../rest/service/impl/GrokServiceImpl.java | 72 +- .../rest/service/impl/HdfsServiceImpl.java | 15 + .../impl/SensorParserConfigServiceImpl.java | 110 +- .../src/main/resources/application-test.yml | 4 + .../src/main/resources/application-vagrant.yml | 3 +- .../apache/metron/rest/config/TestConfig.java | 27 +- .../GrokControllerIntegrationTest.java | 4 +- .../KafkaControllerIntegrationTest.java | 4 +- ...arserExtensionControllerIntegrationTest.java | 165 +++ ...richmentConfigControllerIntegrationTest.java | 3 +- ...IndexingConfigControllerIntegrationTest.java | 3 +- ...orParserConfigControllerIntegrationTest.java | 74 +- .../service/impl/GrokServiceImplNoMockTest.java | 122 ++ .../rest/service/impl/GrokServiceImplTest.java | 31 +- .../impl/SensorParserConfigServiceImplTest.java | 95 +- ...on-parser-test-assembly-0.4.0-archive.tar.gz | Bin 0 -> 24772 bytes .../test/resources/zookeeper/bundle.properties | 21 + .../src/test/resources/zookeeper/global.json | 25 + metron-maven-archetypes/README.md | 57 + .../metron-parser-extension-archetype/README.md | 83 ++ .../metron-parser-extension-archetype/pom.xml | 46 + .../META-INF/maven/archetype-metadata.xml | 81 ++ .../pom.xml | 49 + .../src/main/assembly/assembly.xml | 56 + .../metron-parser-__parserName__-bundle/pom.xml | 41 + .../metron-parser-__parserName__/README.md | 5 + .../metron-parser-__parserName__/pom.xml | 83 ++ .../elasticsearch/__parserName___index.template | 124 ++ .../zookeeper/enrichments/__parserName__.json | 20 + .../zookeeper/indexing/__parserName__.json | 18 + .../zookeeper/parsers/__parserName__.json | 7 + .../__parserClassName__Parser.java | 79 ++ .../src/main/resources/META-INF/LICENSE | 602 +++++++++ .../src/main/resources/META-INF/NOTICE | 40 + .../src/main/resources/patterns/common | 96 ++ .../__parserClassName__ParserTest.java | 51 + .../__parserClassName__IntegrationTest.java | 52 + .../config/zookeeper/bundle.properties | 23 + .../test/resources/config/zookeeper/global.json | 27 + .../src/test/resources/data/parsed/test.parsed | 2 + .../src/test/resources/data/raw/test.raw | 2 + .../src/test/resources/log4j.properties | 37 + .../main/resources/archetype-resources/pom.xml | 36 + .../projects/basic/archetype.properties | 8 + .../src/test/resources/projects/basic/goal.txt | 0 metron-maven-archetypes/pom.xml | 34 + metron-maven-archetypes/project.png | Bin 0 -> 127469 bytes .../org/apache/metron/common/Constants.java | 1 + .../metron/common/bolt/ConfiguredBolt.java | 29 +- .../common/configuration/ConfigurationType.java | 15 +- .../configuration/ConfigurationsUtils.java | 89 +- .../extensions/ParserExtensionConfig.java | 192 +++ .../metron/common/utils/ResourceLoader.java | 174 +++ .../common/cli/ConfigurationsUtilsTest.java | 18 +- .../SensorEnrichmentConfigTest.java | 7 +- .../extensions/ParserExtensionConfigTest.java | 39 + .../metron/common/utils/ResourceLoaderTest.java | 90 ++ .../main/config/zookeeper/enrichments/asa.json | 8 - .../main/config/zookeeper/enrichments/bro.json | 18 - .../config/zookeeper/enrichments/snort.json | 29 - .../config/zookeeper/enrichments/websphere.json | 18 - .../main/config/zookeeper/enrichments/yaf.json | 20 - .../components/ConfigUploadComponent.java | 11 + metron-platform/metron-extensions/README.md | 46 + .../metron-extensions/extension_deployment.md | 20 + .../metron-extensions/extension_packaging.md | 16 + .../metron-extensions/extension_terms.md | 25 + .../metron-parser-extensions/README.md | 9 + .../adding_system_parsers.md | 345 +++++ .../metron-parser-asa-assembly/pom.xml | 50 + .../src/main/assembly/assembly.xml | 65 + .../metron-parser-asa-bundle/pom.xml | 42 + .../metron-parser-asa/README.md | 3 + .../metron-parser-asa/pom.xml | 86 ++ .../main/config/zookeeper/enrichments/asa.json | 8 + .../src/main/config/zookeeper/indexing/asa.json | 18 + .../src/main/config/zookeeper/parsers/asa.json | 7 + .../metron/parsers/asa/BasicAsaParser.java | 240 ++++ .../src/main/resources/META-INF/LICENSE | 602 +++++++++ .../src/main/resources/META-INF/NOTICE | 40 + .../src/main/resources/patterns/asa | 179 +++ .../src/main/resources/patterns/common | 96 ++ .../metron/parsers/asa/BasicAsaParserTest.java | 187 +++ .../asa/integration/AsaIntegrationTest.java | 49 + .../src/test/resources/data/parsed/test.parsed | 128 ++ .../src/test/resources/data/raw/test.raw | 128 ++ .../src/test/resources/log4j.properties | 34 + .../metron-parser-asa-extension/pom.xml | 36 + .../metron-parser-bro-assembly/pom.xml | 49 + .../src/main/assembly/assembly.xml | 42 + .../metron-parser-bro-bundle/pom.xml | 41 + .../metron-parser-bro/README.md | 3 + .../metron-parser-bro/pom.xml | 83 ++ .../config/elasticsearch/bro_index.template | 972 ++++++++++++++ .../main/config/zookeeper/enrichments/bro.json | 17 + .../src/main/config/zookeeper/indexing/bro.json | 18 + .../src/main/config/zookeeper/parsers/bro.json | 6 + .../metron/parsers/bro/BasicBroParser.java | 180 +++ .../apache/metron/parsers/bro/JSONCleaner.java | 91 ++ .../src/main/resources/META-INF/LICENSE | 602 +++++++++ .../src/main/resources/META-INF/NOTICE | 40 + .../metron/parsers/bro/BasicBroParserTest.java | 1189 ++++++++++++++++++ .../parsers/integration/BroIntegrationTest.java | 48 + .../test/resources/TestSchemas/BroSchema.json | 28 + .../test/resources/config/BroParserTest.config | 20 + .../src/test/resources/data/parsed/test.parsed | 27 + .../src/test/resources/data/raw/test.raw | 27 + .../src/test/resources/log4j.properties | 34 + .../metron-parser-bro-extension/pom.xml | 36 + .../metron-parser-bundle-tests/pom.xml | 51 + .../parsers/ASABundleHDFSIntegrationTest.java | 265 ++++ ...etron-parser-test-bundle-1.0-SNAPSHOT.bundle | Bin 0 -> 22129 bytes .../metron-parser-asa-bundle-0.4.0.bundle | Bin 0 -> 45283 bytes .../test/resources/zookeeper/bundle.properties | 23 + .../resources/zookeeper/enrichments/test.json | 75 ++ .../src/test/resources/zookeeper/global.json | 27 + .../test/resources/zookeeper/indexing/test.json | 18 + .../metron-parser-cef-assembly/pom.xml | 49 + .../src/main/assembly/assembly.xml | 42 + .../metron-parser-cef-bundle/pom.xml | 41 + .../metron-parser-cef/README.md | 3 + .../metron-parser-cef/pom.xml | 89 ++ .../main/config/zookeeper/enrichments/cef.json | 19 + .../src/main/config/zookeeper/indexing/cef.json | 18 + .../src/main/config/zookeeper/parsers/cef.json | 7 + .../apache/metron/parsers/cef/CEFParser.java | 274 ++++ .../src/main/resources/META-INF/LICENSE | 602 +++++++++ .../src/main/resources/META-INF/NOTICE | 40 + .../metron/parsers/cef/CEFParserTest.java | 291 +++++ .../src/test/resources/log4j.properties | 34 + .../org/apache/metron/parsers/cef/adallom.cef | 1 + .../apache/metron/parsers/cef/adallom.schema | 37 + .../org/apache/metron/parsers/cef/cyberark.cef | 1 + .../org/apache/metron/parsers/cef/cyberark.json | 21 + .../apache/metron/parsers/cef/cyberark.schema | 38 + .../org/apache/metron/parsers/cef/palo.cef | 1 + .../org/apache/metron/parsers/cef/palo.schema | 38 + .../org/apache/metron/parsers/cef/waf.cef | 1 + .../org/apache/metron/parsers/cef/waf.schema | 67 + .../metron-parser-cef-extension/pom.xml | 36 + .../metron-parser-extensions-testing/pom.xml | 88 ++ .../metron-parser-fireeye-assembly/pom.xml | 49 + .../src/main/assembly/assembly.xml | 53 + .../metron-parser-fireeye-bundle/pom.xml | 41 + .../metron-parser-fireeye/README.md | 3 + .../metron-parser-fireeye/pom.xml | 89 ++ .../config/zookeeper/enrichments/fireeye.json | 20 + .../main/config/zookeeper/indexing/fireeye.json | 18 + .../main/config/zookeeper/parsers/fireeye.json | 7 + .../parsers/fireeye/BasicFireEyeParser.java | 222 ++++ .../src/main/resources/META-INF/LICENSE | 602 +++++++++ .../src/main/resources/META-INF/NOTICE | 40 + .../src/main/resources/patterns/common | 96 ++ .../src/main/resources/patterns/fireeye | 9 + .../parsers/fireeye/BasicFireEyeParserTest.java | 76 ++ .../config/BasicFireEyeParserTest.config | 20 + .../src/test/resources/data/parsed/test.parsed | 2 + .../src/test/resources/data/raw/test.raw | 2 + .../src/test/resources/log4j.properties | 34 + .../resources/logData/FireEyeParserTest.txt | 8 + .../metron-parser-fireeye-extension/pom.xml | 36 + .../metron-parser-ise-assembly/pom.xml | 49 + .../src/main/assembly/assembly.xml | 42 + .../metron-parser-ise-bundle/pom.xml | 41 + .../metron-parser-ise/README.md | 3 + .../metron-parser-ise/pom.xml | 89 ++ .../main/config/zookeeper/enrichments/ise.json | 20 + .../src/main/config/zookeeper/indexing/ise.json | 18 + .../src/main/config/zookeeper/parsers/ise.json | 7 + .../metron/parsers/ise/BasicIseParser.java | 101 ++ .../apache/metron/parsers/ise/ISEParser.java | 665 ++++++++++ .../org/apache/metron/parsers/ise/ISEParser.jj | 12 + .../metron/parsers/ise/ISEParserConstants.java | 69 + .../parsers/ise/ISEParserTokenManager.java | 676 ++++++++++ .../metron/parsers/ise/JavaCharStream.java | 633 ++++++++++ .../metron/parsers/ise/ParseException.java | 204 +++ .../org/apache/metron/parsers/ise/Token.java | 148 +++ .../metron/parsers/ise/TokenMgrError.java | 164 +++ .../src/main/resources/META-INF/LICENSE | 602 +++++++++ .../src/main/resources/META-INF/NOTICE | 40 + .../metron/parsers/ise/BasicIseParserTest.java | 56 + .../test/resources/TestSchemas/IseSchema.json | 21 + .../resources/config/BasicIseParserTest.config | 20 + .../src/test/resources/log4j.properties | 34 + .../test/resources/logData/IseParserTest.txt | 308 +++++ .../metron-parser-ise-extension/pom.xml | 36 + .../metron-parser-lancope-assembly/pom.xml | 49 + .../src/main/assembly/assembly.xml | 42 + .../metron-parser-lancope-bundle/pom.xml | 41 + .../metron-parser-lancope/README.md | 3 + .../metron-parser-lancope/pom.xml | 89 ++ .../config/zookeeper/enrichments/lancope.json | 20 + .../main/config/zookeeper/indexing/lancope.json | 18 + .../main/config/zookeeper/parsers/lancope.json | 7 + .../parsers/lancope/BasicLancopeParser.java | 95 ++ .../src/main/resources/META-INF/LICENSE | 602 +++++++++ .../src/main/resources/META-INF/NOTICE | 40 + .../parsers/lancope/BasicLancopeParserTest.java | 57 + .../resources/TestSchemas/LancopeSchema.json | 28 + .../config/BasicLancopeParserTest.config | 20 + .../src/test/resources/log4j.properties | 34 + .../resources/logData/LancopeParserTest.txt | 1 + .../metron-parser-lancope-extension/pom.xml | 36 + .../metron-parser-logstash-assembly/pom.xml | 49 + .../src/main/assembly/assembly.xml | 42 + .../metron-parser-logstash-bundle/pom.xml | 41 + .../metron-parser-logstash/README.md | 3 + .../metron-parser-logstash/pom.xml | 83 ++ .../config/zookeeper/enrichments/logstash.json | 20 + .../config/zookeeper/indexing/logstash.json | 18 + .../main/config/zookeeper/parsers/logstash.json | 7 + .../parsers/logstash/BasicLogstashParser.java | 95 ++ .../src/main/resources/META-INF/LICENSE | 602 +++++++++ .../src/main/resources/META-INF/NOTICE | 40 + .../logstash/BasicLogstashParserTest.java | 46 + .../src/test/resources/log4j.properties | 34 + .../metron-parser-logstash-extension/pom.xml | 36 + .../metron-parser-paloalto-assembly/pom.xml | 49 + .../src/main/assembly/assembly.xml | 42 + .../metron-parser-paloalto-bundle/pom.xml | 41 + .../metron-parser-paloalto/README.md | 3 + .../metron-parser-paloalto/pom.xml | 89 ++ .../config/zookeeper/enrichments/paloalto.json | 20 + .../config/zookeeper/indexing/paloalto.json | 18 + .../main/config/zookeeper/parsers/paloalto.json | 7 + .../paloalto/BasicPaloAltoFirewallParser.java | 215 ++++ .../src/main/resources/META-INF/LICENSE | 602 +++++++++ .../src/main/resources/META-INF/NOTICE | 40 + .../BasicPaloAltoFirewallParserTest.java | 57 + .../BasicPaloAltoFirewallParserTest.config | 20 + .../src/test/resources/log4j.properties | 34 + .../logData/PaloAltoFirewallParserTest.txt | 2 + .../metron-parser-paloalto-extension/pom.xml | 36 + .../metron-parser-snort-assembly/pom.xml | 49 + .../src/main/assembly/assembly.xml | 53 + .../metron-parser-snort-bundle/pom.xml | 41 + .../metron-parser-snort/README.md | 3 + .../metron-parser-snort/pom.xml | 83 ++ .../config/elasticsearch/snort_index.template | 183 +++ .../config/zookeeper/enrichments/snort.json | 29 + .../main/config/zookeeper/indexing/snort.json | 18 + .../main/config/zookeeper/parsers/snort.json | 6 + .../metron/parsers/snort/BasicSnortParser.java | 214 ++++ .../src/main/resources/META-INF/LICENSE | 602 +++++++++ .../src/main/resources/META-INF/NOTICE | 40 + .../integration/SnortIntegrationTest.java | 49 + .../metron/parsers/snort/SnortParserTest.java | 149 +++ .../src/test/resources/data/parsed/test.parsed | 3 + .../src/test/resources/data/raw/test.raw | 3 + .../src/test/resources/log4j.properties | 34 + .../metron-parser-snort-extension/pom.xml | 36 + .../metron-parser-sourcefire-assembly/pom.xml | 49 + .../src/main/assembly/assembly.xml | 53 + .../metron-parser-sourcefire-bundle/pom.xml | 41 + .../metron-parser-sourcefire/README.md | 3 + .../metron-parser-sourcefire/pom.xml | 89 ++ .../zookeeper/enrichments/sourcefire.json | 20 + .../config/zookeeper/indexing/sourcefire.json | 18 + .../config/zookeeper/parsers/sourcefire.json | 7 + .../sourcefire/BasicSourcefireParser.java | 129 ++ .../src/main/resources/META-INF/LICENSE | 602 +++++++++ .../src/main/resources/META-INF/NOTICE | 40 + .../src/main/resources/patterns/common | 96 ++ .../src/main/resources/patterns/sourcefire | 30 + .../sourcefire/BasicSourcefireParserTest.java | 58 + .../resources/TestSchemas/SourcefireSchema.json | 34 + .../config/BasicSourcefireParserTest.config | 20 + .../src/test/resources/global.json | 27 + .../src/test/resources/log4j.properties | 34 + .../resources/logData/SourcefireParserTest.txt | 3 + .../metron-parser-sourcefire-extension/pom.xml | 36 + .../metron-parser-squid-assembly/pom.xml | 49 + .../src/main/assembly/assembly.xml | 55 + .../metron-parser-squid-bundle/pom.xml | 41 + .../metron-parser-squid/README.md | 3 + .../metron-parser-squid/pom.xml | 83 ++ .../config/zookeeper/enrichments/squid.json | 20 + .../main/config/zookeeper/indexing/squid.json | 18 + .../main/config/zookeeper/parsers/squid.json | 19 + .../src/main/resources/META-INF/LICENSE | 602 +++++++++ .../src/main/resources/META-INF/NOTICE | 40 + .../src/main/resources/patterns/common | 96 ++ .../src/main/resources/patterns/squid | 2 + .../integration/SquidIntegrationTest.java | 48 + .../metron/parsers/squid/SquidParserTest.java | 102 ++ .../src/test/resources/data/parsed/test.parsed | 2 + .../src/test/resources/data/raw/test.raw | 2 + .../src/test/resources/log4j.properties | 34 + .../metron-parser-squid-extension/pom.xml | 38 + .../metron-parser-websphere-assembly/pom.xml | 49 + .../src/main/assembly/assembly.xml | 53 + .../metron-parser-websphere-bundle/pom.xml | 41 + .../metron-parser-websphere/README.md | 3 + .../metron-parser-websphere/pom.xml | 83 ++ .../config/zookeeper/enrichments/websphere.json | 18 + .../config/zookeeper/indexing/websphere.json | 18 + .../config/zookeeper/parsers/websphere.json | 11 + .../parsers/websphere/GrokWebSphereParser.java | 143 +++ .../src/main/resources/META-INF/LICENSE | 602 +++++++++ .../src/main/resources/META-INF/NOTICE | 40 + .../src/main/resources/patterns/common | 96 ++ .../src/main/resources/patterns/websphere | 37 + .../integration/WebSphereIntegrationTest.java | 52 + .../websphere/GrokWebSphereParserTest.java | 269 ++++ .../src/test/resources/data/parsed/test.parsed | 5 + .../src/test/resources/data/raw/test.raw | 5 + .../src/test/resources/global.json | 27 + .../src/test/resources/log4j.properties | 34 + .../metron-parser-websphere-extension/pom.xml | 36 + .../metron-parser-yaf-assembly/pom.xml | 49 + .../src/main/assembly/assembly.xml | 55 + .../metron-parser-yaf-bundle/pom.xml | 41 + .../metron-parser-yaf/README.md | 3 + .../metron-parser-yaf/pom.xml | 83 ++ .../config/elasticsearch/yaf_index.template | 205 +++ .../main/config/zookeeper/enrichments/yaf.json | 20 + .../src/main/config/zookeeper/indexing/yaf.json | 18 + .../src/main/config/zookeeper/parsers/yaf.json | 18 + .../src/main/resources/META-INF/LICENSE | 602 +++++++++ .../src/main/resources/META-INF/NOTICE | 40 + .../src/main/resources/patterns/common | 96 ++ .../src/main/resources/patterns/yaf | 2 + .../parsers/integration/YafIntegrationTest.java | 48 + .../metron/parsers/yaf/YafParserTest.java | 101 ++ .../test/resources/data/indexed/test.indexed | 10 + .../src/test/resources/data/parsed/test.parsed | 10 + .../src/test/resources/data/raw/test.raw | 10 + .../src/test/resources/log4j.properties | 34 + .../metron-parser-yaf-extension/pom.xml | 38 + .../parser_extension_deployment.md | 240 ++++ .../parser_extension_packaging.md | 84 ++ .../metron-parser-extensions/pom.xml | 97 ++ metron-platform/metron-extensions/pom.xml | 60 + .../src/main/config/zookeeper/indexing/asa.json | 18 - .../src/main/config/zookeeper/indexing/bro.json | 17 - .../main/config/zookeeper/indexing/snort.json | 17 - .../config/zookeeper/indexing/websphere.json | 18 - .../src/main/config/zookeeper/indexing/yaf.json | 17 - .../src/main/config/zookeeper/bundle.properties | 20 + .../extensions/parsers/metron-test-parsers.json | 7 + .../src/main/config/zookeeper/global.json | 3 +- .../src/main/sample/data/asa/parsed/asa_parsed | 128 -- .../src/main/sample/data/asa/raw/asa_raw | 128 -- .../sample/data/bro/parsed/BroExampleParsed | 27 - .../main/sample/data/bro/raw/BroExampleOutput | 27 - .../main/sample/data/snort/parsed/SnortParsed | 3 - .../src/main/sample/data/snort/raw/SnortOutput | 3 - .../sample/data/squid/parsed/SquidExampleParsed | 2 - .../sample/data/squid/raw/SquidExampleOutput | 2 - .../data/websphere/parsed/WebsphereParsed | 5 - .../data/websphere/raw/WebsphereOutput.txt | 5 - .../src/main/sample/data/yaf/indexed/YafIndexed | 10 - .../sample/data/yaf/parsed/YafExampleParsed | 10 - .../main/sample/data/yaf/raw/YafExampleOutput | 10 - metron-platform/metron-management/README.md | 8 +- .../management/ConfigurationFunctions.java | 52 +- .../management/ConfigurationFunctionsTest.java | 53 +- .../management/ParserConfigFunctionsTest.java | 6 +- metron-platform/metron-parsers/README.md | 6 +- metron-platform/metron-parsers/pom.xml | 10 + .../src/main/config/zookeeper/parsers/asa.json | 7 - .../src/main/config/zookeeper/parsers/bro.json | 5 - .../main/config/zookeeper/parsers/snort.json | 5 - .../main/config/zookeeper/parsers/squid.json | 19 - .../config/zookeeper/parsers/websphere.json | 11 - .../src/main/config/zookeeper/parsers/yaf.json | 18 - .../org/apache/metron/parsers/BasicParser.java | 1 + .../org/apache/metron/parsers/GrokParser.java | 211 ---- .../metron/parsers/asa/BasicAsaParser.java | 240 ---- .../apache/metron/parsers/bolt/ParserBolt.java | 25 +- .../metron/parsers/bolt/ParserLoader.java | 95 ++ .../metron/parsers/bro/BasicBroParser.java | 180 --- .../apache/metron/parsers/bro/JSONCleaner.java | 91 -- .../apache/metron/parsers/cef/CEFParser.java | 274 ---- .../parsers/fireeye/BasicFireEyeParser.java | 222 ---- .../apache/metron/parsers/grok/GrokBuilder.java | 231 ++++ .../apache/metron/parsers/grok/GrokParser.java | 175 +++ .../parsers/interfaces/MessageParser.java | 3 + .../metron/parsers/ise/BasicIseParser.java | 101 -- .../apache/metron/parsers/ise/ISEParser.java | 660 ---------- .../org/apache/metron/parsers/ise/ISEParser.jj | 12 - .../metron/parsers/ise/ISEParserConstants.java | 69 - .../parsers/ise/ISEParserTokenManager.java | 676 ---------- .../metron/parsers/ise/JavaCharStream.java | 633 ---------- .../metron/parsers/ise/ParseException.java | 204 --- .../org/apache/metron/parsers/ise/Token.java | 148 --- .../metron/parsers/ise/TokenMgrError.java | 164 --- .../parsers/lancope/BasicLancopeParser.java | 95 -- .../parsers/logstash/BasicLogstashParser.java | 95 -- .../paloalto/BasicPaloAltoFirewallParser.java | 215 ---- .../metron/parsers/snort/BasicSnortParser.java | 214 ---- .../sourcefire/BasicSourcefireParser.java | 129 -- .../parsers/topology/ParserTopologyBuilder.java | 10 +- .../parsers/websphere/GrokWebSphereParser.java | 143 --- .../src/main/resources/patterns/asa | 179 --- .../src/main/resources/patterns/fireeye | 9 - .../src/main/resources/patterns/sourcefire | 30 - .../src/main/resources/patterns/squid | 2 - .../src/main/resources/patterns/websphere | 37 - .../src/main/resources/patterns/yaf | 2 - .../apache/metron/parsers/GrokParserTest.java | 95 -- .../metron/parsers/SampleGrokParserTest.java | 97 -- .../apache/metron/parsers/SnortParserTest.java | 150 --- .../apache/metron/parsers/SquidParserTest.java | 101 -- .../apache/metron/parsers/YafParserTest.java | 100 -- .../metron/parsers/asa/BasicAsaParserTest.java | 188 --- .../metron/parsers/bro/BasicBroParserTest.java | 1189 ------------------ .../metron/parsers/cef/CEFParserTest.java | 291 ----- .../parsers/fireeye/BasicFireEyeParserTest.java | 76 -- .../metron/parsers/grok/GrokBuilderTest.java | 83 ++ .../metron/parsers/grok/GrokParserTest.java | 106 ++ .../parsers/grok/SampleGrokParserTest.java | 97 ++ .../parsers/integration/AsaIntegrationTest.java | 37 - .../parsers/integration/BroIntegrationTest.java | 37 - .../integration/JSONMapIntegrationTest.java | 4 +- .../integration/ParserIntegrationTest.java | 62 +- .../integration/SnortIntegrationTest.java | 37 - .../integration/SquidIntegrationTest.java | 37 - .../integration/WebSphereIntegrationTest.java | 40 - .../parsers/integration/YafIntegrationTest.java | 37 - .../validation/PathedSampleDataValidation.java | 61 + .../validation/SampleDataValidation.java | 22 +- .../metron/parsers/ise/BasicIseParserTest.java | 56 - .../parsers/lancope/BasicLancopeParserTest.java | 57 - .../BasicPaloAltoFirewallParserTest.java | 57 - .../sourcefire/BasicSourcefireParserTest.java | 58 - .../websphere/GrokWebSphereParserTest.java | 260 ---- ...pleHbaseEnrichmentWriterIntegrationTest.java | 32 +- .../integration/WriterBoltIntegrationTest.java | 47 +- .../test/resources/TestSchemas/BroSchema.json | 28 - .../test/resources/TestSchemas/IseSchema.json | 21 - .../resources/TestSchemas/LancopeSchema.json | 28 - .../test/resources/TestSchemas/PcapSchema.json | 22 - .../resources/TestSchemas/SourcefireSchema.json | 34 - .../config/BasicFireEyeParserTest.config | 20 - .../resources/config/BasicIseParserTest.config | 20 - .../config/BasicLancopeParserTest.config | 20 - .../BasicPaloAltoFirewallParserTest.config | 20 - .../config/BasicSourcefireParserTest.config | 20 - .../test/resources/config/BroParserTest.config | 20 - .../resources/logData/FireEyeParserTest.txt | 8 - .../test/resources/logData/IseParserTest.txt | 308 ----- .../resources/logData/LancopeParserTest.txt | 1 - .../logData/PaloAltoFirewallParserTest.txt | 2 - .../resources/logData/SourcefireParserTest.txt | 3 - .../org/apache/metron/parsers/cef/adallom.cef | 1 - .../apache/metron/parsers/cef/adallom.schema | 37 - .../org/apache/metron/parsers/cef/cyberark.cef | 1 - .../org/apache/metron/parsers/cef/cyberark.json | 21 - .../apache/metron/parsers/cef/cyberark.schema | 38 - .../org/apache/metron/parsers/cef/palo.cef | 1 - .../org/apache/metron/parsers/cef/palo.schema | 38 - .../org/apache/metron/parsers/cef/waf.cef | 1 - .../org/apache/metron/parsers/cef/waf.schema | 67 - .../src/test/resources/otherPatterns/common | 96 ++ .../src/test/resources/patterns/test | 2 + .../java/org/apache/metron/TestConstants.java | 4 + .../metron/test/utils/ResourceCopier.java | 96 ++ .../metron/test/utils/SampleDataUtils.java | 17 +- metron-platform/pom.xml | 1 + pom.xml | 17 +- .../metron-maas-service/index.html | 2 +- .../metron-management/index.html | 6 +- .../metron-platform/metron-parsers/index.html | 6 +- 555 files changed, 38637 insertions(+), 9757 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 12fd7cd..bcb3642 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ target/ */target/ *dependency-reduced-pom.xml +*.versionsBackup .idea *.iml *.iws http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 917b184..d85a7ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,7 @@ before_install: - time build_utils/verify_licenses.sh install: + - cd bundles-maven-plugin && mvn -q install && cd .. - time mvn -q -T 2C -DskipTests clean install script: http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 8599f0f..308c82a 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,12 @@ To start exploring the capabilities of Apache Metron [follow these instructions # Building Metron +Build the maven plugin required for Metron ( this should only have to be one once ) + +``` +$cd bundles-maven-plugin && mvn -q install && cd .. +``` + Build the full project and run tests: ``` $ mvn clean install @@ -109,10 +115,13 @@ component and Apache Kafka as the unified data bus. Some high level links to the relevant subparts of the architecture, for more information: -* [Parsers](metron-platform/metron-parsers) : Parsing data from kafka into the Metron data model and passing it downstream to Enrichment. +* [Parsers](metron-platform/metron-parsers) : Base classes for parsing data from kafka into the Metron data model and passing it downstream to Enrichment. * [Enrichment](metron-platform/metron-enrichment) : Enriching data post-parsing and providing the ability to tag a message as an alert and assign a risk triage level via a custom rule language. * [Indexing](metron-platform/metron-indexing) : Indexing the data post-enrichment into HDFS, Elasticsearch or Solr. +Metron Extensions +* [Metron Parser Extensions](metron-platform/metron-extensions/metron-parser-extensions) : Parser Extensions parsing data from supported sources + Some useful utilities that cross all of these parts of the architecture: * [Stellar](metron-platform/metron-common) : A custom data transformation language that is used throughout metron from simple field transformation to expressing triage rules. * [Model as a Service](metron-analytics/metron-maas-service) : A Yarn application which can deploy machine learning and statistical models onto the cluster along with the associated Stellar functions to be able to call out to them in a scalable manner. http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/bundles-lib/README.md ---------------------------------------------------------------------- diff --git a/bundles-lib/README.md b/bundles-lib/README.md new file mode 100644 index 0000000..d2b41a4 --- /dev/null +++ b/bundles-lib/README.md @@ -0,0 +1,171 @@ +# Apache Metron Bundles + +Apache Metron Bundles and this documentation are a derivitave of the [Apache Nifi](http://www.nifi.apache.org) [NARs](http://nifi.apache.org/developer-guide.html). + +When software from many different organizations is all hosted within +the same environment, Java ClassLoaders quickly +become a concern. If multiple components have a dependency on the same +library but each depends on a different +version, many problems arise, typically resulting in unexpected +behavior or `NoClassDefFoundError` errors occurring. +In order to prevent these issues from becoming problematic, Apache NiFi +introduces the notion of a NiFi Archive, or NAR. The Apache Metron project has adapted and extended the NAR system as +Apache Metron Bundles. + +A BUNDLE allows several components and their dependencies to be packaged +together into a single package. +The BUNDLE package is then provided ClassLoader isolation from other BUNDLES +packages. Developers should always deploy their Apache Metron Extensions as BUNDLE packages. + +To achieve this, a developer creates a new Maven Artifact, which we +refer to as the BUNDLE artifact. The packaging is +set to `bundle`. The `dependencies` section of the POM is then created so +that the BUNDLE has a dependency on all Extension Components that are to be included within the BUNDLE. + +In order to use a packaging of `bundle`, we must use the `bundles-maven-plugin` module. +This is included by adding the following snippet to the bundle's pom.xml:> +> 0.4.1 below should be the current metron version +```xml +<build> + <plugins> + <plugin> + <groupId>org.apache.metron</groupId> + <artifactId>bundles-maven-plugin</artifactId> + <version>0.4.1</version> + <extensions>true</extensions> + </plugin> + </plugins> +</build> +``` + +The bundles-maven-plugin is included in the projects created by Apache Metron Extension maven archetypes. + + +The BUNDLE is able to have one dependency that is of type `bundle`. If more +than one dependency is specified that is of type +`bundle`, then the bundles-maven-plugin will error. If BUNDLE A adds a +dependency on BUNDLE B, this will *not* result in +BUNDLE A packaging all of the components of BUNDLE B. Rather, this will add +a `Bundle-Dependency-Id` element to the `MANIFEST.MF` +file of BUNDLE A. This will result in setting the ClassLoader of BUNDLE B as +the Parent ClassLoader of BUNDLE A. In this case, +we refer to BUNDLE B as the _Parent_ of BUNDLE A. + +## Per-Instance ClassLoading + +The bundles-lib provides the `@RequiresInstanceClassLoading` annotation to further expand and isolate the libraries +available on a componentâs classpath. You can annotate a extension class with `@RequiresInstanceClassLoading` +to indicate that the instance ClassLoader for the component requires a copy of all the resources in the +component's BUNDLE ClassLoader. When `@RequiresInstanceClassLoading` is not present, the +instance ClassLoader simply has it's parent ClassLoader set to the BUNDLE ClassLoader, rather than +copying resources. + +Because @RequiresInstanceClassLoading copies resources from the BUNDLE ClassLoader for each instance of the +extension, use this capability judiciously in an environment where many extensions may be present. If ten instances of one extension are created, all classes +from the component's BUNDLE ClassLoader are loaded into memory ten times. This could eventually increase the +memory footprint significantly when enough instances of the component are created. + + +## Apache VFS + +The bundles-lib utilizes the Apache VFS library for loading bundles. Bundles, which are zip files of a known structure, containing +jars of dependencies may be loaded by VFS as File Systems *themselves*, and in tern each jar in the bundle can be loaded by VFS as a File System. + +The VFSBundleClassloader therefore loads the bundle as a Jar File System, and also each dependency jar as a filesystem. This allows +for the complete loading of the bundle without having to unpack the bundle into a working directory. + +This is significantly different from the original Nifi implementation. + +## BundleSystem + +The BundleSystem class provides a useful and simple interface for using Bundles and instantiated class instances. +While the raw classes may be used, in many cases the BundleSystem will be sufficient and reduce complexity. + +Without BundleSystem, the minimum required to instantiate a class (simplified): + +```java +public static Optional<MessageParser<JSONObject>> loadParser(Map stormConfig, CuratorFramework client, SensorParserConfig parserConfig){ + MessageParser<JSONObject> parser = null; + try { + // fetch the BundleProperties from zookeeper + Optional<BundleProperties> bundleProperties = getBundleProperties(client); + + // create the FileSystemManager + FileSystemManager fileSystemManager = FileSystemManagerFactory.createFileSystemManager(new String[] {props.getArchiveExtension()}); + + // ADD in the classes we are going to support for plugins + ArrayList<Class> classes = new ArrayList<>(); + for( Map.Entry<String,String> entry : props.getBundleExtensionTypes().entrySet()){ + classes.add(Class.forName(entry.getValue())); + } + + // create the SystemBundle ( the bundle for the types that may be in the system classloader already ) + Bundle systemBundle = ExtensionManager.createSystemBundle(fileSystemManager, props); + + // get the correct bundle library directories from properties + List<URI> libDirs = props.getBundleLibraryDirectories(); + List<FileObject> libFileObjects = new ArrayList<>(); + for(URI libUri : libDirs){ + FileObject fileObject = fileSystemManager.resolveFile(libUri); + if(fileObject.exists()){ + libFileObjects.add(fileObject); + } + } + + // Initialize everything + BundleClassLoaders.getInstance().init(fileSystemManager, libFileObjects, props); + ExtensionManager.getInstance().init(classes, systemBundle, BundleClassLoaders.getInstance().getBundles()); + + // everything is ready, create our instance + parser = BundleThreadContextClassLoader.createInstance(parserConfig.getParserClassName(),MessageParser.class,props); + + }catch(Exception e){ + LOG.error("Failed to load parser " + parserConfig.getParserClassName(),e); + return Optional.empty(); + } + return Optional.of(parser); + } + +``` + +This is a lot to do, and a lot of unnecessary boilerplate code + +Using the BundleSystem class however: + +```java +public static Optional<MessageParser<JSONObject>> loadParser(Map stormConfig, + CuratorFramework client, SensorParserConfig parserConfig) { + MessageParser<JSONObject> parser = null; + try { + // fetch the BundleProperties from zookeeper + Optional<BundleProperties> bundleProperties = getBundleProperties(client); + BundleProperties props = bundleProperties.get(); + + // create the BundleSystem + // we only need to pass in the properties + BundleSystem bundleSystem = new BundleSystem.Builder().withBundleProperties(props).build(); + + // create our instance + parser = bundleSystem + .createInstance(parserConfig.getParserClassName(), MessageParser.class); + } else { + LOG.error("BundleProperties are missing!"); + } + } catch (Exception e) { + LOG.error("Failed to load parser " + parserConfig.getParserClassName(), e); + return Optional.empty(); + } + return Optional.of(parser); +} + +``` + +As we can see, this is much easier. + +With the BundleSystem, you may the defaults by calling the builder with : + +- withSystemBundle +- withFileSystemManager +- withExtensionClasses + + http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/bundles-lib/pom.xml ---------------------------------------------------------------------- diff --git a/bundles-lib/pom.xml b/bundles-lib/pom.xml new file mode 100644 index 0000000..0aecde9 --- /dev/null +++ b/bundles-lib/pom.xml @@ -0,0 +1,185 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>Metron</artifactId> + <version>0.4.1</version> + </parent> + + <artifactId>bundles-lib</artifactId> + <packaging>jar</packaging> + + <licenses> + <license> + <name>The Apache Software License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + <distribution>repo</distribution> + </license> + </licenses> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.accumulo</groupId> + <artifactId>accumulo-start</artifactId> + <version>${global_accumulo_version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + </exclusion> + <exclusion> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.atteo.classindex</groupId> + <artifactId>classindex</artifactId> + <version>${global_classindex_version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${global_hadoop_version}</version> + <exclusions> + <exclusion> + <artifactId>servlet-api</artifactId> + <groupId>javax.servlet</groupId> + </exclusion> + <exclusion> + <artifactId>slf4j-log4j12</artifactId> + <groupId>org.slf4j</groupId> + </exclusion> + <exclusion> + <artifactId>log4j</artifactId> + <groupId>log4j</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <version>${global_hadoop_version}</version> + <exclusions> + <exclusion> + <artifactId>servlet-api</artifactId> + <groupId>javax.servlet</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + <version>${global_hbase_version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> + </dependency> + <!-- from accumulo --> + <!-- <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-vfs2-project</artifactId> + <version>2.1</version> + </dependency> --> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>${global_slf4j_version}</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <version>${global_slf4j_version}</version> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>${global_junit_version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.powermock</groupId> + <artifactId>powermock-module-junit4</artifactId> + <version>1.6.6</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${global_hadoop_version}</version> + <classifier>tests</classifier> + <exclusions> + <exclusion> + <artifactId>slf4j-log4j12</artifactId> + <groupId>org.slf4j</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <version>${global_hadoop_version}</version> + <classifier>tests</classifier> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${global_hadoop_version}</version> + <classifier>tests</classifier> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-integration-test</artifactId> + <version>${project.parent.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>${global_jar_version}</version> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/bundles-lib/src/main/java/org/apache/metron/bundles/BundleClassLoaders.java ---------------------------------------------------------------------- diff --git a/bundles-lib/src/main/java/org/apache/metron/bundles/BundleClassLoaders.java b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleClassLoaders.java new file mode 100644 index 0000000..946c71a --- /dev/null +++ b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleClassLoaders.java @@ -0,0 +1,353 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.bundles; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.lang.invoke.MethodHandles; +import java.net.URISyntaxException; +import java.util.*; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileSystemManager; +import org.apache.metron.bundles.bundle.Bundle; +import org.apache.metron.bundles.bundle.BundleCoordinates; +import org.apache.metron.bundles.bundle.BundleDetails; +import org.apache.metron.bundles.util.BundleProperties; +import org.apache.metron.bundles.util.BundleSelector; +import org.apache.metron.bundles.util.FileUtils; +import org.apache.metron.bundles.util.BundleUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A singleton class used to initialize the extension and framework classloaders. + */ +public final class BundleClassLoaders { + + private static volatile BundleClassLoaders bundleClassLoaders; + private static volatile InitContext initContext; + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** + * Holds the context from {@code BundleClassLoaders} initialization, + * being the coordinate to bundle mapping. + * + * After initialization these are not changed, and as such they + * are immutable. + * + */ + private final static class InitContext { + + private final List<FileObject> extensionDirs; + private final Map<String, Bundle> bundles; + private final BundleProperties properties; + + private InitContext( + final List<FileObject> extensionDirs, + final Map<String, Bundle> bundles, + final BundleProperties properties) { + this.extensionDirs = ImmutableList.copyOf(extensionDirs); + this.bundles = ImmutableMap.copyOf(bundles); + this.properties = properties; + } + } + + private BundleClassLoaders() { + } + + /** + * @return The singleton instance of the BundleClassLoaders + * @throws NotInitializedException if BundleClassLoaders has not been init'd + */ + public static BundleClassLoaders getInstance() throws NotInitializedException { + BundleClassLoaders result = bundleClassLoaders; + if (result == null) { + throw new NotInitializedException("BundleClassLoaders not initialized"); + } + return result; + } + + /** + * Uninitializes the BundleClassloaders. After calling this <code>init</code> must be called + * before the rest of the methods are called afterwards. + * This is for TESTING ONLY at this time. Reset does not unload or clear any loaded classloaders. + */ + @VisibleForTesting + public static void reset() { + synchronized (BundleClassLoaders.class) { + initContext = null; + bundleClassLoaders = null; + } + } + + /** + * Initializes and loads the BundleClassLoaders. This method must be called before the rest of the + * methods to access the classloaders are called. Multiple calls to this method will have no effect, + * unless a different set of extension directories is passed, which will result in an <code>IllegaStateException</code> + * + * @param fileSystemManager the FileSystemManager + * @param extensionsDirs where to find extension artifacts + * @param props BundleProperties + * @throws FileSystemException if any issue occurs while working with the bundle files. + * @throws java.lang.ClassNotFoundException if unable to load class definition + * @throws IllegalStateException when already initialized with a given set of extension directories + * and extensionDirs does not match + */ + public static void init(final FileSystemManager fileSystemManager, final List<FileObject> extensionsDirs, + BundleProperties props) + throws FileSystemException, ClassNotFoundException, URISyntaxException { + if (extensionsDirs == null || fileSystemManager == null) { + throw new NullPointerException("cannot have empty arguments"); + } + synchronized (BundleClassLoaders.class) { + if (bundleClassLoaders != null) { + throw new IllegalStateException("BundleClassloader already exists"); + } + BundleClassLoaders b = new BundleClassLoaders(); + InitContext ic = b.load(fileSystemManager, extensionsDirs, props); + initContext = ic; + bundleClassLoaders = b; + } + } + + private InitContext load(final FileSystemManager fileSystemManager, + final List<FileObject> extensionsDirs, BundleProperties props) + throws FileSystemException, ClassNotFoundException, URISyntaxException { + // get the system classloader + final ClassLoader systemClassLoader = ClassLoader.getSystemClassLoader(); + + // find all bundle files and create class loaders for them. + final Map<String, Bundle> directoryBundleLookup = new LinkedHashMap<>(); + final Map<String, ClassLoader> coordinateClassLoaderLookup = new HashMap<>(); + final Map<String, Set<BundleCoordinates>> idBundleLookup = new HashMap<>(); + + for (FileObject extensionsDir : extensionsDirs) { + // make sure the bundle directory is there and accessible + FileUtils.ensureDirectoryExistAndCanRead(extensionsDir); + + final List<FileObject> bundleDirContents = new ArrayList<>(); + FileObject[] dirFiles = extensionsDir.findFiles(new BundleSelector(props.getArchiveExtension())); + if (dirFiles != null) { + List<FileObject> fileList = Arrays.asList(dirFiles); + bundleDirContents.addAll(fileList); + } + + if (!bundleDirContents.isEmpty()) { + final List<BundleDetails> bundleDetails = new ArrayList<>(); + final Map<String, String> bundleCoordinatesToBundleFile = new HashMap<>(); + + // load the bundle details which includes bundle dependencies + for (final FileObject bundleFile : bundleDirContents) { + if(!bundleFile.exists() || !bundleFile.isFile()) { + continue; + } + BundleDetails bundleDetail = null; + try { + bundleDetail = getBundleDetails(bundleFile, props); + } catch (IllegalStateException e) { + logger.warn("Unable to load BUNDLE {} due to {}, skipping...", + new Object[]{bundleFile.getURL(), e.getMessage()}); + } + + // prevent the application from starting when there are two BUNDLEs with same group, id, and version + final String bundleCoordinate = bundleDetail.getCoordinates().getCoordinates(); + if (bundleCoordinatesToBundleFile.containsKey(bundleCoordinate)) { + final String existingBundleWorkingDir = bundleCoordinatesToBundleFile + .get(bundleCoordinate); + throw new IllegalStateException( + "Unable to load BUNDLE with coordinates " + bundleCoordinate + + " and bundle file " + bundleDetail.getBundleFile() + + " because another BUNDLE with the same coordinates already exists at " + + existingBundleWorkingDir); + } + + bundleDetails.add(bundleDetail); + bundleCoordinatesToBundleFile.put(bundleCoordinate, + bundleDetail.getBundleFile().getURL().toURI().toString()); + } + + for (final Iterator<BundleDetails> bundleDetailsIter = bundleDetails.iterator(); + bundleDetailsIter.hasNext(); ) { + final BundleDetails bundleDetail = bundleDetailsIter.next(); + // populate bundle lookup + idBundleLookup.computeIfAbsent(bundleDetail.getCoordinates().getId(), + id -> new HashSet<>()).add(bundleDetail.getCoordinates()); + } + + int bundleCount; + do { + // record the number of bundles to be loaded + bundleCount = bundleDetails.size(); + + // attempt to create each bundle class loader + for (final Iterator<BundleDetails> bundleDetailsIter = bundleDetails.iterator(); + bundleDetailsIter.hasNext(); ) { + final BundleDetails bundleDetail = bundleDetailsIter.next(); + final BundleCoordinates bundleDependencyCoordinate = bundleDetail + .getDependencyCoordinates(); + + // see if this class loader is eligible for loading + ClassLoader potentialBundleClassLoader = null; + if (bundleDependencyCoordinate == null) { + potentialBundleClassLoader = createBundleClassLoader(fileSystemManager, + bundleDetail.getBundleFile(), ClassLoader.getSystemClassLoader()); + } else { + final String dependencyCoordinateStr = bundleDependencyCoordinate + .getCoordinates(); + + // if the declared dependency has already been loaded + if (coordinateClassLoaderLookup.containsKey(dependencyCoordinateStr)) { + final ClassLoader bundleDependencyClassLoader = coordinateClassLoaderLookup + .get(dependencyCoordinateStr); + potentialBundleClassLoader = createBundleClassLoader( + fileSystemManager, bundleDetail.getBundleFile(), + bundleDependencyClassLoader); + } else { + // get all bundles that match the declared dependency id + final Set<BundleCoordinates> coordinates = idBundleLookup + .get(bundleDependencyCoordinate.getId()); + + // ensure there are known bundles that match the declared dependency id + if (coordinates != null && !coordinates + .contains(bundleDependencyCoordinate)) { + // ensure the declared dependency only has one possible bundle + if (coordinates.size() == 1) { + // get the bundle with the matching id + final BundleCoordinates coordinate = coordinates.stream() + .findFirst().get(); + + // if that bundle is loaded, use it + if (coordinateClassLoaderLookup + .containsKey(coordinate.getCoordinates())) { + logger.warn(String.format( + "While loading '%s' unable to locate exact BUNDLE dependency '%s'. Only found one possible match '%s'. Continuing...", + bundleDetail.getCoordinates().getCoordinates(), + dependencyCoordinateStr, + coordinate.getCoordinates())); + + final ClassLoader bundleDependencyClassLoader = coordinateClassLoaderLookup + .get(coordinate.getCoordinates()); + potentialBundleClassLoader = createBundleClassLoader( + fileSystemManager, bundleDetail.getBundleFile(), + bundleDependencyClassLoader); + } + } + } + } + } + + // if we were able to create the bundle class loader, store it and remove the details + final ClassLoader bundleClassLoader = potentialBundleClassLoader; + if (bundleClassLoader != null) { + directoryBundleLookup + .put(bundleDetail.getBundleFile().getURL().toURI().toString(), + new Bundle(bundleDetail, bundleClassLoader)); + coordinateClassLoaderLookup + .put(bundleDetail.getCoordinates().getCoordinates(), + bundleClassLoader); + bundleDetailsIter.remove(); + } + } + + // attempt to load more if some were successfully loaded this iteration + } while (bundleCount != bundleDetails.size()); + + // see if any bundle couldn't be loaded + for (final BundleDetails bundleDetail : bundleDetails) { + logger.warn(String + .format("Unable to resolve required dependency '%s'. Skipping BUNDLE '%s'", + bundleDetail.getDependencyCoordinates().getId(), + bundleDetail.getBundleFile().getURL().toURI().toString())); + } + } + } + return new InitContext(extensionsDirs, new LinkedHashMap<>(directoryBundleLookup), props); + } + + /** + * Creates a new BundleClassLoader. The parentClassLoader may be null. + * + * @param bundleFile the Bundle File + * @param parentClassLoader parent classloader of bundle + * @return the bundle classloader + * @throws FileSystemException ioe + * @throws ClassNotFoundException cfne + */ + private static ClassLoader createBundleClassLoader(final FileSystemManager fileSystemManager, + final FileObject bundleFile, final ClassLoader parentClassLoader) + throws FileSystemException, ClassNotFoundException { + logger.debug("Loading Bundle file: " + bundleFile.getURL()); + final ClassLoader bundleClassLoader = new VFSBundleClassLoader.Builder() + .withFileSystemManager(fileSystemManager) + .withBundleFile(bundleFile) + .withParentClassloader(parentClassLoader).build(); + logger.info( + "Loaded Bundle file: " + bundleFile.getURL() + " as class loader " + bundleClassLoader); + return bundleClassLoader; + } + + /** + * Loads the details for the specified BUNDLE. The details will be extracted from the manifest + * file. + * + * @param bundleFile the bundle file + * @return details about the Bundle + * @throws FileSystemException ioe + */ + private static BundleDetails getBundleDetails(final FileObject bundleFile, BundleProperties props) + throws FileSystemException { + return BundleUtil.fromBundleFile(bundleFile, props); + } + + /** + * @param extensionFile the bundle file + * @return the bundle for the specified bundle file. Returns null when no bundle exists + * @throws IllegalStateException if the bundles have not been loaded + */ + public Bundle getBundle(final FileObject extensionFile) { + if (initContext == null) { + throw new IllegalStateException("Extensions class loaders have not been loaded."); + } + + try { + return initContext.bundles.get(extensionFile.getURL().toURI().toString()); + } catch (URISyntaxException | FileSystemException e) { + if (logger.isDebugEnabled()) { + logger.debug("Unable to get extension classloader for bundle '{}'", + extensionFile.getName().toString()); + } + return null; + } + } + + /** + * @return the extensions that have been loaded + * @throws IllegalStateException if the extensions have not been loaded + */ + public Set<Bundle> getBundles() { + if (initContext == null) { + throw new IllegalStateException("Bundles have not been loaded."); + } + + return new LinkedHashSet<>(initContext.bundles.values()); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/bundles-lib/src/main/java/org/apache/metron/bundles/BundleCloseable.java ---------------------------------------------------------------------- diff --git a/bundles-lib/src/main/java/org/apache/metron/bundles/BundleCloseable.java b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleCloseable.java new file mode 100644 index 0000000..1f25c0e --- /dev/null +++ b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleCloseable.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.bundles; + +import java.io.Closeable; + +import java.lang.invoke.MethodHandles; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + */ +public class BundleCloseable implements Closeable { + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static BundleCloseable withBundleLoader() throws NotInitializedException{ + final ClassLoader current = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(BundleThreadContextClassLoader.getInstance()); + return new BundleCloseable(current); + } + + /** + * Sets the current thread context class loader to the specific appropriate class loader for the given + * component. If the component requires per-instance class loading then the class loader will be the + * specific class loader for instance with the given identifier, otherwise the class loader will be + * the BundleClassLoader. + * + * @param componentClass the component class + * @param componentIdentifier the identifier of the component + * @return BundleCloseable with the current thread context classloader jailed to the Bundle + * or instance class loader of the component + */ + public static BundleCloseable withComponentBundleLoader(final Class componentClass, final String componentIdentifier) throws NotInitializedException{ + final ClassLoader current = Thread.currentThread().getContextClassLoader(); + + ClassLoader componentClassLoader = ExtensionManager.getInstance().getInstanceClassLoader(componentIdentifier); + if (componentClassLoader == null) { + componentClassLoader = componentClass.getClassLoader(); + } + + Thread.currentThread().setContextClassLoader(componentClassLoader); + return new BundleCloseable(current); + } + + /** + * Sets the current thread context class loader to the provided class loader, and returns a BundleCloseable that will + * return the current thread context class loader to it's previous state. + * + * @param componentBundleLoader the class loader to set as the current thread context class loader + * + * @return BundleCloseable that will return the current thread context class loader to its previous state + */ + public static BundleCloseable withComponentBundleLoader(final ClassLoader componentBundleLoader) { + final ClassLoader current = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(componentBundleLoader); + return new BundleCloseable(current); + } + private final ClassLoader toSet; + + private BundleCloseable(final ClassLoader toSet) { + this.toSet = toSet; + } + + @Override + public void close() { + if (toSet != null) { + Thread.currentThread().setContextClassLoader(toSet); + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/bundles-lib/src/main/java/org/apache/metron/bundles/BundleManifestEntry.java ---------------------------------------------------------------------- diff --git a/bundles-lib/src/main/java/org/apache/metron/bundles/BundleManifestEntry.java b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleManifestEntry.java new file mode 100644 index 0000000..2df4172 --- /dev/null +++ b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleManifestEntry.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.bundles; + +/** + * Enumeration of entries that will be in a BUNDLE MANIFEST file. + */ +public enum BundleManifestEntry { + + PRE_GROUP("-Group"), + PRE_ID("-Id"), + PRE_VERSION("-Version"), + PRE_DEPENDENCY_GROUP("-Dependency-Group"), + PRE_DEPENDENCY_ID("-Dependency-Id"), + PRE_DEPENDENCY_VERSION("-Dependency-Version"), + BUILD_TAG("Build-Tag"), + BUILD_REVISION("Build-Revision"), + BUILD_BRANCH("Build-Branch"), + BUILD_TIMESTAMP("Build-Timestamp"), + BUILD_JDK("Build-Jdk"), + BUILT_BY("Built-By") + ; + + final String manifestName; + + BundleManifestEntry(String manifestName) { + this.manifestName = manifestName; + } + + public String getManifestName() { + return manifestName; + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/5f7454e4/bundles-lib/src/main/java/org/apache/metron/bundles/BundleMapper.java ---------------------------------------------------------------------- diff --git a/bundles-lib/src/main/java/org/apache/metron/bundles/BundleMapper.java b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleMapper.java new file mode 100644 index 0000000..6da5762 --- /dev/null +++ b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleMapper.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.bundles; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.lang.invoke.MethodHandles; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.jar.JarEntry; +import java.util.jar.JarInputStream; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemManager; +import org.apache.metron.bundles.bundle.BundleCoordinates; +import org.apache.metron.bundles.util.BundleProperties; +import org.apache.metron.bundles.util.BundleSelector; +import org.apache.metron.bundles.util.BundleUtil; +import org.apache.metron.bundles.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The BundleMapper loads all the Bundles available to the system and maps their extensions + * + */ +public final class BundleMapper { + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static String archiveExtension = BundleProperties.DEFAULT_ARCHIVE_EXTENSION; + private final static String META_FMT = "META-INF/services/%s"; + + public static ExtensionMapping mapBundles(final FileSystemManager fileSystemManager, + BundleProperties props) { + try { + final List<URI> bundleLibraryDirs = props.getBundleLibraryDirectories(); + final Map<FileObject, BundleCoordinates> bundlesToCoordinates = new HashMap<>(); + archiveExtension = props.getArchiveExtension(); + + final List<FileObject> bundleFiles = new ArrayList<>(); + + for (URI bundleLibraryDir : bundleLibraryDirs) { + + FileObject bundleDir = fileSystemManager.resolveFile(bundleLibraryDir); + + if(bundleDir.exists() == false) { + continue; + } + // Test if the source BUNDLEs can be read + FileUtils.ensureDirectoryExistAndCanRead(bundleDir); + + FileObject[] dirFiles = bundleDir.findFiles(new BundleSelector(archiveExtension)); + if (dirFiles != null) { + List<FileObject> fileList = Arrays.asList(dirFiles); + bundleFiles.addAll(fileList); + } + } + + if (!bundleFiles.isEmpty()) { + for (FileObject bundleFile : bundleFiles) { + bundlesToCoordinates + .put(bundleFile, BundleUtil.coordinateFromBundleFile(bundleFile, props)); + } + } + + final ExtensionMapping extensionMapping = new ExtensionMapping(); + mapExtensions(bundlesToCoordinates, extensionMapping, props); + return extensionMapping; + } catch (IOException | URISyntaxException e) { + logger.warn("Unable to load BUNDLE library bundles due to " + e + + " Will proceed without loading any further bundles"); + if (logger.isDebugEnabled()) { + logger.warn("", e); + } + } + + return null; + } + + private static void mapExtensions(final Map<FileObject, BundleCoordinates> bundlesToCoordinates, + final ExtensionMapping mapping, BundleProperties props) throws IOException { + for (final Map.Entry<FileObject, BundleCoordinates> entry : bundlesToCoordinates.entrySet()) { + final FileObject bundle = entry.getKey(); + final BundleCoordinates bundleCoordinates = entry.getValue(); + + mapExtentionsForCoordinate(mapping, bundleCoordinates, bundle, props); + } + } + + private static void mapExtentionsForCoordinate(final ExtensionMapping mapping, + final BundleCoordinates bundleCoordinates, final FileObject bundle, BundleProperties props) + throws IOException { + final FileObject bundleFileSystem = bundle.getFileSystem().getFileSystemManager() + .createFileSystem(bundle); + final FileObject deps = bundleFileSystem.resolveFile(VFSBundleClassLoader.DEPENDENCY_PATH); + final FileObject[] directoryContents = deps.getChildren(); + if (directoryContents != null) { + for (final FileObject file : directoryContents) { + if (file.getName().getExtension().equals("jar")) { + mapExtensionsForJarFileObject(bundleCoordinates, file, mapping, props); + } + } + } + } + + private static void mapExtensionsForJarFileObject(final BundleCoordinates coordinate, final FileObject jar, + final ExtensionMapping extensionMapping, final BundleProperties props) throws IOException { + final ExtensionMapping jarExtensionMapping = buildExtensionMappingForJar(coordinate, jar, + props); + + // skip if there are not components to document + if (jarExtensionMapping.isEmpty()) { + return; + } + + // merge the extension mapping found in this jar + extensionMapping.merge(jarExtensionMapping); + } + + + private static ExtensionMapping buildExtensionMappingForJar(final BundleCoordinates coordinate, + final FileObject jar, final BundleProperties props) throws IOException { + final ExtensionMapping mapping = new ExtensionMapping(); + + // The BundleProperties has configuration for the extension names and classnames + final Map<String, String> extensions = props.getBundleExtensionTypes(); + if (extensions.isEmpty()) { + logger.info("No Extensions configured in properties"); + return mapping; + } + JarEntry jarEntry; + try (final JarInputStream jarFile = new JarInputStream(jar.getContent().getInputStream())) { + + while ((jarEntry = jarFile.getNextJarEntry()) != null) { + for (Map.Entry<String, String> extensionEntry : extensions.entrySet()) { + if (jarEntry.getName().equals(String.format(META_FMT, extensionEntry.getValue()))) { + mapping.addAllExtensions(extensionEntry.getKey(), coordinate, + buildExtensionMappingForJar(jarFile, jarEntry)); + } + } + } + } + return mapping; + + } + + private static List<String> buildExtensionMappingForJar(final JarInputStream jarFile, + final JarEntry jarEntry) throws IOException { + final List<String> componentNames = new ArrayList<>(); + + if (jarEntry == null) { + return componentNames; + } + final BufferedReader reader = new BufferedReader(new InputStreamReader( + jarFile)); + String line; + while ((line = reader.readLine()) != null) { + final String trimmedLine = line.trim(); + if (!trimmedLine.isEmpty() && !trimmedLine.startsWith("#")) { + final int indexOfPound = trimmedLine.indexOf("#"); + final String effectiveLine = (indexOfPound > 0) ? trimmedLine.substring(0, + indexOfPound) : trimmedLine; + componentNames.add(effectiveLine); + } + } + return componentNames; + } + + private BundleMapper() { + } +}
