[ https://issues.apache.org/jira/browse/KAFKA-133?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Joshua Hartman updated KAFKA-133: --------------------------------- Labels: patch (was: ) Affects Version/s: 0.6 Status: Patch Available (was: Open) commit e6f067d538d4902f64f8397256fe8d967e036bc6 Author: Joshua Hartman <jhart...@linkedin.com> Date: Mon Oct 24 10:56:12 2011 -0700 Initial attempt at sbt upgrade. diff --git a/lib/sbt-launch.jar b/lib/sbt-launch.jar deleted file mode 100644 index 67ee369..0000000 Binary files a/lib/sbt-launch.jar and /dev/null differ diff --git a/project/Build.scala b/project/Build.scala new file mode 100644 index 0000000..085809b --- /dev/null +++ b/project/Build.scala @@ -0,0 +1,147 @@ +import sbt._ +import Keys._ +import xml.NodeSeq + +object BuildSettings { + val exclusions = Seq("javax", "jmxri", "jmxtools", "mail", "jms") + val ivyExclude = <dependencies>{exclusions.map ( e => <exclude module={e}/>)}</dependencies> + + val buildSettings = Defaults.defaultSettings ++ Seq ( + organization := "kafka", + version := "0.7", + crossScalaVersions := Seq("2.8.1", "2.9.1"), + ivyXML := ivyExclude + ) + + def buildPomExtra(pom: NodeSeq, name: String, desc: String) = { + pom ++ Seq( + <name> + {name} + </name>, + <description> + {desc} + </description>, + <url>http://incubator.apache.org/kafka</url>, + <licenses> + <license> + <name>Apache</name> + <url>http://svn.apache.org/repos/asf/incubator/kafka/trunk/LICENSE</url> + <distribution>repo</distribution> + </license> + </licenses>, + <scm> + <url>http://svn.apache.org/repos/asf/incubator/kafka/trunk/</url> + <connection>scm:svn:http://svn.apache.org/repos/asf/incubator/kafka/trunk</connection> + <developerConnection>scm:svn:https://foo.googlecode.com/svn/trunk/</developerConnection> + <connection>scm:git:git://github.com/linkedin-sna/norbert.git</connection> + </scm>, + <developers> + <developer> + <id>jkreps</id> + <name>Jay Kreps</name> + <url>http://www.linkedin.com/in/jaykreps</url> + </developer> + <developer> + <id>junrao</id> + <name>Jun Rao</name> + <url>http://www.linkedin.com/in/junrao</url> + </developer> + <developer> + <id>nehanarkhede</id> + <name>Joshua Hartman</name> + <url>http://www.linkedin.com/in/nehanarkhede</url> + </developer> + </developers> + ) + } +} + +object Resolvers { + val oracleRepo = "Oracle Maven 2 Repository" at "http://download.oracle.com/maven" + val jBossRepo = "JBoss Maven 2 Repository" at "http://repository.jboss.com/maven2" + val kafkaResolvers = Seq(oracleRepo, jBossRepo) +} + +object CoreDependencies { +// TODO jhartman: When sbt 0.11.1 is ready, we can use the following code instead of ivy xml +// val exclusions = Seq("javax", "jmxri", "jmxtools", "mail", "jms") map (n => ExclusionRule(name = n)) +// val log4j = ("log4j" % "log4j" % "1.2.15") excludeAll (exclusions :_*) + + val log4j = ("log4j" % "log4j" % "1.2.15") + val jopt = "net.sf.jopt-simple" % "jopt-simple" % "3.2" + val deps = Seq(log4j, jopt) +} + +object HadoopProducerDependencies { + val avro = "org.apache.avro" % "avro" % "1.4.1" + val jacksonCore = "org.codehaus.jackson" % "jackson-core-asl" % "1.5.5" + val jacksonMapper = "org.codehaus.jackson" % "jackson-mapper-asl" % "1.5.5" + val deps = Seq(avro, jacksonCore, jacksonMapper) +} + +object HadoopConsumerDependencies { + val jodaTime = "joda-time" % "joda-time" % "1.6" + val httpclient = "commons-httpclient" % "commons-httpclient" % "3.1" + val deps = Seq(jodaTime, httpclient) +} + +object TestDependencies { + val easymock = "org.easymock" % "easymock" % "3.0" % "test" + val junit = "junit" % "junit" % "4.1" % "test" + val scalaTest = "org.scalatest" % "scalatest" % "1.2" % "test" + val deps = Seq(easymock, junit, scalaTest) +} + +object KafkaBuild extends Build { + import BuildSettings._ + + lazy val core = Project("core", file("core"), + settings = buildSettings ++ Seq( + libraryDependencies ++= CoreDependencies.deps ++ TestDependencies.deps, + resolvers := Resolvers.kafkaResolvers + ) + ) + + lazy val examples = Project("examples", file("examples"), + settings = buildSettings + ) dependsOn (core) + + + lazy val perf = Project("perf", file("perf"), + settings = buildSettings + ) dependsOn (core) + + lazy val hadoopProducer = Project("hadoop-producer", file("hadoop-producer"), + settings = buildSettings ++ Seq( + libraryDependencies ++= HadoopProducerDependencies.deps + ) + ) dependsOn (core) + + lazy val hadoopConsumer = Project("hadoop-consumer", file("hadoop-consumer"), + settings = buildSettings ++ Seq( + libraryDependencies ++= HadoopConsumerDependencies.deps + ) + ) dependsOn (core) + + lazy val contrib = Project("contrib", file("contrib"), settings = buildSettings) aggregate(hadoopConsumer, hadoopProducer) + + lazy val root = Project("root", file("."), + settings = buildSettings ++ Seq( + pomExtra <<= (pomExtra, name, description) { buildPomExtra } + )) aggregate(core, examples, perf, contrib) + + lazy val full = Project( + id = "kafka", + base = file("full"), + settings = buildSettings ++ Seq( + description := "Includes all of kafka project in one", + libraryDependencies ++= CoreDependencies.deps ++ TestDependencies.deps ++ HadoopProducerDependencies.deps ++ HadoopConsumerDependencies.deps, + + (unmanagedJars in Compile) <<= (projects.map(unmanagedJars in Compile in _).join).map(_.flatten), + (unmanagedSourceDirectories in Compile) <<= projects.map(unmanagedSourceDirectories in Compile in _).join.apply(_.flatten), + (managedSourceDirectories in Compile) <<= projects.map(managedSourceDirectories in Compile in _).join.apply(_.flatten), + + pomExtra <<= (pomExtra, name, description) { buildPomExtra } + ) + ) +} diff --git a/project/build.properties b/project/build.properties deleted file mode 100644 index 7c14cce..0000000 --- a/project/build.properties +++ /dev/null @@ -1,10 +0,0 @@ -#Project properties -#Mon Feb 28 11:55:49 PST 2011 -project.name=Kafka -sbt.version=0.7.5 -project.version=0.7 -build.scala.versions=2.8.0 -contrib.root.dir=contrib -lib.dir=lib -target.dir=target/scala_2.8.0 -dist.dir=dist diff --git a/project/build/KafkaProject.scala b/project/build/KafkaProject.scala index ba682d5..77aafba 100644 --- a/project/build/KafkaProject.scala +++ b/project/build/KafkaProject.scala @@ -1,190 +1,190 @@ -import sbt._ - -class KafkaProject(info: ProjectInfo) extends ParentProject(info) with IdeaProject { - lazy val core = project("core", "core-kafka", new CoreKafkaProject(_)) - lazy val examples = project("examples", "java-examples", new KafkaExamplesProject(_), core) - lazy val perf = project("perf", "perf", new KafkaPerfProject(_), core) - lazy val contrib = project("contrib", "contrib", new ContribProject(_)) - - lazy val releaseZipTask = core.packageDistTask - - val releaseZipDescription = "Compiles every sub project, runs unit tests, creates a deployable release zip file with dependencies, config, and scripts." - lazy val releaseZip = releaseZipTask dependsOn(core.corePackageAction, core.test, examples.examplesPackageAction, perf.perfPackageAction, - contrib.producerPackageAction, contrib.consumerPackageAction) describedAs releaseZipDescription - - class CoreKafkaProject(info: ProjectInfo) extends DefaultProject(info) - with IdeaProject with CoreDependencies with TestDependencies { - val corePackageAction = packageAllAction - - //The issue is going from log4j 1.2.14 to 1.2.15, the developers added some features which required - // some dependencies on various sun and javax packages. - override def ivyXML = - <dependencies> - <exclude module="javax"/> - <exclude module="jmxri"/> - <exclude module="jmxtools"/> - <exclude module="mail"/> - <exclude module="jms"/> - </dependencies> - - override def repositories = Set(ScalaToolsSnapshots, "JBoss Maven 2 Repository" at "http://repository.jboss.com/maven2", - "Oracle Maven 2 Repository" at "http://download.oracle.com/maven", "maven.org" at "http://repo2.maven.org/maven2/") - - override def artifactID = "kafka" - override def filterScalaJars = false - - // build the executable jar's classpath. - // (why is it necessary to explicitly remove the target/{classes,resources} paths? hm.) - def dependentJars = { - val jars = - publicClasspath +++ mainDependencies.scalaJars --- mainCompilePath --- mainResourcesOutputPath - if (jars.get.find { jar => jar.name.startsWith("scala-library-") }.isDefined) { - // workaround bug in sbt: if the compiler is explicitly included, don't include 2 versions - // of the library. - jars --- jars.filter { jar => - jar.absolutePath.contains("/boot/") && jar.name == "scala-library.jar" - } - } else { - jars - } - } - - def dependentJarNames = dependentJars.getFiles.map(_.getName).filter(_.endsWith(".jar")) - override def manifestClassPath = Some(dependentJarNames.map { "libs/" + _ }.mkString(" ")) - - def distName = (artifactID + "-" + projectVersion.value) - def distPath = "dist" / distName ## - - def configPath = "config" ## - def configOutputPath = distPath / "config" - - def binPath = "bin" ## - def binOutputPath = distPath / "bin" - - def distZipName = { - "%s-%s.zip".format(artifactID, projectVersion.value) - } - - lazy val packageDistTask = task { - distPath.asFile.mkdirs() - (distPath / "libs").asFile.mkdirs() - binOutputPath.asFile.mkdirs() - configOutputPath.asFile.mkdirs() - - FileUtilities.copyFlat(List(jarPath), distPath, log).left.toOption orElse - FileUtilities.copyFlat(dependentJars.get, distPath / "libs", log).left.toOption orElse - FileUtilities.copy((configPath ***).get, configOutputPath, log).left.toOption orElse - FileUtilities.copy((binPath ***).get, binOutputPath, log).left.toOption orElse - FileUtilities.zip((("dist" / distName) ##).get, "dist" / distZipName, true, log) - None - } - - val PackageDistDescription = "Creates a deployable zip file with dependencies, config, and scripts." - lazy val packageDist = packageDistTask dependsOn(`package`, `test`) describedAs PackageDistDescription - - val cleanDist = cleanTask("dist" ##) describedAs("Erase any packaged distributions.") - override def cleanAction = super.cleanAction dependsOn(cleanDist) - - override def javaCompileOptions = super.javaCompileOptions ++ - List(JavaCompileOption("-source"), JavaCompileOption("1.5")) - } - - class KafkaExamplesProject(info: ProjectInfo) extends DefaultProject(info) - with IdeaProject - with CoreDependencies { - val examplesPackageAction = packageAllAction - val dependsOnCore = core - //The issue is going from log4j 1.2.14 to 1.2.15, the developers added some features which required - // some dependencies on various sun and javax packages. - override def ivyXML = - <dependencies> - <exclude module="javax"/> - <exclude module="jmxri"/> - <exclude module="jmxtools"/> - <exclude module="mail"/> - <exclude module="jms"/> - </dependencies> - - override def artifactID = "kafka-java-examples" - override def filterScalaJars = false - } - - class KafkaPerfProject(info: ProjectInfo) extends DefaultProject(info) - with IdeaProject - with CoreDependencies { - val perfPackageAction = packageAllAction - val dependsOnCore = core - //The issue is going from log4j 1.2.14 to 1.2.15, the developers added some features which required - // some dependencies on various sun and javax packages. - override def ivyXML = - <dependencies> - <exclude module="javax"/> - <exclude module="jmxri"/> - <exclude module="jmxtools"/> - <exclude module="mail"/> - <exclude module="jms"/> - </dependencies> - - override def artifactID = "kafka-perf" - override def filterScalaJars = false - } - - class ContribProject(info: ProjectInfo) extends ParentProject(info) with IdeaProject { - lazy val hadoopProducer = project("hadoop-producer", "hadoop producer", - new HadoopProducerProject(_), core) - lazy val hadoopConsumer = project("hadoop-consumer", "hadoop consumer", - new HadoopConsumerProject(_), core) - - val producerPackageAction = hadoopProducer.producerPackageAction - val consumerPackageAction = hadoopConsumer.consumerPackageAction - - class HadoopProducerProject(info: ProjectInfo) extends DefaultProject(info) - with IdeaProject - with CoreDependencies { - val producerPackageAction = packageAllAction - override def ivyXML = - <dependencies> - <exclude module="netty"/> - <exclude module="javax"/> - <exclude module="jmxri"/> - <exclude module="jmxtools"/> - <exclude module="mail"/> - <exclude module="jms"/> - </dependencies> - - val avro = "org.apache.avro" % "avro" % "1.4.1" - val jacksonCore = "org.codehaus.jackson" % "jackson-core-asl" % "1.5.5" - val jacksonMapper = "org.codehaus.jackson" % "jackson-mapper-asl" % "1.5.5" - } - - class HadoopConsumerProject(info: ProjectInfo) extends DefaultProject(info) - with IdeaProject - with CoreDependencies { - val consumerPackageAction = packageAllAction - override def ivyXML = - <dependencies> - <exclude module="netty"/> - <exclude module="javax"/> - <exclude module="jmxri"/> - <exclude module="jmxtools"/> - <exclude module="mail"/> - <exclude module="jms"/> - </dependencies> - - val jodaTime = "joda-time" % "joda-time" % "1.6" - val httpclient = "commons-httpclient" % "commons-httpclient" % "3.1" - } - } - - trait TestDependencies { - val easymock = "org.easymock" % "easymock" % "3.0" % "test" - val junit = "junit" % "junit" % "4.1" % "test" - val scalaTest = "org.scalatest" % "scalatest" % "1.2" % "test" - } - - trait CoreDependencies { - val log4j = "log4j" % "log4j" % "1.2.15" - val jopt = "net.sf.jopt-simple" % "jopt-simple" % "3.2" - } - -} +//import sbt._ +// +//class KafkaProject(info: ProjectInfo) extends ParentProject(info) with IdeaProject { +// lazy val core = project("core", "core-kafka", new CoreKafkaProject(_)) +// lazy val examples = project("examples", "java-examples", new KafkaExamplesProject(_), core) +// lazy val perf = project("perf", "perf", new KafkaPerfProject(_), core) +// lazy val contrib = project("contrib", "contrib", new ContribProject(_)) +// +// lazy val releaseZipTask = core.packageDistTask +// +// val releaseZipDescription = "Compiles every sub project, runs unit tests, creates a deployable release zip file with dependencies, config, and scripts." +// lazy val releaseZip = releaseZipTask dependsOn(core.corePackageAction, core.test, examples.examplesPackageAction, perf.perfPackageAction, +// contrib.producerPackageAction, contrib.consumerPackageAction) describedAs releaseZipDescription +// +// class CoreKafkaProject(info: ProjectInfo) extends DefaultProject(info) +// with IdeaProject with CoreDependencies with TestDependencies { +// val corePackageAction = packageAllAction +// +// //The issue is going from log4j 1.2.14 to 1.2.15, the developers added some features which required +// // some dependencies on various sun and javax packages. +// override def ivyXML = +// <dependencies> +// <exclude module="javax"/> +// <exclude module="jmxri"/> +// <exclude module="jmxtools"/> +// <exclude module="mail"/> +// <exclude module="jms"/> +// </dependencies> +// +// override def repositories = Set(ScalaToolsSnapshots, "JBoss Maven 2 Repository" at "http://repository.jboss.com/maven2", +// "Oracle Maven 2 Repository" at "http://download.oracle.com/maven", "maven.org" at "http://repo2.maven.org/maven2/") +// +// override def artifactID = "kafka" +// override def filterScalaJars = false +// +// // build the executable jar's classpath. +// // (why is it necessary to explicitly remove the target/{classes,resources} paths? hm.) +// def dependentJars = { +// val jars = +// publicClasspath +++ mainDependencies.scalaJars --- mainCompilePath --- mainResourcesOutputPath +// if (jars.get.find { jar => jar.name.startsWith("scala-library-") }.isDefined) { +// // workaround bug in sbt: if the compiler is explicitly included, don't include 2 versions +// // of the library. +// jars --- jars.filter { jar => +// jar.absolutePath.contains("/boot/") && jar.name == "scala-library.jar" +// } +// } else { +// jars +// } +// } +// +// def dependentJarNames = dependentJars.getFiles.map(_.getName).filter(_.endsWith(".jar")) +// override def manifestClassPath = Some(dependentJarNames.map { "libs/" + _ }.mkString(" ")) +// +// def distName = (artifactID + "-" + projectVersion.value) +// def distPath = "dist" / distName ## +// +// def configPath = "config" ## +// def configOutputPath = distPath / "config" +// +// def binPath = "bin" ## +// def binOutputPath = distPath / "bin" +// +// def distZipName = { +// "%s-%s.zip".format(artifactID, projectVersion.value) +// } +// +// lazy val packageDistTask = task { +// distPath.asFile.mkdirs() +// (distPath / "libs").asFile.mkdirs() +// binOutputPath.asFile.mkdirs() +// configOutputPath.asFile.mkdirs() +// +// FileUtilities.copyFlat(List(jarPath), distPath, log).left.toOption orElse +// FileUtilities.copyFlat(dependentJars.get, distPath / "libs", log).left.toOption orElse +// FileUtilities.copy((configPath ***).get, configOutputPath, log).left.toOption orElse +// FileUtilities.copy((binPath ***).get, binOutputPath, log).left.toOption orElse +// FileUtilities.zip((("dist" / distName) ##).get, "dist" / distZipName, true, log) +// None +// } +// +// val PackageDistDescription = "Creates a deployable zip file with dependencies, config, and scripts." +// lazy val packageDist = packageDistTask dependsOn(`package`, `test`) describedAs PackageDistDescription +// +// val cleanDist = cleanTask("dist" ##) describedAs("Erase any packaged distributions.") +// override def cleanAction = super.cleanAction dependsOn(cleanDist) +// +// override def javaCompileOptions = super.javaCompileOptions ++ +// List(JavaCompileOption("-source"), JavaCompileOption("1.5")) +// } +// +// class KafkaExamplesProject(info: ProjectInfo) extends DefaultProject(info) +// with IdeaProject +// with CoreDependencies { +// val examplesPackageAction = packageAllAction +// val dependsOnCore = core +// //The issue is going from log4j 1.2.14 to 1.2.15, the developers added some features which required +// // some dependencies on various sun and javax packages. +// override def ivyXML = +// <dependencies> +// <exclude module="javax"/> +// <exclude module="jmxri"/> +// <exclude module="jmxtools"/> +// <exclude module="mail"/> +// <exclude module="jms"/> +// </dependencies> +// +// override def artifactID = "kafka-java-examples" +// override def filterScalaJars = false +// } +// +// class KafkaPerfProject(info: ProjectInfo) extends DefaultProject(info) +// with IdeaProject +// with CoreDependencies { +// val perfPackageAction = packageAllAction +// val dependsOnCore = core +// //The issue is going from log4j 1.2.14 to 1.2.15, the developers added some features which required +// // some dependencies on various sun and javax packages. +// override def ivyXML = +// <dependencies> +// <exclude module="javax"/> +// <exclude module="jmxri"/> +// <exclude module="jmxtools"/> +// <exclude module="mail"/> +// <exclude module="jms"/> +// </dependencies> +// +// override def artifactID = "kafka-perf" +// override def filterScalaJars = false +// } +// +// class ContribProject(info: ProjectInfo) extends ParentProject(info) with IdeaProject { +// lazy val hadoopProducer = project("hadoop-producer", "hadoop producer", +// new HadoopProducerProject(_), core) +// lazy val hadoopConsumer = project("hadoop-consumer", "hadoop consumer", +// new HadoopConsumerProject(_), core) +// +// val producerPackageAction = hadoopProducer.producerPackageAction +// val consumerPackageAction = hadoopConsumer.consumerPackageAction +// +// class HadoopProducerProject(info: ProjectInfo) extends DefaultProject(info) +// with IdeaProject +// with CoreDependencies { +// val producerPackageAction = packageAllAction +// override def ivyXML = +// <dependencies> +// <exclude module="netty"/> +// <exclude module="javax"/> +// <exclude module="jmxri"/> +// <exclude module="jmxtools"/> +// <exclude module="mail"/> +// <exclude module="jms"/> +// </dependencies> +// +// val avro = "org.apache.avro" % "avro" % "1.4.1" +// val jacksonCore = "org.codehaus.jackson" % "jackson-core-asl" % "1.5.5" +// val jacksonMapper = "org.codehaus.jackson" % "jackson-mapper-asl" % "1.5.5" +// } +// +// class HadoopConsumerProject(info: ProjectInfo) extends DefaultProject(info) +// with IdeaProject +// with CoreDependencies { +// val consumerPackageAction = packageAllAction +// override def ivyXML = +// <dependencies> +// <exclude module="netty"/> +// <exclude module="javax"/> +// <exclude module="jmxri"/> +// <exclude module="jmxtools"/> +// <exclude module="mail"/> +// <exclude module="jms"/> +// </dependencies> +// +// val jodaTime = "joda-time" % "joda-time" % "1.6" +// val httpclient = "commons-httpclient" % "commons-httpclient" % "3.1" +// } +// } +// +// trait TestDependencies { +// val easymock = "org.easymock" % "easymock" % "3.0" % "test" +// val junit = "junit" % "junit" % "4.1" % "test" +// val scalaTest = "org.scalatest" % "scalatest" % "1.2" % "test" +// } +// +// trait CoreDependencies { +// val log4j = "log4j" % "log4j" % "1.2.15" +// val jopt = "net.sf.jopt-simple" % "jopt-simple" % "3.2" +// } +// +//} diff --git a/project/plugins/Plugins.scala b/project/plugins/Plugins.scala deleted file mode 100644 index 6938e30..0000000 --- a/project/plugins/Plugins.scala +++ /dev/null @@ -1,6 +0,0 @@ -import sbt._ - -class Plugins(info: ProjectInfo) extends PluginDefinition(info) { - val repo = "GH-pages repo" at "http://mpeltonen.github.com/maven/" - val idea = "com.github.mpeltonen" % "sbt-idea-plugin" % "0.1-SNAPSHOT" -} diff --git a/sbt b/sbt index 7d8b41e..657e990 100755 --- a/sbt +++ b/sbt @@ -1 +1,2 @@ -java -Xmx1024M -XX:MaxPermSize=512m -jar `dirname $0`/lib/sbt-launch.jar "$@" +java -Xmx1024M -XX:MaxPermSize=512m -jar `dirname $0`/sbt-launch.jar "$@" + > Publish kafka jar to a public maven repository > ---------------------------------------------- > > Key: KAFKA-133 > URL: https://issues.apache.org/jira/browse/KAFKA-133 > Project: Kafka > Issue Type: Improvement > Affects Versions: 0.6 > Reporter: Neha Narkhede > Labels: patch > > The released kafka jar must be download manually and then deploy to a private > repository before they can be used by a developer using maven2. > Similar to other Apache projects, it will be nice to have a way to publish > Kafka releases to a public maven repo. > In the past, we gave it a try using sbt publish to Sonatype Nexus maven repo, > but ran into some authentication problems. It will be good to revisit this > and get it resolved. -- This message is automatically generated by JIRA. If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.apache.org/jira/secure/ContactAdministrators!default.jspa For more information on JIRA, see: http://www.atlassian.com/software/jira