Merge pull request #14 from intenthq/upgrade-dependencies

Upgrade all dependencies, restructure project, bump to 1.4.0.

Author: albertpastrana

.travis.yml

@@ -2,16 +2,16 @@ language: scala
 before_install:
   - sudo apt-get update -qq
   - sudo apt-get install libboost-dev libboost-test-dev libboost-program-options-dev libevent-dev automake libtool flex bison pkg-config g++ libssl-dev
-  - wget http://www.us.apache.org/dist/thrift/0.9.2/thrift-0.9.2.tar.gz
-  - tar xfz thrift-0.9.2.tar.gz && mv thrift-0.9.2 .thrift-0.9.2
-  - cd .thrift-0.9.2 && ./configure --with-java --without-c_glib --without-cpp --without-csharp --without-python --without-ruby --without-haskell --without-perl --without-php --without-erlang --without-go --without-d --without-nodejs --without-lua && make && sudo make install && cd -
+  - wget http://www.us.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz
+  - tar xfz thrift-0.9.3.tar.gz && mv thrift-0.9.3 .thrift-0.9.3
+  - cd .thrift-0.9.3 && ./configure --with-java --without-c_glib --without-cpp --without-csharp --without-python --without-ruby --without-haskell --without-perl --without-php --without-erlang --without-go --without-d --without-nodejs --without-lua && make && sudo make install && cd -
 scala:
    - 2.11.7
 script:
-  - sbt ++2.11.7 test
-  - sbt ++2.10.5 test
+  - sbt ++2.11.8 test
+  - sbt ++2.10.6 test
 cache:
   directories:
-   - ~/.thrift-0.9.2
+   - ~/.thrift-0.9.3
    - ~/.ivy2
    - ~/.sbt

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015 Intent HQ
+Copyright (c) 2015-2017 Intent HQ
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -12,29 +12,29 @@ The following top level dependencies are published in Maven central:
 
 **Thrift support**:
 ```
-"com.intenthq.pucket" %% "pucket-thrift" % "1.3.0"
+"com.intenthq.pucket" %% "pucket-thrift" % "1.4.0"
 ```
 
 **Avro support**:
 ```
-"com.intenthq.pucket" %% "pucket-avro" % "1.3.0"
+"com.intenthq.pucket" %% "pucket-avro" % "1.4.0"
 ```
 
 **Spark connectors**:
 ```
-"com.intenthq.pucket" %% "pucket-spark" % "1.3.0"
+"com.intenthq.pucket" %% "pucket-spark" % "1.4.0"
 ```
 
 **MapReduce integration**:
 ```
-"com.intenthq.pucket" %% "pucket-mapreduce" % "1.3.0"
+"com.intenthq.pucket" %% "pucket-mapreduce" % "1.4.0"
 ```
 
 These dependencies should be combined depending on your usages; for example if you use Thrift and Spark then use the following:
 
 ```
-"com.intenthq.pucket" %% "pucket-thrift" % "1.3.0"
-"com.intenthq.pucket" %% "pucket-spark" % "1.3.0"
+"com.intenthq.pucket" %% "pucket-thrift" % "1.4.0"
+"com.intenthq.pucket" %% "pucket-spark" % "1.4.0"
 ```
 
 

avro/src/main/scala/com/intenthq/pucket/avro/AvroPucket.scala

@@ -1,5 +1,7 @@
 package com.intenthq.pucket.avro
 
+import scala.language.higherKinds
+
 import com.intenthq.pucket.avro.writer.AvroWriter
 import com.intenthq.pucket.reader.Reader
 import com.intenthq.pucket.util.PucketPartitioner

build.sbt

@@ -1,12 +1,13 @@
 import com.github.bigtoast.sbtthrift.ThriftPlugin
 import sbt.ExclusionRule
-import com.typesafe.sbt.SbtGit.{GitKeys => git}
+import com.typesafe.sbt.SbtGit.GitKeys._
 
-val specs2Ver = "3.6.4"
+val specs2Ver = "3.8.6"
 val parquetVer = "1.8.1"
 val hadoopVer = "2.7.3"
-val sparkVer = "2.0.0"
-val circeVersion = "0.5.1"
+val sparkVer = "2.1.0"
+val circeVersion = "0.7.0"
+val scalazVersion = "7.2.9"
 
 val pomInfo = (
   <url>https://github.com/intenthq/pucket</url>
@@ -29,11 +30,9 @@ val pomInfo = (
   </developers>
 )
 
-def excludeServlet(deps: Seq[ModuleID]) = deps.map(_.exclude("javax.servlet", "servlet-api"))
-
 lazy val commonSettings = Seq(
   organization := "com.intenthq.pucket",
-  version := "1.3.0",
+  version := "1.4.0",
   scalaVersion := "2.11.8",
   publishArtifact in Test := false,
   pomIncludeRepository := { _ => false },
@@ -47,21 +46,25 @@ lazy val commonSettings = Seq(
   pomExtra := pomInfo,
   resolvers += Opts.resolver.mavenLocalFile,
   autoAPIMappings := true,
-  libraryDependencies ++= excludeServlet(Seq(
-    "org.scalaz" %% "scalaz-core" % "7.1.3",
-    "org.mortbay.jetty" % "servlet-api" % "3.0.20100224" % "provided",
-    "org.apache.hadoop" % "hadoop-common" % hadoopVer % "provided",
-    "org.apache.hadoop" % "hadoop-mapreduce-client-core" % hadoopVer % "provided",
-    "org.apache.parquet" % "parquet-column" % parquetVer,
-    "org.apache.parquet" % "parquet-hadoop" % parquetVer
-  )),
   libraryDependencies ++= Seq(
-    "io.circe" %% "circe-core",
-    "io.circe" %% "circe-generic",
-    "io.circe" %% "circe-parser"
-  ).map(_ % circeVersion),
-  dependencyOverrides += "org.slf4j" % "slf4j-log4j12" % "1.7.12",
-  dependencyOverrides += "org.slf4j" % "slf4j-api" % "1.7.12",
+    "org.apache.hadoop" % "hadoop-common" % hadoopVer % "provided,test",
+    "org.slf4j" % "slf4j-api" % "1.7.24" % "test",
+    "org.slf4j" % "jcl-over-slf4j" % "1.7.24" % "test",
+    "org.slf4j" % "jul-to-slf4j" % "1.7.24" % "test",
+    "org.slf4j" % "log4j-over-slf4j" % "1.7.24" % "test",
+    "ch.qos.logback" % "logback-classic" % "1.2.1" % "test"
+  ),
+  excludeDependencies ++= Seq(
+    // Our logging strategy is to use Logback (logback-classic) which provides
+    // an SLF4J compatible API. We then import as many SLF4J bridges we can
+    // so that every logging library effectively works with our Logback based
+    // logging. Thus, anything outside of this needs to be excluded.
+    SbtExclusionRule(organization = "commons-logging"),
+    SbtExclusionRule(organization = "log4j", name = "log4j"),
+    SbtExclusionRule(organization = "org.slf4j", name = "slf4j-simple"),
+    SbtExclusionRule(organization = "org.slf4j", name = "slf4j-log4j12"),
+    SbtExclusionRule(organization = "org.slf4j", name = "slf4j-jdk14")
+  ),
   resolvers ++= Seq(
     Resolver.typesafeRepo("releases"),
     Resolver.sonatypeRepo("public"),
@@ -72,98 +75,79 @@ lazy val commonSettings = Seq(
   )
 )
 
-
 lazy val core = (project in file("core")).
   settings(commonSettings: _*).
   settings(
     name := "pucket-core",
     libraryDependencies ++= Seq(
-      "org.apache.commons" % "commons-lang3" % "3.4",
+      "org.scalaz" %% "scalaz-core" % scalazVersion,
+      "org.apache.commons" % "commons-lang3" % "3.5",
+      "io.circe" %% "circe-core" % circeVersion,
+      "io.circe" %% "circe-generic" % circeVersion,
+      "io.circe" %% "circe-parser" % circeVersion,
+      "org.apache.parquet" % "parquet-hadoop" % parquetVer,
+
       "org.specs2" %% "specs2-core" % specs2Ver % "test",
       "org.specs2" %% "specs2-scalacheck" % specs2Ver % "test"
     )
   )
 
-lazy val test = (project in file("test")).
+lazy val mapreduce = (project in file("mapreduce")).
   settings(commonSettings: _*).
   settings(
-    name := "pucket-test",
-    libraryDependencies ++= excludeServlet(Seq(
-      "org.slf4j" % "jul-to-slf4j" % "1.7.12",
-      "org.specs2" %% "specs2-core" % specs2Ver,
-      "org.specs2" %% "specs2-scalacheck" % specs2Ver,
-      "org.typelevel" %% "scalaz-specs2" % "0.4.0",
-      "org.scalaz" %% "scalaz-scalacheck-binding" % "7.1.3",
-      "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVer,
-      "org.apache.hadoop" % "hadoop-common" % hadoopVer,
-      "org.apache.hadoop" % "hadoop-mapreduce-client-core" % hadoopVer,
-      "org.apache.spark" %% "spark-core" % sparkVer excludeAll(
-        ExclusionRule(organization = "org.slf4j"),
-        ExclusionRule(organization = "log4j"),
-        ExclusionRule(organization = "org.scala-lang"),
-        ExclusionRule(organization = "javax.servlet", name = "servlet-api")
-        )
-    ))
-  ).dependsOn(core, mapreduce, spark)
+    name := "pucket-mapreduce",
+    libraryDependencies ++= Seq(
+      "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVer % "provided,test",
+      "org.specs2" %% "specs2-core" % specs2Ver % "test",
+      "org.specs2" %% "specs2-scalacheck" % specs2Ver % "test"
+    )
+  ).dependsOn(core % "compile->compile;test->test")
+
+lazy val spark = (project in file("spark")).
+  settings(commonSettings: _*).
+  settings(
+    name := "pucket-spark",
+    libraryDependencies ++= Seq(
+      "org.apache.spark" %% "spark-core" % sparkVer % "provided,test",
+      "org.specs2" %% "specs2-core" % specs2Ver % "test",
+      "org.typelevel" %% "scalaz-specs2" % "0.5.0" % "test"
+    )
+  ).dependsOn(core % "compile->compile;test->test", mapreduce % "compile->compile;test->test")
 
 lazy val thrift = (project in file("thrift")).
   settings(ThriftPlugin.thriftSettings: _*).
   settings(commonSettings: _*).
   settings(
     name := "pucket-thrift",
     libraryDependencies ++= Seq(
-      "org.apache.thrift" % "libthrift" % "0.9.2",
+      "org.apache.thrift" % "libthrift" % "0.9.3",
       "org.apache.parquet" % "parquet-thrift" % parquetVer
     )
-  ).dependsOn(core, test % "test->compile")
+  ).dependsOn(core % "compile->compile;test->test", mapreduce % "test->test", spark % "test->test")
 
 
 lazy val avro = (project in file("avro")).
-  settings(sbtavro.SbtAvro.avroSettings : _*).
+  enablePlugins(SbtAvro).
   settings(commonSettings: _*).
   settings(
     name := "pucket-avro",
-    libraryDependencies ++= excludeServlet(Seq(
+    libraryDependencies ++= Seq(
       "org.apache.avro" % "avro" % "1.7.7",
       "org.apache.avro" % "avro-compiler" % "1.7.7",
       "org.apache.parquet" % "parquet-avro" % parquetVer,
-      "com.twitter" %% "chill-avro" % "0.7.0" % "test"
-      ))
-  ).dependsOn(core, test % "test->compile")
-
-
-lazy val mapreduce = (project in file("mapreduce")).
-  settings(commonSettings: _*).
-  settings(
-    name := "pucket-mapreduce",
-    libraryDependencies ++= excludeServlet(Seq(
-      "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVer % "provided"
-    ))
-  ).dependsOn(core)
-
-lazy val spark = (project in file("spark")).
-  settings(commonSettings: _*).
-  settings(
-    name := "pucket-spark",
-    libraryDependencies ++= excludeServlet(Seq(
-      "org.apache.spark" %% "spark-core" % sparkVer % "provided" excludeAll(
-        ExclusionRule(organization = "org.slf4j"),
-        ExclusionRule(organization = "log4j"),
-        ExclusionRule(organization = "org.scala-lang"),
-        ExclusionRule(organization = "org.scalatest"),
-        ExclusionRule(organization = "javax.servlet", name = "servlet-api")
-       )
-    ))
-  ).dependsOn(core, mapreduce)
+      "com.twitter" %% "chill-avro" % "0.8.4" % "test"
+    )
+  ).dependsOn(core % "compile->compile;test->test", mapreduce % "test->test", spark % "test->test")
 
 lazy val pucket = (project in file(".")).
   settings(commonSettings: _*).
-  settings(unidocSettings: _*).
-  settings(site.settings ++ ghpages.settings: _*).
+  enablePlugins(ScalaUnidocPlugin, GhpagesPlugin).
   settings(
     name := "pucket",
     publishArtifact := false,
     publishTo := Some(Resolver.file("Unused transient repository", file("target/unusedrepo"))),
-    site.addMappingsToSiteDir(mappings in (ScalaUnidoc, packageDoc), "latest/api"),
-    git.gitRemoteRepo := "[email protected]:intenthq/pucket.git"
-  ).aggregate(core, test, thrift, avro, mapreduce, spark)
+    gitRemoteRepo := "[email protected]:intenthq/pucket.git",
+    siteSubdirName in ScalaUnidoc := "latest/api",
+    addMappingsToSiteDir(mappings in (ScalaUnidoc, packageDoc), siteSubdirName in ScalaUnidoc),
+    ghpagesNoJekyll := true
+  ).aggregate(core, thrift, avro, mapreduce, spark)

core/src/main/scala/com/intenthq/pucket/Pucket.scala

@@ -1,5 +1,6 @@
 package com.intenthq.pucket
 
+import scala.language.higherKinds
 import java.util.UUID
 
 import com.intenthq.pucket.reader.Reader
@@ -243,4 +244,3 @@ object Pucket {
       _ <- \/.fromTryCatchNonFatal(input.close())
     } yield metadata
 }
-

core/src/test/resources/logback.xml

@@ -0,0 +1,11 @@
+<configuration debug="true">
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="ERROR">
+        <appender-ref ref="STDOUT" />
+    </root>
+</configuration>

test/src/main/scala/com/intenthq/pucket/PucketSpec.scala renamed to core/src/test/scala/com/intenthq/pucket/PucketSpec.scala

@@ -9,7 +9,6 @@ trait TestLogging {
   SLF4JBridgeHandler.removeHandlersForRootLogger()
   LogManager.getLogManager.reset()
   SLF4JBridgeHandler.install()
-  Logger.getLogger("global").setLevel(Level.OFF)
   Class.forName(classOf[Log].getName)
   val parquetLogger = Logger.getLogger("parquet")
   parquetLogger.getHandlers.foreach(parquetLogger.removeHandler)