From f4ad8e7a7365111a51f268cc968e908f3b8d49cc Mon Sep 17 00:00:00 2001 From: ibuenros Date: Fri, 29 Jan 2016 11:07:02 -0800 Subject: [PATCH 1/2] Release 0.6.2 --- CHANGELOG | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index af7411d4b4e..852bb0d849f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,56 @@ -GOBBLIN 0.6.1 +GOBBLIN 0.6.2 ============= +## NEW FEATURES +* [Admin Dashboard] Added a web based GUI for exploring running and finished jobs in a running Gobblin daemon (thanks Eric Ogren). +* [Admin Dashboard] Added a CLI for finding jobs in the job history store and seeing their run details (thanks Eric Ogren). +* [Configuration Management] WIP: Configuration management library. Will enable Gobblin to be dataset aware, ie. to dynamically load and apply different configurations to each dataset in a single Gobblin job. +** APIs: APIs for configuration stores and configuration client. +** Configuration Library: loads low level configurations from a configuration store, resolves configuration dependencies / imports, and performs value interpolation. +* [Distcp] Allow using *.ready files as markers for files that should be copied, and deletion of *.ready files once the file has been copied. +* [Distcp] Added file filters to recursive copyable dataset for distcp. Allows to only copy files satisfying a filter under a base directory. +* [Distcp] Copied files that fail to be published are persisted for future runs. Future runs can recover the already copied file instead of re-doing the byte transfer. +* [JDBC] Can use password encryption for JDBC sources. +* [YARN] Added email notifications on YARN application shutdown. +* [YARN] Added event notifications on YARN container status changes. +* [Metrics] Added metric filters based on name and type of the metrics. +* [Dataset Management] POC embedded sql for config-driven retention management. +* [Exactly Once] POC for Gobblin managed exactly once semantics on publisher. + +## BUG FIXES +* **Core** File based source includes previously failed WorkUnits event if there are no new files in the source (thanks Joel Baranick). +* **Core** Ensure that output file list does not contain duplicates due to task retries (thanks Joel Baranick). +* **Core** Fix NPE in CliOptions. +* **Core/YARN** Limit Props -> Typesafe Config conversion to a few keys to prevent overwriting of certain properties. +* **Utility** Fixed writer mkdirs for S3. +* **Metrics** Made Scheduled Reporter threads into daemon threads to prevent hanging application. +* **Metrics** Fixed enqueuing of events on event reporters that was causing job failure if event frequency was too high. +* **Build** Fix POM dependencies on gobblin-rest-api. +* **Build** Added conjars and cloudera repository to all projects (fixes builds for certain users). +* **Build** Fix the distribution tarball creation (thanks Joel Baranick). +* **Build** Added option to exclude Hadoop and Hive jars from distribution tarball. +* **Build** Removed log4j.properties from runtime resources. +* **Compaction** Fixed main class in compaction manifest file (thanks Lorand Bendig). +* **JDBC** Correctly close JDBC connections. + +## IMPROVEMENTS +* [Build] Add support for publishing libraries to maven local (thanks Joel Baranick). +* [Build] In preparation to Gradle 2 migration, added ext. prefix to custom gradle properties. +* [Build] Can generate project dependencies graph in dot format. +* [Metrics] Migrated Kafka reporter and Output stream reporter to Root Metrics Reporter managed reporting. +* [Metrics] The last metric emission in the application has a "final" tag for easier Hive identification. +* [Metrics] Metrics for Gobblin on YARN include cluster tags. +* [Hive] Upgraded Hive to version 1.0.1. +* [Distcp] Add file size to distcp success notifications. +* [Distcp] Each work unit in distcp contains exactly one Copyable File. +* [Distcp] Copy source can set upstream timestamps for SLA events emitted on publish time. +* [Scheduling] Added Gobblin Oozie config files. +* [Documentation] Improved javadocs. + + +GOBBLIN 0.6.1 +------------- + ## BUG FIXES - **Build/release** Adding build instrumentation for generation of rest-api-* artifacts From 116c79bd7bdfd022cb29fe80cd0bdcf4724bf31a Mon Sep 17 00:00:00 2001 From: ibuenros Date: Tue, 2 Feb 2016 11:24:21 -0800 Subject: [PATCH 2/2] Fixed pom generation for gobblin-rest-api project. --- build.gradle | 146 +++++++++--------- .../gobblin-rest-api/build.gradle | 16 +- 2 files changed, 82 insertions(+), 80 deletions(-) diff --git a/build.gradle b/build.gradle index c205fef1577..8b15e53c880 100644 --- a/build.gradle +++ b/build.gradle @@ -256,7 +256,7 @@ if (ext.javadocVersion.indexOf('-') > 0) { } ext.javadocPackages = new HashSet() -subprojects.each{Project pr -> +subprojects.each{Project pr -> if (file(pr.projectDir.absolutePath + "/src/main/java").exists()) { rootProject.ext.javadocPackages += pr.name } @@ -264,17 +264,85 @@ subprojects.each{Project pr -> subprojects { plugins.withType(JavaPlugin) { - + // Sometimes generating javadocs can lead to OOM. This may needs to be increased. // Also force javadocs to pick up system proxy settings if available javadoc { options.jFlags('-Xmx256m', '-Djava.net.useSystemProxies=true'); } - + rootProject.tasks.javadocTarball.dependsOn project.tasks.javadoc if ( rootProject.ext.javadocPackages.contains(project.name)) { rootProject.tasks.javadocTarball.into(project.name){from(fileTree(dir: "${project.buildDir}/docs/javadoc/"))} - } + } + } +} + +ext.pomAttributes = { + name "${project.name}" + packaging 'jar' + // optionally artifactId can be defined here + description 'Gobblin Ingestion Framework' + url 'https://github.com/linkedin/gobblin/' + + scm { + connection 'scm:git:git@github.com:linkedin/gobblin.git' + developerConnection 'scm:git:git@github.com:linkedin/gobblin.git' + url 'git@github.com:linkedin/gobblin.git' + } + + licenses { + license { + name 'The Apache License, Version 2.0' + url 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + + developers { + developer { + name 'Abhishek Tiwari' + organization 'LinkedIn' + } + developer { + name 'Chavdar Botev' + organization 'LinkedIn' + } + developer { + name 'Issac Buenrostro' + organization 'LinkedIn' + } + developer { + name 'Min Tu' + organization 'LinkedIn' + } + developer { + name 'Narasimha Veeramreddy' + organization 'LinkedIn' + } + developer { + name 'Pradhan Cadabam' + organization 'LinkedIn' + } + developer { + name 'Sahil Takiar' + organization 'LinkedIn' + } + developer { + name 'Shirshanka Das' + organization 'LinkedIn' + } + developer { + name 'Yinan Li' + organization 'LinkedIn' + } + developer { + name 'Ying Dai' + organization 'LinkedIn' + } + developer { + name 'Ziyang Liu' + organization 'LinkedIn' + } } } @@ -380,73 +448,7 @@ subprojects { authentication(userName: ossrhUsername, password: ossrhPassword) } - pom.project { - name "${project.name}" - packaging 'jar' - // optionally artifactId can be defined here - description 'Gobblin Ingestion Framework' - url 'https://github.com/linkedin/gobblin/' - - scm { - connection 'scm:git:git@github.com:linkedin/gobblin.git' - developerConnection 'scm:git:git@github.com:linkedin/gobblin.git' - url 'git@github.com:linkedin/gobblin.git' - } - - licenses { - license { - name 'The Apache License, Version 2.0' - url 'http://www.apache.org/licenses/LICENSE-2.0.txt' - } - } - - developers { - developer { - name 'Abhishek Tiwari' - organization 'LinkedIn' - } - developer { - name 'Chavdar Botev' - organization 'LinkedIn' - } - developer { - name 'Issac Buenrostro' - organization 'LinkedIn' - } - developer { - name 'Min Tu' - organization 'LinkedIn' - } - developer { - name 'Narasimha Veeramreddy' - organization 'LinkedIn' - } - developer { - name 'Pradhan Cadabam' - organization 'LinkedIn' - } - developer { - name 'Sahil Takiar' - organization 'LinkedIn' - } - developer { - name 'Shirshanka Das' - organization 'LinkedIn' - } - developer { - name 'Yinan Li' - organization 'LinkedIn' - } - developer { - name 'Ying Dai' - organization 'LinkedIn' - } - developer { - name 'Ziyang Liu' - organization 'LinkedIn' - } - } - } + pom.project pomAttributes } } } @@ -483,7 +485,7 @@ subprojects { "https://dropwizard.github.io/metrics/${dropwizardMetricsVersion}/apidocs/" rootProject.ext.javadocPackages.each { tasks.javadoc.options.linksOffline "http://linkedin.github.io/gobblin/javadoc/${javadocVersion}/${it}/", - "${rootProject.buildDir}/${it}/docs/javadoc/" + "${rootProject.buildDir}/${it}/docs/javadoc/" } afterEvaluate { diff --git a/gobblin-rest-service/gobblin-rest-api/build.gradle b/gobblin-rest-service/gobblin-rest-api/build.gradle index e20e6c223a6..ed7b30858d7 100644 --- a/gobblin-rest-service/gobblin-rest-api/build.gradle +++ b/gobblin-rest-service/gobblin-rest-api/build.gradle @@ -28,15 +28,15 @@ if (rootProject.ext.publishToMaven) { configure(uploadArchives.repositories.mavenDeployer) { addFilter('gobblin-rest-api-rest-client') {artifact, file -> artifact.name == 'gobblin-rest-api-rest-client' - }.artifactId = 'gobblin-rest-api-rest-client' + }.project(rootProject.pomAttributes).artifactId = 'gobblin-rest-api-rest-client' addFilter('gobblin-rest-api-data-template') {artifact, file -> artifact.name == 'gobblin-rest-api-data-template' - }.artifactId = 'gobblin-rest-api-data-template' + }.project(rootProject.pomAttributes).artifactId = 'gobblin-rest-api-data-template' addFilter('gobblin-rest-api') {artifact, file -> artifact.name == 'gobblin-rest-api' - }.artifactId = 'gobblin-rest-api' + }.project(rootProject.pomAttributes).artifactId = 'gobblin-rest-api' // artifact names for 'data-model', 'avro-schema' and 'rest-model' may be added as well if needed } } @@ -44,15 +44,15 @@ if (rootProject.ext.publishToMaven) { configure(install.repositories.mavenInstaller) { addFilter('gobblin-rest-api-rest-client') {artifact, file -> artifact.name == 'gobblin-rest-api-rest-client' - }.artifactId = 'gobblin-rest-api-rest-client' + }.project(rootProject.pomAttributes).artifactId = 'gobblin-rest-api-rest-client' addFilter('gobblin-rest-api-data-template') {artifact, file -> artifact.name == 'gobblin-rest-api-data-template' - }.artifactId = 'gobblin-rest-api-data-template' + }.project(rootProject.pomAttributes).artifactId = 'gobblin-rest-api-data-template' addFilter('gobblin-rest-api') {artifact, file -> artifact.name == 'gobblin-rest-api' - }.artifactId = 'gobblin-rest-api' + }.project(rootProject.pomAttributes).artifactId = 'gobblin-rest-api' // artifact names for 'data-model', 'avro-schema' and 'rest-model' may be added as well if needed } @@ -101,8 +101,8 @@ task restClientJavadocJar(type: Jar, dependsOn: 'restClientJavadoc') { classifier = 'javadoc' } -artifacts { - archives dataTemplateSourcesJar, dataTemplateJavadocJar, restClientSourcesJar, restClientJavadocJar +artifacts { + archives dataTemplateSourcesJar, dataTemplateJavadocJar, restClientSourcesJar, restClientJavadocJar } buildscript {