gobblin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [77/77] [abbrv] incubator-gobblin git commit: Merge pull request #2031 from abti/move_packages_to_apache
Date Mon, 31 Jul 2017 07:53:35 GMT
Merge pull request #2031 from abti/move_packages_to_apache


Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/5457af88
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/5457af88
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/5457af88

Branch: refs/heads/master
Commit: 5457af88d56b8fb89b172129fd1ff24ecdd4eba8
Parents: 9b68773 ee39a46
Author: Abhishek Tiwari <abhishektiwari.btech@gmail.com>
Authored: Mon Jul 31 00:51:25 2017 -0700
Committer: Abhishek Tiwari <abhishektiwari.btech@gmail.com>
Committed: Mon Jul 31 00:51:25 2017 -0700

----------------------------------------------------------------------
 bin/gobblin-admin.sh                            |    2 +-
 bin/gobblin-aws.sh                              |    2 +-
 bin/gobblin-cluster-master.sh                   |    2 +-
 bin/gobblin-cluster-worker.sh                   |    2 +-
 bin/gobblin-compaction.sh                       |    4 +-
 bin/gobblin-mapreduce.sh                        |    2 +-
 bin/gobblin-service.sh                          |    2 +-
 bin/gobblin-standalone.sh                       |    2 +-
 bin/gobblin-yarn.sh                             |    2 +-
 bin/gobblin.sh                                  |    2 +-
 bin/historystore-manager.sh                     |    2 +-
 bin/statestore-checker.sh                       |    2 +-
 bin/statestore-cleaner.sh                       |    2 +-
 .../gobblin/gradle/BuildProperties.groovy       |   50 -
 .../groovy/gobblin/gradle/BuildProperty.groovy  |   23 -
 .../gobblin/gradle/BuildProperties.groovy       |   50 +
 .../apache/gobblin/gradle/BuildProperty.groovy  |   23 +
 conf/aws/application.conf                       |    4 +-
 conf/gobblin-mapreduce.properties               |    2 +-
 conf/gobblin-standalone.properties              |    2 +-
 conf/service/application.conf                   |    2 +-
 conf/standalone/application.conf                |    2 +-
 conf/yarn/application.conf                      |    2 +-
 .../main/java/gobblin/admin/AdminWebServer.java |  130 --
 .../admin/DefaultAdminWebServerFactory.java     |   39 -
 .../src/main/java/gobblin/cli/AdminClient.java  |  137 --
 .../src/main/java/gobblin/cli/Cli.java          |  196 ---
 .../main/java/gobblin/cli/CliTablePrinter.java  |  200 ---
 .../src/main/java/gobblin/cli/Command.java      |   24 -
 .../src/main/java/gobblin/cli/JobCommand.java   |  215 ---
 .../java/gobblin/cli/JobInfoPrintUtils.java     |  256 ----
 .../apache/gobblin/admin/AdminWebServer.java    |  130 ++
 .../admin/DefaultAdminWebServerFactory.java     |   39 +
 .../org/apache/gobblin/cli/AdminClient.java     |  137 ++
 .../main/java/org/apache/gobblin/cli/Cli.java   |  196 +++
 .../org/apache/gobblin/cli/CliTablePrinter.java |  200 +++
 .../java/org/apache/gobblin/cli/Command.java    |   24 +
 .../java/org/apache/gobblin/cli/JobCommand.java |  215 +++
 .../apache/gobblin/cli/JobInfoPrintUtils.java   |  256 ++++
 .../java/gobblin/admin/AdminWebServerTest.java  |   92 --
 .../gobblin/admin/AdminWebServerTest.java       |   92 ++
 .../src/main/java/gobblin/Constructs.java       |   68 -
 .../src/main/java/gobblin/ack/Ackable.java      |   43 -
 .../gobblin/ack/BasicAckableForTesting.java     |   38 -
 .../java/gobblin/ack/HierarchicalAckable.java   |  122 --
 .../src/main/java/gobblin/annotation/Alias.java |   39 -
 .../src/main/java/gobblin/annotation/Alpha.java |   31 -
 .../src/main/java/gobblin/annotation/Beta.java  |   32 -
 .../main/java/gobblin/annotation/Stable.java    |   31 -
 .../java/gobblin/broker/BrokerConstants.java    |   25 -
 .../java/gobblin/broker/ResourceCoordinate.java |   36 -
 .../main/java/gobblin/broker/ResourceEntry.java |   47 -
 .../main/java/gobblin/broker/SimpleScope.java   |   33 -
 .../java/gobblin/broker/SimpleScopeType.java    |   74 --
 .../gobblin_scopes/GobblinScopeInstance.java    |   37 -
 .../gobblin_scopes/GobblinScopeTypes.java       |   81 --
 .../broker/gobblin_scopes/JobScopeInstance.java |   65 -
 .../gobblin_scopes/TaskScopeInstance.java       |   59 -
 .../java/gobblin/broker/iface/ConfigView.java   |   47 -
 .../broker/iface/NoSuchScopeException.java      |   28 -
 .../broker/iface/NotConfiguredException.java    |   32 -
 .../gobblin/broker/iface/ScopeInstance.java     |   42 -
 .../java/gobblin/broker/iface/ScopeType.java    |   67 -
 .../gobblin/broker/iface/ScopedConfigView.java  |   34 -
 .../broker/iface/SharedResourceFactory.java     |   46 -
 .../iface/SharedResourceFactoryResponse.java    |   24 -
 .../gobblin/broker/iface/SharedResourceKey.java |   38 -
 .../broker/iface/SharedResourcesBroker.java     |  123 --
 .../broker/iface/SubscopedBrokerBuilder.java    |   47 -
 .../main/java/gobblin/codec/StreamCodec.java    |   53 -
 .../java/gobblin/commit/CommitSequence.java     |  157 ---
 .../gobblin/commit/CommitSequenceStore.java     |   73 --
 .../main/java/gobblin/commit/CommitStep.java    |   44 -
 .../java/gobblin/commit/CommitStepBase.java     |   73 --
 .../java/gobblin/commit/DeliverySemantics.java  |   56 -
 .../gobblin/compat/hadoop/TextSerializer.java   |  140 --
 .../gobblin/compat/hadoop/WritableShim.java     |   35 -
 .../configuration/ConfigurationKeys.java        |  824 ------------
 .../configuration/ImmutableWorkUnitState.java   |   94 --
 .../java/gobblin/configuration/SourceState.java |  287 ----
 .../main/java/gobblin/configuration/State.java  |  567 --------
 .../java/gobblin/configuration/StateUtils.java  |   48 -
 .../gobblin/configuration/WorkUnitState.java    |  482 -------
 .../main/java/gobblin/converter/Converter.java  |  169 ---
 .../converter/DataConversionException.java      |   41 -
 .../converter/SchemaConversionException.java    |   44 -
 .../initializer/ConverterInitializer.java       |   28 -
 .../initializer/NoopConverterInitializer.java   |   43 -
 .../java/gobblin/crypto/CredentialStore.java    |   39 -
 .../gobblin/crypto/CredentialStoreProvider.java |   34 -
 .../java/gobblin/crypto/EncryptionProvider.java |   35 -
 .../src/main/java/gobblin/dataset/Dataset.java  |   28 -
 .../java/gobblin/dataset/DatasetsFinder.java    |   47 -
 .../java/gobblin/dataset/FileSystemDataset.java |   30 -
 .../gobblin/dataset/IterableDatasetFinder.java  |   36 -
 .../dataset/IterableDatasetFinderImpl.java      |   41 -
 .../src/main/java/gobblin/fork/CopyHelper.java  |   81 --
 .../gobblin/fork/CopyNotSupportedException.java |   32 -
 .../src/main/java/gobblin/fork/Copyable.java    |   34 -
 .../main/java/gobblin/fork/ForkOperator.java    |   71 -
 .../src/main/java/gobblin/fork/Forker.java      |  186 ---
 .../java/gobblin/initializer/Initializer.java   |   39 -
 .../gobblin/initializer/NoopInitializer.java    |   33 -
 .../java/gobblin/metadata/MetadataMerger.java   |   42 -
 .../java/gobblin/password/PasswordManager.java  |  233 ----
 .../java/gobblin/publisher/DataPublisher.java   |  128 --
 .../publisher/SingleTaskDataPublisher.java      |   78 --
 .../gobblin/publisher/UnpublishedHandling.java  |   40 -
 .../qualitychecker/row/RowLevelPolicy.java      |   84 --
 .../qualitychecker/task/TaskLevelPolicy.java    |   62 -
 .../FieldDoesNotExistException.java             |   27 -
 .../recordaccess/IncorrectTypeException.java    |   31 -
 .../gobblin/recordaccess/RecordAccessor.java    |   94 --
 .../recordaccess/RecordAccessorException.java   |   31 -
 .../recordaccess/RecordAccessorProvider.java    |   29 -
 .../gobblin/records/ControlMessageHandler.java  |   44 -
 .../records/FlushControlMessageHandler.java     |   50 -
 .../gobblin/records/RecordStreamConsumer.java   |   32 -
 .../gobblin/records/RecordStreamProcessor.java  |   59 -
 .../records/RecordStreamWithMetadata.java       |   75 --
 .../runtime/BasicTestControlMessage.java        |   39 -
 .../runtime/api/AdminWebServerFactory.java      |   44 -
 .../src/main/java/gobblin/runtime/api/Spec.java |   31 -
 .../runtime/api/SpecExecutorInstance.java       |   71 -
 .../api/SpecExecutorInstanceConsumer.java       |   30 -
 .../api/SpecExecutorInstanceProducer.java       |   44 -
 .../src/main/java/gobblin/source/Source.java    |   97 --
 .../gobblin/source/WorkUnitStreamSource.java    |   39 -
 .../extractor/CheckpointableWatermark.java      |   44 -
 .../source/extractor/ComparableWatermark.java   |   25 -
 .../source/extractor/DataRecordException.java   |   31 -
 .../gobblin/source/extractor/Extractor.java     |  134 --
 .../source/extractor/StreamingExtractor.java    |   41 -
 .../gobblin/source/extractor/Watermark.java     |   51 -
 .../source/extractor/WatermarkInterval.java     |   59 -
 .../extractor/WatermarkSerializerHelper.java    |   54 -
 .../source/workunit/BasicWorkUnitStream.java    |  144 ---
 .../java/gobblin/source/workunit/Extract.java   |  310 -----
 .../gobblin/source/workunit/ExtractFactory.java |   66 -
 .../source/workunit/ImmutableExtract.java       |  120 --
 .../source/workunit/ImmutableWorkUnit.java      |  104 --
 .../MissingExtractAttributeException.java       |   32 -
 .../gobblin/source/workunit/MultiWorkUnit.java  |  155 ---
 .../java/gobblin/source/workunit/WorkUnit.java  |  348 -----
 .../source/workunit/WorkUnitBinPacker.java      |   34 -
 .../gobblin/source/workunit/WorkUnitStream.java |   72 --
 .../source/workunit/WorkUnitWeighter.java       |   40 -
 .../java/gobblin/stream/ControlMessage.java     |   29 -
 .../gobblin/stream/FlushControlMessage.java     |   45 -
 .../java/gobblin/stream/RecordEnvelope.java     |  137 --
 .../main/java/gobblin/stream/StreamEntity.java  |  177 ---
 .../java/gobblin/testing/AssertWithBackoff.java |  216 ----
 .../java/gobblin/util/ClassAliasResolver.java   |  125 --
 .../main/java/gobblin/util/CompletedFuture.java |   56 -
 .../src/main/java/gobblin/util/Decorator.java   |   32 -
 .../main/java/gobblin/util/DecoratorUtils.java  |   66 -
 .../src/main/java/gobblin/util/FinalState.java  |   35 -
 .../java/gobblin/util/RecordCountProvider.java  |   62 -
 .../main/java/gobblin/writer/DataWriter.java    |  101 --
 .../java/gobblin/writer/DataWriterBuilder.java  |  144 ---
 .../main/java/gobblin/writer/Destination.java   |   80 --
 .../gobblin/writer/FluentDataWriterBuilder.java |   68 -
 .../java/gobblin/writer/FsWriterMetrics.java    |   73 --
 .../gobblin/writer/PartitionIdentifier.java     |   30 -
 .../java/gobblin/writer/WatermarkStorage.java   |   52 -
 .../java/gobblin/writer/WriterOutputFormat.java |   53 -
 .../main/java/gobblin/writer/WriterWrapper.java |   35 -
 .../initializer/NoopWriterInitializer.java      |   41 -
 .../writer/initializer/WriterInitializer.java   |   24 -
 .../java/org/apache/gobblin/Constructs.java     |   68 +
 .../java/org/apache/gobblin/ack/Ackable.java    |   43 +
 .../gobblin/ack/BasicAckableForTesting.java     |   38 +
 .../apache/gobblin/ack/HierarchicalAckable.java |  122 ++
 .../org/apache/gobblin/annotation/Alias.java    |   39 +
 .../org/apache/gobblin/annotation/Alpha.java    |   31 +
 .../org/apache/gobblin/annotation/Beta.java     |   32 +
 .../org/apache/gobblin/annotation/Stable.java   |   31 +
 .../apache/gobblin/broker/BrokerConstants.java  |   25 +
 .../gobblin/broker/ResourceCoordinate.java      |   36 +
 .../apache/gobblin/broker/ResourceEntry.java    |   47 +
 .../org/apache/gobblin/broker/SimpleScope.java  |   33 +
 .../apache/gobblin/broker/SimpleScopeType.java  |   74 ++
 .../gobblin_scopes/GobblinScopeInstance.java    |   37 +
 .../gobblin_scopes/GobblinScopeTypes.java       |   81 ++
 .../broker/gobblin_scopes/JobScopeInstance.java |   65 +
 .../gobblin_scopes/TaskScopeInstance.java       |   59 +
 .../apache/gobblin/broker/iface/ConfigView.java |   47 +
 .../broker/iface/NoSuchScopeException.java      |   28 +
 .../broker/iface/NotConfiguredException.java    |   32 +
 .../gobblin/broker/iface/ScopeInstance.java     |   42 +
 .../apache/gobblin/broker/iface/ScopeType.java  |   67 +
 .../gobblin/broker/iface/ScopedConfigView.java  |   34 +
 .../broker/iface/SharedResourceFactory.java     |   46 +
 .../iface/SharedResourceFactoryResponse.java    |   24 +
 .../gobblin/broker/iface/SharedResourceKey.java |   38 +
 .../broker/iface/SharedResourcesBroker.java     |  123 ++
 .../broker/iface/SubscopedBrokerBuilder.java    |   47 +
 .../org/apache/gobblin/codec/StreamCodec.java   |   53 +
 .../apache/gobblin/commit/CommitSequence.java   |  157 +++
 .../gobblin/commit/CommitSequenceStore.java     |   73 ++
 .../org/apache/gobblin/commit/CommitStep.java   |   44 +
 .../apache/gobblin/commit/CommitStepBase.java   |   73 ++
 .../gobblin/commit/DeliverySemantics.java       |   56 +
 .../gobblin/compat/hadoop/TextSerializer.java   |  140 ++
 .../gobblin/compat/hadoop/WritableShim.java     |   35 +
 .../configuration/ConfigurationKeys.java        |  824 ++++++++++++
 .../configuration/ImmutableWorkUnitState.java   |   94 ++
 .../gobblin/configuration/SourceState.java      |  287 ++++
 .../org/apache/gobblin/configuration/State.java |  567 ++++++++
 .../gobblin/configuration/StateUtils.java       |   48 +
 .../gobblin/configuration/WorkUnitState.java    |  482 +++++++
 .../org/apache/gobblin/converter/Converter.java |  169 +++
 .../converter/DataConversionException.java      |   41 +
 .../converter/SchemaConversionException.java    |   44 +
 .../initializer/ConverterInitializer.java       |   28 +
 .../initializer/NoopConverterInitializer.java   |   43 +
 .../apache/gobblin/crypto/CredentialStore.java  |   39 +
 .../gobblin/crypto/CredentialStoreProvider.java |   34 +
 .../gobblin/crypto/EncryptionProvider.java      |   35 +
 .../org/apache/gobblin/dataset/Dataset.java     |   28 +
 .../apache/gobblin/dataset/DatasetsFinder.java  |   47 +
 .../gobblin/dataset/FileSystemDataset.java      |   30 +
 .../gobblin/dataset/IterableDatasetFinder.java  |   36 +
 .../dataset/IterableDatasetFinderImpl.java      |   41 +
 .../org/apache/gobblin/fork/CopyHelper.java     |   81 ++
 .../gobblin/fork/CopyNotSupportedException.java |   32 +
 .../java/org/apache/gobblin/fork/Copyable.java  |   34 +
 .../org/apache/gobblin/fork/ForkOperator.java   |   71 +
 .../java/org/apache/gobblin/fork/Forker.java    |  186 +++
 .../apache/gobblin/initializer/Initializer.java |   39 +
 .../gobblin/initializer/NoopInitializer.java    |   33 +
 .../apache/gobblin/metadata/MetadataMerger.java |   42 +
 .../gobblin/password/PasswordManager.java       |  233 ++++
 .../apache/gobblin/publisher/DataPublisher.java |  128 ++
 .../publisher/SingleTaskDataPublisher.java      |   78 ++
 .../gobblin/publisher/UnpublishedHandling.java  |   40 +
 .../qualitychecker/row/RowLevelPolicy.java      |   84 ++
 .../qualitychecker/task/TaskLevelPolicy.java    |   62 +
 .../FieldDoesNotExistException.java             |   27 +
 .../recordaccess/IncorrectTypeException.java    |   31 +
 .../gobblin/recordaccess/RecordAccessor.java    |   94 ++
 .../recordaccess/RecordAccessorException.java   |   31 +
 .../recordaccess/RecordAccessorProvider.java    |   29 +
 .../gobblin/records/ControlMessageHandler.java  |   44 +
 .../records/FlushControlMessageHandler.java     |   50 +
 .../gobblin/records/RecordStreamConsumer.java   |   32 +
 .../gobblin/records/RecordStreamProcessor.java  |   59 +
 .../records/RecordStreamWithMetadata.java       |   75 ++
 .../runtime/BasicTestControlMessage.java        |   39 +
 .../runtime/api/AdminWebServerFactory.java      |   44 +
 .../org/apache/gobblin/runtime/api/Spec.java    |   31 +
 .../runtime/api/SpecExecutorInstance.java       |   71 +
 .../api/SpecExecutorInstanceConsumer.java       |   30 +
 .../api/SpecExecutorInstanceProducer.java       |   44 +
 .../java/org/apache/gobblin/source/Source.java  |   97 ++
 .../gobblin/source/WorkUnitStreamSource.java    |   39 +
 .../extractor/CheckpointableWatermark.java      |   44 +
 .../source/extractor/ComparableWatermark.java   |   25 +
 .../source/extractor/DataRecordException.java   |   31 +
 .../gobblin/source/extractor/Extractor.java     |  134 ++
 .../source/extractor/StreamingExtractor.java    |   41 +
 .../gobblin/source/extractor/Watermark.java     |   51 +
 .../source/extractor/WatermarkInterval.java     |   59 +
 .../extractor/WatermarkSerializerHelper.java    |   54 +
 .../source/workunit/BasicWorkUnitStream.java    |  144 +++
 .../apache/gobblin/source/workunit/Extract.java |  310 +++++
 .../gobblin/source/workunit/ExtractFactory.java |   66 +
 .../source/workunit/ImmutableExtract.java       |  120 ++
 .../source/workunit/ImmutableWorkUnit.java      |  104 ++
 .../MissingExtractAttributeException.java       |   32 +
 .../gobblin/source/workunit/MultiWorkUnit.java  |  155 +++
 .../gobblin/source/workunit/WorkUnit.java       |  348 +++++
 .../source/workunit/WorkUnitBinPacker.java      |   34 +
 .../gobblin/source/workunit/WorkUnitStream.java |   72 ++
 .../source/workunit/WorkUnitWeighter.java       |   40 +
 .../apache/gobblin/stream/ControlMessage.java   |   29 +
 .../gobblin/stream/FlushControlMessage.java     |   45 +
 .../apache/gobblin/stream/RecordEnvelope.java   |  137 ++
 .../org/apache/gobblin/stream/StreamEntity.java |  177 +++
 .../gobblin/testing/AssertWithBackoff.java      |  216 ++++
 .../apache/gobblin/util/ClassAliasResolver.java |  125 ++
 .../apache/gobblin/util/CompletedFuture.java    |   56 +
 .../java/org/apache/gobblin/util/Decorator.java |   32 +
 .../org/apache/gobblin/util/DecoratorUtils.java |   66 +
 .../org/apache/gobblin/util/FinalState.java     |   35 +
 .../gobblin/util/RecordCountProvider.java       |   62 +
 .../org/apache/gobblin/writer/DataWriter.java   |  101 ++
 .../gobblin/writer/DataWriterBuilder.java       |  144 +++
 .../org/apache/gobblin/writer/Destination.java  |   80 ++
 .../gobblin/writer/FluentDataWriterBuilder.java |   68 +
 .../apache/gobblin/writer/FsWriterMetrics.java  |   73 ++
 .../gobblin/writer/PartitionIdentifier.java     |   30 +
 .../apache/gobblin/writer/WatermarkStorage.java |   52 +
 .../gobblin/writer/WriterOutputFormat.java      |   53 +
 .../apache/gobblin/writer/WriterWrapper.java    |   35 +
 .../initializer/NoopWriterInitializer.java      |   41 +
 .../writer/initializer/WriterInitializer.java   |   24 +
 .../java/com/linkedin/gobblin/TestAlias.java    |    2 +-
 .../gobblin/ack/HierarchicalAckableTest.java    |  122 --
 .../gobblin_scopes/GobblinScopesTest.java       |   54 -
 .../java/gobblin/compat/TextSerializerTest.java |   79 --
 .../java/gobblin/configuration/StateTest.java   |   84 --
 .../java/gobblin/converter/ConverterTest.java   |  145 ---
 .../test/java/gobblin/fork/CopyHelperTest.java  |  104 --
 .../src/test/java/gobblin/fork/ForkerTest.java  |  141 --
 .../gobblin/password/PasswordManagerTest.java   |   99 --
 .../gobblin/source/extractor/TestWatermark.java |   52 -
 .../gobblin/source/extractor/WatermarkTest.java |   73 --
 .../source/workunit/ExtractFactoryTest.java     |   45 -
 .../java/gobblin/stream/RecordEnvelopeTest.java |  170 ---
 .../java/gobblin/stream/StreamEntityTest.java   |  149 ---
 .../gobblin/testing/AssertWithBackoffTest.java  |  110 --
 .../gobblin/util/ClassAliasResolverTest.java    |   78 --
 .../gobblin/writer/FsWriterMetricsTest.java     |   45 -
 .../gobblin/ack/HierarchicalAckableTest.java    |  122 ++
 .../gobblin_scopes/GobblinScopesTest.java       |   54 +
 .../gobblin/compat/TextSerializerTest.java      |   79 ++
 .../apache/gobblin/configuration/StateTest.java |   84 ++
 .../apache/gobblin/converter/ConverterTest.java |  145 +++
 .../org/apache/gobblin/fork/CopyHelperTest.java |  104 ++
 .../org/apache/gobblin/fork/ForkerTest.java     |  141 ++
 .../gobblin/password/PasswordManagerTest.java   |   99 ++
 .../gobblin/source/extractor/TestWatermark.java |   52 +
 .../gobblin/source/extractor/WatermarkTest.java |   73 ++
 .../source/workunit/ExtractFactoryTest.java     |   45 +
 .../gobblin/stream/RecordEnvelopeTest.java      |  170 +++
 .../apache/gobblin/stream/StreamEntityTest.java |  149 +++
 .../gobblin/testing/AssertWithBackoffTest.java  |  110 ++
 .../gobblin/util/ClassAliasResolverTest.java    |   78 ++
 .../gobblin/writer/FsWriterMetricsTest.java     |   45 +
 .../values/auditor/ValueAuditGenerator.java     |   93 --
 .../auditor/ValueAuditRuntimeMetadata.java      |  196 ---
 .../column/AbstractColumnProjectionPolicy.java  |   52 -
 .../policy/column/ColumnProjectionPolicy.java   |   52 -
 .../DefaultColumnProjectionPolicyFactory.java   |   78 --
 .../ProjectAllColumnProjectionPolicy.java       |   58 -
 .../policy/row/AbstractRowSelectionPolicy.java  |   40 -
 .../row/DefaultRowSelectionPolicyFactory.java   |   82 --
 .../values/policy/row/RowSelectionPolicy.java   |   34 -
 .../policy/row/SelectAllRowSelectionPolicy.java |   46 -
 .../gobblin/audit/values/sink/AuditSink.java    |   36 -
 .../values/sink/DefaultAuditSinkFactory.java    |   81 --
 .../gobblin/audit/values/sink/FsAuditSink.java  |  124 --
 .../values/auditor/ValueAuditGenerator.java     |   93 ++
 .../auditor/ValueAuditRuntimeMetadata.java      |  196 +++
 .../column/AbstractColumnProjectionPolicy.java  |   52 +
 .../policy/column/ColumnProjectionPolicy.java   |   52 +
 .../DefaultColumnProjectionPolicyFactory.java   |   78 ++
 .../ProjectAllColumnProjectionPolicy.java       |   58 +
 .../policy/row/AbstractRowSelectionPolicy.java  |   40 +
 .../row/DefaultRowSelectionPolicyFactory.java   |   82 ++
 .../values/policy/row/RowSelectionPolicy.java   |   34 +
 .../policy/row/SelectAllRowSelectionPolicy.java |   46 +
 .../gobblin/audit/values/sink/AuditSink.java    |   36 +
 .../values/sink/DefaultAuditSinkFactory.java    |   81 ++
 .../gobblin/audit/values/sink/FsAuditSink.java  |  124 ++
 .../gobblin/audit/values/FsAuditSinkTest.java   |   75 --
 .../java/gobblin/audit/values/MockSink.java     |   38 -
 .../audit/values/ValueAuditGeneratorTest.java   |   66 -
 .../values/ValueAuditRuntimeMetadataTest.java   |   67 -
 .../gobblin/audit/values/FsAuditSinkTest.java   |   75 ++
 .../apache/gobblin/audit/values/MockSink.java   |   38 +
 .../audit/values/ValueAuditGeneratorTest.java   |   66 +
 .../values/ValueAuditRuntimeMetadataTest.java   |   67 +
 .../gobblin/aws/AWSClusterSecurityManager.java  |  131 --
 .../gobblin/aws/AWSJobConfigurationManager.java |  235 ----
 .../src/main/java/gobblin/aws/AWSSdkClient.java |  524 --------
 .../java/gobblin/aws/AWSShutdownHandler.java    |  100 --
 .../gobblin/aws/CloudInitScriptBuilder.java     |  272 ----
 .../gobblin/aws/GobblinAWSClusterLauncher.java  |  668 ----------
 .../gobblin/aws/GobblinAWSClusterManager.java   |  187 ---
 .../aws/GobblinAWSConfigurationKeys.java        |  152 ---
 .../java/gobblin/aws/GobblinAWSTaskRunner.java  |  199 ---
 .../main/java/gobblin/aws/GobblinAWSUtils.java  |  137 --
 .../java/gobblin/aws/Log4jConfigHelper.java     |   62 -
 .../gobblin/aws/AWSClusterSecurityManager.java  |  131 ++
 .../gobblin/aws/AWSJobConfigurationManager.java |  235 ++++
 .../org/apache/gobblin/aws/AWSSdkClient.java    |  524 ++++++++
 .../apache/gobblin/aws/AWSShutdownHandler.java  |  100 ++
 .../gobblin/aws/CloudInitScriptBuilder.java     |  272 ++++
 .../gobblin/aws/GobblinAWSClusterLauncher.java  |  668 ++++++++++
 .../gobblin/aws/GobblinAWSClusterManager.java   |  187 +++
 .../aws/GobblinAWSConfigurationKeys.java        |  152 +++
 .../gobblin/aws/GobblinAWSTaskRunner.java       |  199 +++
 .../org/apache/gobblin/aws/GobblinAWSUtils.java |  137 ++
 .../apache/gobblin/aws/Log4jConfigHelper.java   |   62 +
 .../aws/AWSJobConfigurationManagerTest.java     |  154 ---
 .../gobblin/aws/CloudInitScriptBuilderTest.java |   96 --
 .../aws/GobblinAWSClusterLauncherTest.java      |  287 ----
 .../aws/AWSJobConfigurationManagerTest.java     |  154 +++
 .../gobblin/aws/CloudInitScriptBuilderTest.java |   96 ++
 .../aws/GobblinAWSClusterLauncherTest.java      |  287 ++++
 .../GobblinAWSClusterLauncherTest.conf          |    6 +-
 .../src/test/resources/masterCloudInit.sh       |    2 +-
 .../src/test/resources/workerCloudInit.sh       |    2 +-
 .../cluster/ClusterEventMetadataGenerator.java  |   58 -
 .../java/gobblin/cluster/ContainerMetrics.java  |   69 -
 .../GobblinClusterConfigurationKeys.java        |   83 --
 .../gobblin/cluster/GobblinClusterManager.java  |  762 -----------
 .../cluster/GobblinClusterMetricTagNames.java   |   42 -
 .../gobblin/cluster/GobblinClusterUtils.java    |   64 -
 .../gobblin/cluster/GobblinHelixConstants.java  |   29 -
 .../java/gobblin/cluster/GobblinHelixJob.java   |   92 --
 .../cluster/GobblinHelixJobLauncher.java        |  396 ------
 .../cluster/GobblinHelixJobScheduler.java       |  203 ---
 .../cluster/GobblinHelixMessagingService.java   |  203 ---
 .../java/gobblin/cluster/GobblinHelixTask.java  |  201 ---
 .../gobblin/cluster/GobblinHelixTaskDriver.java |  273 ----
 .../cluster/GobblinHelixTaskFactory.java        |   97 --
 .../cluster/GobblinHelixTaskStateTracker.java   |  105 --
 .../java/gobblin/cluster/GobblinTaskRunner.java |  493 -------
 .../gobblin/cluster/GobblinTaskStateModel.java  |   51 -
 .../cluster/GobblinTaskStateModelFactory.java   |   61 -
 .../gobblin/cluster/HelixMessageSubTypes.java   |   45 -
 .../main/java/gobblin/cluster/HelixUtils.java   |   71 -
 .../cluster/JobConfigurationManager.java        |  125 --
 .../java/gobblin/cluster/NoopReplyHandler.java  |   47 -
 .../ScheduledJobConfigurationManager.java       |  151 ---
 .../StreamingJobConfigurationManager.java       |  158 ---
 .../event/ClusterManagerShutdownRequest.java    |   32 -
 .../event/DeleteJobConfigArrivalEvent.java      |   61 -
 .../cluster/event/NewJobConfigArrivalEvent.java |   60 -
 .../event/UpdateJobConfigArrivalEvent.java      |   59 -
 .../cluster/ClusterEventMetadataGenerator.java  |   58 +
 .../gobblin/cluster/ContainerMetrics.java       |   69 +
 .../GobblinClusterConfigurationKeys.java        |   83 ++
 .../gobblin/cluster/GobblinClusterManager.java  |  762 +++++++++++
 .../cluster/GobblinClusterMetricTagNames.java   |   42 +
 .../gobblin/cluster/GobblinClusterUtils.java    |   64 +
 .../gobblin/cluster/GobblinHelixConstants.java  |   29 +
 .../apache/gobblin/cluster/GobblinHelixJob.java |   92 ++
 .../cluster/GobblinHelixJobLauncher.java        |  396 ++++++
 .../cluster/GobblinHelixJobScheduler.java       |  203 +++
 .../cluster/GobblinHelixMessagingService.java   |  203 +++
 .../gobblin/cluster/GobblinHelixTask.java       |  201 +++
 .../gobblin/cluster/GobblinHelixTaskDriver.java |  273 ++++
 .../cluster/GobblinHelixTaskFactory.java        |   97 ++
 .../cluster/GobblinHelixTaskStateTracker.java   |  105 ++
 .../gobblin/cluster/GobblinTaskRunner.java      |  493 +++++++
 .../gobblin/cluster/GobblinTaskStateModel.java  |   51 +
 .../cluster/GobblinTaskStateModelFactory.java   |   61 +
 .../gobblin/cluster/HelixMessageSubTypes.java   |   45 +
 .../org/apache/gobblin/cluster/HelixUtils.java  |   71 +
 .../cluster/JobConfigurationManager.java        |  125 ++
 .../gobblin/cluster/NoopReplyHandler.java       |   47 +
 .../ScheduledJobConfigurationManager.java       |  151 +++
 .../StreamingJobConfigurationManager.java       |  158 +++
 .../event/ClusterManagerShutdownRequest.java    |   32 +
 .../event/DeleteJobConfigArrivalEvent.java      |   61 +
 .../cluster/event/NewJobConfigArrivalEvent.java |   60 +
 .../event/UpdateJobConfigArrivalEvent.java      |   59 +
 .../ClusterEventMetadataGeneratorTest.java      |  104 --
 .../gobblin/cluster/GobblinClusterKillTest.java |  389 ------
 .../cluster/GobblinClusterManagerTest.java      |  184 ---
 .../cluster/GobblinHelixJobLauncherTest.java    |  210 ---
 .../gobblin/cluster/GobblinHelixTaskTest.java   |  193 ---
 .../gobblin/cluster/GobblinTaskRunnerTest.java  |  116 --
 .../gobblin/cluster/HelixMessageTestBase.java   |   36 -
 .../java/gobblin/cluster/HelixUtilsTest.java    |   90 --
 .../cluster/JobConfigurationManagerTest.java    |  124 --
 .../test/java/gobblin/cluster/TestHelper.java   |  110 --
 .../TestShutdownMessageHandlerFactory.java      |   90 --
 .../ClusterEventMetadataGeneratorTest.java      |  104 ++
 .../gobblin/cluster/GobblinClusterKillTest.java |  392 ++++++
 .../cluster/GobblinClusterManagerTest.java      |  184 +++
 .../cluster/GobblinHelixJobLauncherTest.java    |  210 +++
 .../gobblin/cluster/GobblinHelixTaskTest.java   |  193 +++
 .../gobblin/cluster/GobblinTaskRunnerTest.java  |  116 ++
 .../gobblin/cluster/HelixMessageTestBase.java   |   36 +
 .../apache/gobblin/cluster/HelixUtilsTest.java  |   90 ++
 .../cluster/JobConfigurationManagerTest.java    |  124 ++
 .../org/apache/gobblin/cluster/TestHelper.java  |  110 ++
 .../TestShutdownMessageHandlerFactory.java      |   90 ++
 .../resources/GobblinClusterKillTestJob.conf    |    8 +-
 .../resources/GobblinHelixJobLauncherTest.conf  |    6 +-
 .../src/test/resources/reference.conf           |    2 +-
 .../java/gobblin/compaction/CliOptions.java     |  110 --
 .../main/java/gobblin/compaction/Compactor.java |   39 -
 .../compaction/CompactorCreationException.java  |   30 -
 .../gobblin/compaction/CompactorFactory.java    |   49 -
 .../compaction/ReflectionCompactorFactory.java  |   55 -
 .../action/CompactionCompleteAction.java        |   35 -
 .../CompactionCompleteFileOperationAction.java  |  148 ---
 .../CompactionHiveRegistrationAction.java       |   83 --
 .../action/CompactionMarkDirectoryAction.java   |   82 --
 .../compaction/audit/AuditCountClient.java      |   28 -
 .../audit/AuditCountClientFactory.java          |   28 -
 .../audit/KafkaAuditCountHttpClient.java        |  176 ---
 .../audit/KafkaAuditCountHttpClientFactory.java |   32 -
 .../audit/PinotAuditCountHttpClient.java        |  162 ---
 .../audit/PinotAuditCountHttpClientFactory.java |   32 -
 .../RecompactionCombineCondition.java           |  131 --
 .../conditions/RecompactionCondition.java       |   34 -
 .../RecompactionConditionBasedOnDuration.java   |   89 --
 .../RecompactionConditionBasedOnFileCount.java  |   59 -
 .../RecompactionConditionBasedOnRatio.java      |  109 --
 .../RecompactionConditionFactory.java           |   24 -
 .../gobblin/compaction/dataset/Dataset.java     |  461 -------
 .../compaction/dataset/DatasetHelper.java       |  199 ---
 .../compaction/dataset/DatasetsFinder.java      |  168 ---
 .../dataset/SimpleDatasetsFinder.java           |   60 -
 .../dataset/TimeBasedSubDirDatasetsFinder.java  |  258 ----
 .../event/CompactionSlaEventHelper.java         |  158 ---
 .../gobblin/compaction/event/DedupeStatus.java  |   23 -
 .../compaction/hive/AvroExternalTable.java      |  238 ----
 .../compaction/hive/CompactionRunner.java       |  140 --
 .../java/gobblin/compaction/hive/HdfsIO.java    |   90 --
 .../gobblin/compaction/hive/HdfsReader.java     |   64 -
 .../gobblin/compaction/hive/HdfsWriter.java     |   68 -
 .../gobblin/compaction/hive/HiveAttribute.java  |  118 --
 .../compaction/hive/HiveManagedTable.java       |   96 --
 .../java/gobblin/compaction/hive/HiveTable.java |  117 --
 .../compaction/hive/SerialCompactor.java        |  353 -----
 .../HiveRegistrationCompactorListener.java      |   48 -
 .../CompactionLauncherWriter.java               |   84 --
 .../CompactionLauncherWriterBuilder.java        |   34 -
 .../HiveMetadataForCompactionExtractor.java     |   88 --
 ...veMetadataForCompactionExtractorFactory.java |   37 -
 .../hivebasedconstructs/MRCompactionEntity.java |   47 -
 .../listeners/CompactorCompletionListener.java  |   26 -
 .../CompactorCompletionListenerFactory.java     |   23 -
 .../compaction/listeners/CompactorListener.java |   37 -
 .../CompactorListenerCreationException.java     |   30 -
 .../listeners/CompactorListenerFactory.java     |   44 -
 .../ReflectionCompactorListenerFactory.java     |   65 -
 .../listeners/SerialCompactorListener.java      |   38 -
 .../SimpleCompactorCompletionListener.java      |   53 -
 .../CompactionAvroJobConfigurator.java          |  335 -----
 .../mapreduce/MRCompactionRunner.java           |  112 --
 .../compaction/mapreduce/MRCompactionTask.java  |  115 --
 .../mapreduce/MRCompactionTaskFactory.java      |   23 -
 .../compaction/mapreduce/MRCompactor.java       | 1001 --------------
 .../mapreduce/MRCompactorJobPropCreator.java    |  344 -----
 .../mapreduce/MRCompactorJobRunner.java         |  659 ----------
 .../avro/AvroDeltaFieldNameProvider.java        |   33 -
 .../avro/AvroKeyCombineFileRecordReader.java    |   80 --
 .../avro/AvroKeyCompactorOutputCommitter.java   |  101 --
 .../avro/AvroKeyCompactorOutputFormat.java      |   46 -
 .../mapreduce/avro/AvroKeyDedupReducer.java     |  125 --
 .../mapreduce/avro/AvroKeyMapper.java           |  124 --
 .../AvroKeyRecursiveCombineFileInputFormat.java |  124 --
 .../avro/ConfBasedDeltaFieldProvider.java       |   55 -
 .../FieldAttributeBasedDeltaFieldsProvider.java |  107 --
 .../avro/MRCompactorAvroKeyDedupJobRunner.java  |  339 -----
 .../compaction/parser/CompactionPathParser.java |  144 ---
 .../compaction/source/CompactionFailedTask.java |   59 -
 .../compaction/source/CompactionSource.java     |  472 -------
 .../compaction/suite/CompactionAvroSuite.java   |  134 --
 .../suite/CompactionAvroSuiteFactory.java       |   31 -
 .../compaction/suite/CompactionSuite.java       |   80 --
 .../suite/CompactionSuiteFactory.java           |   28 -
 .../compaction/suite/CompactionSuiteUtils.java  |   44 -
 .../verify/CompactionAuditCountVerifier.java    |  183 ---
 .../verify/CompactionThresholdVerifier.java     |  102 --
 .../verify/CompactionTimeRangeVerifier.java     |   82 --
 .../compaction/verify/CompactionVerifier.java   |   16 -
 .../verify/DataCompletenessVerifier.java        |  208 ---
 .../verify/InputRecordCountHelper.java          |  108 --
 .../apache/gobblin/compaction/CliOptions.java   |  110 ++
 .../apache/gobblin/compaction/Compactor.java    |   39 +
 .../compaction/CompactorCreationException.java  |   30 +
 .../gobblin/compaction/CompactorFactory.java    |   49 +
 .../compaction/ReflectionCompactorFactory.java  |   55 +
 .../action/CompactionCompleteAction.java        |   35 +
 .../CompactionCompleteFileOperationAction.java  |  148 +++
 .../CompactionHiveRegistrationAction.java       |   83 ++
 .../action/CompactionMarkDirectoryAction.java   |   82 ++
 .../compaction/audit/AuditCountClient.java      |   28 +
 .../audit/AuditCountClientFactory.java          |   28 +
 .../audit/KafkaAuditCountHttpClient.java        |  176 +++
 .../audit/KafkaAuditCountHttpClientFactory.java |   32 +
 .../audit/PinotAuditCountHttpClient.java        |  162 +++
 .../audit/PinotAuditCountHttpClientFactory.java |   32 +
 .../RecompactionCombineCondition.java           |  131 ++
 .../conditions/RecompactionCondition.java       |   34 +
 .../RecompactionConditionBasedOnDuration.java   |   89 ++
 .../RecompactionConditionBasedOnFileCount.java  |   59 +
 .../RecompactionConditionBasedOnRatio.java      |  109 ++
 .../RecompactionConditionFactory.java           |   24 +
 .../gobblin/compaction/dataset/Dataset.java     |  461 +++++++
 .../compaction/dataset/DatasetHelper.java       |  199 +++
 .../compaction/dataset/DatasetsFinder.java      |  168 +++
 .../dataset/SimpleDatasetsFinder.java           |   60 +
 .../dataset/TimeBasedSubDirDatasetsFinder.java  |  258 ++++
 .../event/CompactionSlaEventHelper.java         |  158 +++
 .../gobblin/compaction/event/DedupeStatus.java  |   23 +
 .../compaction/hive/AvroExternalTable.java      |  238 ++++
 .../compaction/hive/CompactionRunner.java       |  140 ++
 .../apache/gobblin/compaction/hive/HdfsIO.java  |   90 ++
 .../gobblin/compaction/hive/HdfsReader.java     |   64 +
 .../gobblin/compaction/hive/HdfsWriter.java     |   68 +
 .../gobblin/compaction/hive/HiveAttribute.java  |  118 ++
 .../compaction/hive/HiveManagedTable.java       |   96 ++
 .../gobblin/compaction/hive/HiveTable.java      |  117 ++
 .../compaction/hive/SerialCompactor.java        |  353 +++++
 .../HiveRegistrationCompactorListener.java      |   48 +
 .../CompactionLauncherWriter.java               |   84 ++
 .../CompactionLauncherWriterBuilder.java        |   34 +
 .../HiveMetadataForCompactionExtractor.java     |   88 ++
 ...veMetadataForCompactionExtractorFactory.java |   37 +
 .../hivebasedconstructs/MRCompactionEntity.java |   47 +
 .../listeners/CompactorCompletionListener.java  |   26 +
 .../CompactorCompletionListenerFactory.java     |   23 +
 .../compaction/listeners/CompactorListener.java |   37 +
 .../CompactorListenerCreationException.java     |   30 +
 .../listeners/CompactorListenerFactory.java     |   44 +
 .../ReflectionCompactorListenerFactory.java     |   65 +
 .../listeners/SerialCompactorListener.java      |   38 +
 .../SimpleCompactorCompletionListener.java      |   53 +
 .../CompactionAvroJobConfigurator.java          |  335 +++++
 .../mapreduce/MRCompactionRunner.java           |  112 ++
 .../compaction/mapreduce/MRCompactionTask.java  |  115 ++
 .../mapreduce/MRCompactionTaskFactory.java      |   23 +
 .../compaction/mapreduce/MRCompactor.java       | 1001 ++++++++++++++
 .../mapreduce/MRCompactorJobPropCreator.java    |  344 +++++
 .../mapreduce/MRCompactorJobRunner.java         |  659 ++++++++++
 .../avro/AvroDeltaFieldNameProvider.java        |   33 +
 .../avro/AvroKeyCombineFileRecordReader.java    |   80 ++
 .../avro/AvroKeyCompactorOutputCommitter.java   |  101 ++
 .../avro/AvroKeyCompactorOutputFormat.java      |   46 +
 .../mapreduce/avro/AvroKeyDedupReducer.java     |  125 ++
 .../mapreduce/avro/AvroKeyMapper.java           |  124 ++
 .../AvroKeyRecursiveCombineFileInputFormat.java |  124 ++
 .../avro/ConfBasedDeltaFieldProvider.java       |   55 +
 .../FieldAttributeBasedDeltaFieldsProvider.java |  107 ++
 .../avro/MRCompactorAvroKeyDedupJobRunner.java  |  339 +++++
 .../compaction/parser/CompactionPathParser.java |  144 +++
 .../compaction/source/CompactionFailedTask.java |   59 +
 .../compaction/source/CompactionSource.java     |  472 +++++++
 .../compaction/suite/CompactionAvroSuite.java   |  134 ++
 .../suite/CompactionAvroSuiteFactory.java       |   31 +
 .../compaction/suite/CompactionSuite.java       |   80 ++
 .../suite/CompactionSuiteFactory.java           |   28 +
 .../compaction/suite/CompactionSuiteUtils.java  |   44 +
 .../verify/CompactionAuditCountVerifier.java    |  183 +++
 .../verify/CompactionThresholdVerifier.java     |  102 ++
 .../verify/CompactionTimeRangeVerifier.java     |   82 ++
 .../compaction/verify/CompactionVerifier.java   |   16 +
 .../verify/DataCompletenessVerifier.java        |  208 +++
 .../verify/InputRecordCountHelper.java          |  108 ++
 .../mapreduce/MRCompactionTaskTest.java         |  300 -----
 ...obRunnerFilenameRecordCountProviderTest.java |   83 --
 .../mapreduce/RenameSourceDirectoryTest.java    |  142 --
 .../mapreduce/avro/AvroKeyDedupReducerTest.java |  176 ---
 .../avro/ConfBasedDeltaFieldProviderTest.java   |   56 -
 ...ldAttributeBasedDeltaFieldsProviderTest.java |   65 -
 .../MRCompactorAvroKeyDedupJobRunnerTest.java   |   76 --
 .../conditions/RecompactionConditionTest.java   |  186 ---
 .../suite/TestCompactionSuiteFactories.java     |   19 -
 .../compaction/suite/TestCompactionSuites.java  |   50 -
 .../verify/PinotAuditCountVerifierTest.java     |  102 --
 .../mapreduce/MRCompactionTaskTest.java         |  300 +++++
 ...obRunnerFilenameRecordCountProviderTest.java |   83 ++
 .../mapreduce/RenameSourceDirectoryTest.java    |  142 ++
 .../mapreduce/avro/AvroKeyDedupReducerTest.java |  176 +++
 .../avro/ConfBasedDeltaFieldProviderTest.java   |   56 +
 ...ldAttributeBasedDeltaFieldsProviderTest.java |   65 +
 .../MRCompactorAvroKeyDedupJobRunnerTest.java   |   76 ++
 .../conditions/RecompactionConditionTest.java   |  186 +++
 .../suite/TestCompactionSuiteFactories.java     |   19 +
 .../compaction/suite/TestCompactionSuites.java  |   50 +
 .../verify/PinotAuditCountVerifierTest.java     |  102 ++
 .../dedup-schema/dedup-schema-with-pkey.avsc    |    2 +-
 .../dedup-schema/dedup-schema-without-pkey.avsc |    2 +-
 .../gobblin/config/client/ConfigClient.java     |  403 ------
 .../gobblin/config/client/ConfigClientCLI.java  |   70 -
 .../config/client/ConfigClientCache.java        |   47 -
 .../config/client/ConfigClientUtils.java        |  209 ---
 .../client/ConfigStoreFactoryRegister.java      |   55 -
 .../gobblin/config/client/package-info.java     |   25 -
 .../gobblin/config/client/ConfigClient.java     |  403 ++++++
 .../gobblin/config/client/ConfigClientCLI.java  |   70 +
 .../config/client/ConfigClientCache.java        |   47 +
 .../config/client/ConfigClientUtils.java        |  209 +++
 .../client/ConfigStoreFactoryRegister.java      |   55 +
 .../gobblin/config/client/package-info.java     |   25 +
 .../gobblin/config/client/TestConfigClient.java |  266 ----
 .../config/client/TestConfigClientUtils.java    |  116 --
 .../gobblin/config/client/TestConfigClient.java |  266 ++++
 .../config/client/TestConfigClientUtils.java    |  116 ++
 ...onfigStoreFactoryDoesNotExistsException.java |   47 -
 .../client/api/VersionStabilityPolicy.java      |   66 -
 .../impl/CircularDependencyException.java       |   33 -
 .../common/impl/ConfigStoreBackedTopology.java  |  143 --
 .../impl/ConfigStoreBackedValueInspector.java   |  188 ---
 .../impl/ConfigStoreTopologyInspector.java      |   97 --
 .../common/impl/ConfigStoreValueInspector.java  |   74 --
 .../config/common/impl/ImportTraverser.java     |  161 ---
 .../config/common/impl/InMemoryTopology.java    |  224 ----
 .../common/impl/InMemoryValueInspector.java     |  179 ---
 .../impl/SingleLinkedListConfigKeyPath.java     |  113 --
 .../gobblin/config/store/api/ConfigKeyPath.java |   61 -
 .../gobblin/config/store/api/ConfigStore.java   |  110 --
 .../store/api/ConfigStoreCreationException.java |   45 -
 .../config/store/api/ConfigStoreFactory.java    |   59 -
 .../store/api/ConfigStoreWithBatchFetches.java  |   58 -
 .../store/api/ConfigStoreWithImportedBy.java    |   51 -
 .../ConfigStoreWithImportedByRecursively.java   |   50 -
 .../store/api/ConfigStoreWithResolution.java    |   73 --
 .../api/ConfigStoreWithStableVersioning.java    |   52 -
 .../store/api/VersionDoesNotExistException.java |   55 -
 .../gobblin/config/store/api/package-info.java  |   24 -
 .../store/deploy/ClasspathConfigSource.java     |  109 --
 .../config/store/deploy/ConfigStream.java       |   34 -
 .../gobblin/config/store/deploy/Deployable.java |   39 -
 .../store/deploy/DeployableConfigSource.java    |   38 -
 .../config/store/deploy/DeploymentConfig.java   |   38 -
 .../config/store/deploy/FsDeploymentConfig.java |   69 -
 .../config/store/deploy/StoreDeployer.java      |   71 -
 .../DefaultCapableLocalConfigStoreFactory.java  |   62 -
 .../hdfs/SimpleHDFSConfigStoreFactory.java      |   87 --
 .../store/hdfs/SimpleHDFSStoreMetadata.java     |  142 --
 .../hdfs/SimpleHadoopFilesystemConfigStore.java |  546 --------
 ...impleHadoopFilesystemConfigStoreFactory.java |  278 ----
 .../hdfs/SimpleLocalHDFSConfigStoreFactory.java |   71 -
 ...onfigStoreFactoryDoesNotExistsException.java |   47 +
 .../client/api/VersionStabilityPolicy.java      |   66 +
 .../impl/CircularDependencyException.java       |   33 +
 .../common/impl/ConfigStoreBackedTopology.java  |  143 ++
 .../impl/ConfigStoreBackedValueInspector.java   |  188 +++
 .../impl/ConfigStoreTopologyInspector.java      |   97 ++
 .../common/impl/ConfigStoreValueInspector.java  |   74 ++
 .../config/common/impl/ImportTraverser.java     |  161 +++
 .../config/common/impl/InMemoryTopology.java    |  224 ++++
 .../common/impl/InMemoryValueInspector.java     |  179 +++
 .../impl/SingleLinkedListConfigKeyPath.java     |  113 ++
 .../gobblin/config/store/api/ConfigKeyPath.java |   61 +
 .../gobblin/config/store/api/ConfigStore.java   |  110 ++
 .../store/api/ConfigStoreCreationException.java |   45 +
 .../config/store/api/ConfigStoreFactory.java    |   59 +
 .../store/api/ConfigStoreWithBatchFetches.java  |   58 +
 .../store/api/ConfigStoreWithImportedBy.java    |   51 +
 .../ConfigStoreWithImportedByRecursively.java   |   50 +
 .../store/api/ConfigStoreWithResolution.java    |   73 ++
 .../api/ConfigStoreWithStableVersioning.java    |   52 +
 .../store/api/VersionDoesNotExistException.java |   55 +
 .../gobblin/config/store/api/package-info.java  |   24 +
 .../store/deploy/ClasspathConfigSource.java     |  109 ++
 .../config/store/deploy/ConfigStream.java       |   34 +
 .../gobblin/config/store/deploy/Deployable.java |   39 +
 .../store/deploy/DeployableConfigSource.java    |   38 +
 .../config/store/deploy/DeploymentConfig.java   |   38 +
 .../config/store/deploy/FsDeploymentConfig.java |   69 +
 .../config/store/deploy/StoreDeployer.java      |   71 +
 .../DefaultCapableLocalConfigStoreFactory.java  |   62 +
 .../hdfs/SimpleHDFSConfigStoreFactory.java      |   87 ++
 .../store/hdfs/SimpleHDFSStoreMetadata.java     |  142 ++
 .../hdfs/SimpleHadoopFilesystemConfigStore.java |  546 ++++++++
 ...impleHadoopFilesystemConfigStoreFactory.java |  278 ++++
 .../hdfs/SimpleLocalHDFSConfigStoreFactory.java |   71 +
 .../gobblin.config.store.api.ConfigStoreFactory |   20 -
 ....gobblin.config.store.api.ConfigStoreFactory |   20 +
 .../java/gobblin/config/TestEnvironment.java    |   35 -
 .../config/common/impl/ImportTraverserTest.java |  102 --
 .../common/impl/TestCircularDependency.java     |  233 ----
 .../impl/TestConfigStoreValueInspector.java     |   86 --
 .../common/impl/TestInMemoryTopology.java       |  290 -----
 .../impl/TestSingleLinkedListConfigKeyPath.java |   66 -
 .../store/hdfs/SimpleHDFSStoreMetadataTest.java |   98 --
 .../store/hdfs/SimpleHdfsConfigStoreTest.java   |  271 ----
 .../SimpleHdfsConfigureStoreFactoryTest.java    |  120 --
 .../apache/gobblin/config/TestEnvironment.java  |   35 +
 .../config/common/impl/ImportTraverserTest.java |  102 ++
 .../common/impl/TestCircularDependency.java     |  233 ++++
 .../impl/TestConfigStoreValueInspector.java     |   86 ++
 .../common/impl/TestInMemoryTopology.java       |  290 +++++
 .../impl/TestSingleLinkedListConfigKeyPath.java |   66 +
 .../store/hdfs/SimpleHDFSStoreMetadataTest.java |   98 ++
 .../store/hdfs/SimpleHdfsConfigStoreTest.java   |  271 ++++
 .../SimpleHdfsConfigureStoreFactoryTest.java    |  120 ++
 .../FineGrainedWatermarkTrackerBenchmark.java   |  140 --
 .../FineGrainedWatermarkTrackerBenchmark.java   |  140 ++
 .../src/main/java/gobblin/async/Callback.java   |    8 -
 .../compression/CompressionConfigParser.java    |   64 -
 .../gobblin/compression/CompressionFactory.java |   45 -
 .../gobblin/converter/AsyncConverter1to1.java   |  103 --
 .../converter/AvroToAvroConverterBase.java      |   38 -
 .../gobblin/converter/IdentityConverter.java    |   37 -
 .../gobblin/converter/SamplingConverter.java    |   79 --
 .../gobblin/converter/SingleRecordIterable.java |   46 -
 .../gobblin/converter/ToAvroConverterBase.java  |   41 -
 .../filter/AvroProjectionConverter.java         |   91 --
 .../filter/AvroSchemaFieldRemover.java          |  154 ---
 ...blinTrackingEventFlattenFilterConverter.java |  143 --
 .../gobblin/crypto/CredentialStoreFactory.java  |   60 -
 .../gobblin/crypto/EncryptionConfigParser.java  |  242 ----
 .../java/gobblin/crypto/EncryptionFactory.java  |   74 --
 .../instrumented/GobblinMetricsKeys.java        |   46 -
 .../gobblin/instrumented/Instrumentable.java    |   93 --
 .../java/gobblin/instrumented/Instrumented.java |  322 -----
 .../converter/InstrumentedConverter.java        |   40 -
 .../converter/InstrumentedConverterBase.java    |  197 ---
 .../InstrumentedConverterDecorator.java         |  111 --
 .../extractor/InstrumentedExtractor.java        |   51 -
 .../extractor/InstrumentedExtractorBase.java    |  270 ----
 .../InstrumentedExtractorDecorator.java         |  115 --
 .../fork/InstrumentedForkOperator.java          |   37 -
 .../fork/InstrumentedForkOperatorBase.java      |  176 ---
 .../fork/InstrumentedForkOperatorDecorator.java |   86 --
 .../InstrumentedRowLevelPolicy.java             |   40 -
 .../InstrumentedRowLevelPolicyBase.java         |  173 ---
 .../InstrumentedRowLevelPolicyDecorator.java    |   79 --
 .../writer/InstrumentedDataWriter.java          |   40 -
 .../writer/InstrumentedDataWriterBase.java      |  297 -----
 .../writer/InstrumentedDataWriterDecorator.java |  160 ---
 ...trumentedPartitionedDataWriterDecorator.java |   54 -
 .../DefaultCheckpointableWatermark.java         |   79 --
 .../source/extractor/JobCommitPolicy.java       |  106 --
 .../source/extractor/WorkUnitRetryPolicy.java   |   91 --
 .../extractor/extract/AbstractSource.java       |  142 --
 .../extractor/extract/EventBasedExtractor.java  |   33 -
 .../extractor/extract/EventBasedSource.java     |   31 -
 .../source/extractor/extract/LongWatermark.java |   80 --
 .../limiter/LimiterConfigurationKeys.java       |   11 -
 .../java/gobblin/test/AnyToJsonConverter.java   |   58 -
 .../java/gobblin/test/AnyToStringConverter.java |   45 -
 .../java/gobblin/test/SequentialTestSource.java |  311 -----
 .../src/main/java/gobblin/test/TestRecord.java  |   37 -
 .../writer/AcknowledgableRecordEnvelope.java    |   68 -
 .../gobblin/writer/AcknowledgableWatermark.java |   85 --
 .../java/gobblin/writer/AsyncDataWriter.java    |   45 -
 .../java/gobblin/writer/AsyncWriterManager.java |  567 --------
 .../src/main/java/gobblin/writer/Batch.java     |  187 ---
 .../java/gobblin/writer/BatchAccumulator.java   |  115 --
 .../gobblin/writer/BatchAsyncDataWriter.java    |   46 -
 .../gobblin/writer/BufferedAsyncDataWriter.java |  203 ---
 .../java/gobblin/writer/BytesBoundedBatch.java  |  101 --
 .../writer/FineGrainedWatermarkTracker.java     |  344 -----
 .../writer/FutureWrappedWriteCallback.java      |  118 --
 .../gobblin/writer/GenericWriteResponse.java    |   50 -
 .../writer/GenericWriteResponseWrapper.java     |   28 -
 .../gobblin/writer/LastWatermarkTracker.java    |   97 --
 .../writer/MultiWriterWatermarkManager.java     |  162 ---
 .../writer/MultiWriterWatermarkTracker.java     |  153 ---
 .../writer/PartitionAwareDataWriterBuilder.java |   78 --
 .../main/java/gobblin/writer/RecordFuture.java  |   73 --
 .../java/gobblin/writer/RecordMetadata.java     |   38 -
 .../writer/SequentialBasedBatchAccumulator.java |  261 ----
 .../java/gobblin/writer/SyncDataWriter.java     |   44 -
 .../writer/TrackerBasedWatermarkManager.java    |  145 ---
 .../gobblin/writer/WatermarkAwareWriter.java    |   65 -
 .../writer/WatermarkAwareWriterWrapper.java     |   57 -
 .../java/gobblin/writer/WatermarkManager.java   |   96 --
 .../java/gobblin/writer/WatermarkTracker.java   |   46 -
 .../gobblin/writer/WatermarkTrackerFactory.java |   92 --
 .../main/java/gobblin/writer/WriteCallback.java |   32 -
 .../main/java/gobblin/writer/WriteResponse.java |   62 -
 .../gobblin/writer/WriteResponseFuture.java     |   67 -
 .../gobblin/writer/WriteResponseMapper.java     |   28 -
 .../writer/exception/NonTransientException.java |   40 -
 .../writer/partitioner/WriterPartitioner.java   |   45 -
 .../java/org/apache/gobblin/async/Callback.java |    8 +
 .../compression/CompressionConfigParser.java    |   64 +
 .../gobblin/compression/CompressionFactory.java |   45 +
 .../gobblin/converter/AsyncConverter1to1.java   |  103 ++
 .../converter/AvroToAvroConverterBase.java      |   38 +
 .../gobblin/converter/IdentityConverter.java    |   37 +
 .../gobblin/converter/SamplingConverter.java    |   79 ++
 .../gobblin/converter/SingleRecordIterable.java |   46 +
 .../gobblin/converter/ToAvroConverterBase.java  |   41 +
 .../filter/AvroProjectionConverter.java         |   91 ++
 .../filter/AvroSchemaFieldRemover.java          |  154 +++
 ...blinTrackingEventFlattenFilterConverter.java |  143 ++
 .../gobblin/crypto/CredentialStoreFactory.java  |   60 +
 .../gobblin/crypto/EncryptionConfigParser.java  |  242 ++++
 .../gobblin/crypto/EncryptionFactory.java       |   74 ++
 .../instrumented/GobblinMetricsKeys.java        |   46 +
 .../gobblin/instrumented/Instrumentable.java    |   93 ++
 .../gobblin/instrumented/Instrumented.java      |  322 +++++
 .../converter/InstrumentedConverter.java        |   40 +
 .../converter/InstrumentedConverterBase.java    |  197 +++
 .../InstrumentedConverterDecorator.java         |  111 ++
 .../extractor/InstrumentedExtractor.java        |   51 +
 .../extractor/InstrumentedExtractorBase.java    |  270 ++++
 .../InstrumentedExtractorDecorator.java         |  115 ++
 .../fork/InstrumentedForkOperator.java          |   37 +
 .../fork/InstrumentedForkOperatorBase.java      |  176 +++
 .../fork/InstrumentedForkOperatorDecorator.java |   86 ++
 .../InstrumentedRowLevelPolicy.java             |   40 +
 .../InstrumentedRowLevelPolicyBase.java         |  173 +++
 .../InstrumentedRowLevelPolicyDecorator.java    |   79 ++
 .../writer/InstrumentedDataWriter.java          |   40 +
 .../writer/InstrumentedDataWriterBase.java      |  297 +++++
 .../writer/InstrumentedDataWriterDecorator.java |  160 +++
 ...trumentedPartitionedDataWriterDecorator.java |   54 +
 .../DefaultCheckpointableWatermark.java         |   79 ++
 .../source/extractor/JobCommitPolicy.java       |  106 ++
 .../source/extractor/WorkUnitRetryPolicy.java   |   91 ++
 .../extractor/extract/AbstractSource.java       |  142 ++
 .../extractor/extract/EventBasedExtractor.java  |   33 +
 .../extractor/extract/EventBasedSource.java     |   31 +
 .../source/extractor/extract/LongWatermark.java |   80 ++
 .../limiter/LimiterConfigurationKeys.java       |   11 +
 .../apache/gobblin/test/AnyToJsonConverter.java |   58 +
 .../gobblin/test/AnyToStringConverter.java      |   45 +
 .../gobblin/test/SequentialTestSource.java      |  311 +++++
 .../org/apache/gobblin/test/TestRecord.java     |   37 +
 .../writer/AcknowledgableRecordEnvelope.java    |   68 +
 .../gobblin/writer/AcknowledgableWatermark.java |   85 ++
 .../apache/gobblin/writer/AsyncDataWriter.java  |   45 +
 .../gobblin/writer/AsyncWriterManager.java      |  567 ++++++++
 .../java/org/apache/gobblin/writer/Batch.java   |  187 +++
 .../apache/gobblin/writer/BatchAccumulator.java |  115 ++
 .../gobblin/writer/BatchAsyncDataWriter.java    |   46 +
 .../gobblin/writer/BufferedAsyncDataWriter.java |  203 +++
 .../gobblin/writer/BytesBoundedBatch.java       |  101 ++
 .../writer/FineGrainedWatermarkTracker.java     |  344 +++++
 .../writer/FutureWrappedWriteCallback.java      |  118 ++
 .../gobblin/writer/GenericWriteResponse.java    |   50 +
 .../writer/GenericWriteResponseWrapper.java     |   28 +
 .../gobblin/writer/LastWatermarkTracker.java    |   97 ++
 .../writer/MultiWriterWatermarkManager.java     |  162 +++
 .../writer/MultiWriterWatermarkTracker.java     |  153 +++
 .../writer/PartitionAwareDataWriterBuilder.java |   78 ++
 .../org/apache/gobblin/writer/RecordFuture.java |   73 ++
 .../apache/gobblin/writer/RecordMetadata.java   |   38 +
 .../writer/SequentialBasedBatchAccumulator.java |  261 ++++
 .../apache/gobblin/writer/SyncDataWriter.java   |   44 +
 .../writer/TrackerBasedWatermarkManager.java    |  145 +++
 .../gobblin/writer/WatermarkAwareWriter.java    |   65 +
 .../writer/WatermarkAwareWriterWrapper.java     |   57 +
 .../apache/gobblin/writer/WatermarkManager.java |   96 ++
 .../apache/gobblin/writer/WatermarkTracker.java |   46 +
 .../gobblin/writer/WatermarkTrackerFactory.java |   92 ++
 .../apache/gobblin/writer/WriteCallback.java    |   32 +
 .../apache/gobblin/writer/WriteResponse.java    |   62 +
 .../gobblin/writer/WriteResponseFuture.java     |   67 +
 .../gobblin/writer/WriteResponseMapper.java     |   28 +
 .../writer/exception/NonTransientException.java |   40 +
 .../writer/partitioner/WriterPartitioner.java   |   45 +
 .../converter/AsyncConverter1to1Test.java       |  155 ---
 .../converter/SamplingConverterTest.java        |   74 --
 .../converter/SingleRecordIterableTest.java     |   43 -
 .../filter/AvroSchemaFieldRemoverTest.java      |   68 -
 ...TrackingEventFlattenFilterConverterTest.java |   71 -
 .../crypto/EncryptionConfigParserTest.java      |  161 ---
 .../converter/InstrumentedConverterTest.java    |   97 --
 .../extractor/InstrumentedExtractorTest.java    |  135 --
 .../fork/InstrumentedForkOperatorTest.java      |  128 --
 .../InstrumentedRowLevelPolicyTest.java         |   90 --
 .../writer/InstrumentedDataWriterTest.java      |  149 ---
 .../extractor/extract/AbstractSourceTest.java   |  218 ----
 .../gobblin/writer/AsyncWriterManagerTest.java  |  312 -----
 .../writer/FineGrainedWatermarkTrackerTest.java |  230 ----
 .../writer/MultiWriterWatermarkManagerTest.java |  352 -----
 .../gobblin/writer/WatermarkTrackerTest.java    |   67 -
 .../converter/AsyncConverter1to1Test.java       |  155 +++
 .../converter/SamplingConverterTest.java        |   74 ++
 .../converter/SingleRecordIterableTest.java     |   43 +
 .../filter/AvroSchemaFieldRemoverTest.java      |   68 +
 ...TrackingEventFlattenFilterConverterTest.java |   71 +
 .../crypto/EncryptionConfigParserTest.java      |  161 +++
 .../converter/InstrumentedConverterTest.java    |   97 ++
 .../extractor/InstrumentedExtractorTest.java    |  135 ++
 .../fork/InstrumentedForkOperatorTest.java      |  128 ++
 .../InstrumentedRowLevelPolicyTest.java         |   90 ++
 .../writer/InstrumentedDataWriterTest.java      |  149 +++
 .../extractor/extract/AbstractSourceTest.java   |  218 ++++
 .../gobblin/writer/AsyncWriterManagerTest.java  |  312 +++++
 .../writer/FineGrainedWatermarkTrackerTest.java |  230 ++++
 .../writer/MultiWriterWatermarkManagerTest.java |  352 +++++
 .../gobblin/writer/WatermarkTrackerTest.java    |   67 +
 .../java/gobblin/async/AsyncDataDispatcher.java |  175 ---
 .../main/java/gobblin/async/AsyncRequest.java   |   74 --
 .../java/gobblin/async/AsyncRequestBuilder.java |   31 -
 .../main/java/gobblin/async/BufferedRecord.java |   34 -
 .../java/gobblin/async/DispatchException.java   |   46 -
 .../java/gobblin/commit/FsRenameCommitStep.java |  150 ---
 .../SpeculativeAttemptAwareConstruct.java       |   37 -
 .../main/java/gobblin/config/ConfigBuilder.java |  104 --
 .../java/gobblin/converter/EmptyIterable.java   |   26 -
 .../GobblinMetricsPinotFlattenerConverter.java  |   87 --
 .../gobblin/converter/StringSchemaInjector.java |   54 -
 .../avro/AvroFieldRetrieverConverter.java       |   76 --
 .../avro/AvroRecordToAvroWritableConverter.java |   59 -
 .../avro/AvroToAvroCopyableConverter.java       |   59 -
 .../converter/avro/AvroToBytesConverter.java    |   73 --
 .../converter/avro/BytesToAvroConverter.java    |   73 --
 .../avro/FlattenNestedKeyConverter.java         |  145 ---
 .../avro/JsonElementConversionFactory.java      |  536 --------
 .../avro/JsonIntermediateToAvroConverter.java   |  166 ---
 .../avro/UnsupportedDateTypeException.java      |   27 -
 .../converter/csv/CsvToJsonConverter.java       |   95 --
 .../converter/csv/CsvToJsonConverterV2.java     |  210 ---
 .../filter/AvroFieldsPickConverter.java         |  212 ---
 .../converter/filter/AvroFilterConverter.java   |   92 --
 .../http/AvroToRestJsonEntryConverter.java      |  106 --
 .../java/gobblin/converter/http/RestEntry.java  |   35 -
 .../ConverterInitializerFactory.java            |   81 --
 .../initializer/MultiConverterInitializer.java  |   44 -
 .../JsonStringToJsonIntermediateConverter.java  |  115 --
 .../converter/json/JsonToStringConverter.java   |   49 -
 .../objectstore/ObjectStoreConverter.java       |   48 -
 .../objectstore/ObjectStoreDeleteConverter.java |   86 --
 .../converter/serde/HiveSerDeConverter.java     |  129 --
 .../string/ObjectToStringConverter.java         |   42 -
 .../converter/string/StringFilterConverter.java |   72 --
 .../string/StringSplitterConverter.java         |   63 -
 .../string/StringSplitterToListConverter.java   |   70 -
 .../string/StringToBytesConverter.java          |   45 -
 .../converter/string/TextToStringConverter.java |   54 -
 .../gobblin/fork/CopyableGenericRecord.java     |   48 -
 .../main/java/gobblin/fork/CopyableSchema.java  |   41 -
 .../java/gobblin/fork/IdentityForkOperator.java |   77 --
 .../http/DefaultHttpClientConfigurator.java     |  148 ---
 .../gobblin/http/HttpClientConfigurator.java    |   53 -
 .../http/HttpClientConfiguratorLoader.java      |   79 --
 .../gobblin/initializer/MultiInitializer.java   |   60 -
 .../src/main/java/gobblin/net/Request.java      |   28 -
 .../policies/avro/AvroHeaderGuidPolicy.java     |   47 -
 .../avro/AvroHeaderTimestampPolicy.java         |   55 -
 .../AvroRecordTimestampLowerBoundPolicy.java    |   41 -
 .../gobblin/policies/count/RowCountPolicy.java  |   52 -
 .../policies/count/RowCountRangePolicy.java     |   54 -
 .../schema/SchemaCompatibilityPolicy.java       |   54 -
 .../policies/schema/SchemaRowCheckPolicy.java   |   34 -
 .../time/RecordTimestampLowerBoundPolicy.java   |   81 --
 .../gobblin/publisher/BaseDataPublisher.java    |  688 ----------
 .../BaseDataPublisherWithHiveRegistration.java  |   55 -
 .../publisher/CommitSequencePublisher.java      |   94 --
 .../publisher/HiveRegistrationPublisher.java    |  167 ---
 .../java/gobblin/publisher/NoopPublisher.java   |   96 --
 .../java/gobblin/publisher/TaskPublisher.java   |   89 --
 .../gobblin/publisher/TaskPublisherBuilder.java |   40 -
 .../publisher/TaskPublisherBuilderFactory.java  |   29 -
 .../publisher/TimePartitionedDataPublisher.java |   68 -
 .../publisher/TimestampDataPublisher.java       |   94 --
 ...estampDataPublisherWithHiveRegistration.java |   51 -
 .../row/RowLevelErrFileWriter.java              |   76 --
 .../row/RowLevelPolicyCheckResults.java         |   92 --
 .../row/RowLevelPolicyChecker.java              |  211 ---
 .../row/RowLevelPolicyCheckerBuilder.java       |   90 --
 .../RowLevelPolicyCheckerBuilderFactory.java    |   27 -
 .../task/TaskLevelPolicyCheckResults.java       |   37 -
 .../task/TaskLevelPolicyChecker.java            |   48 -
 .../task/TaskLevelPolicyCheckerBuilder.java     |   90 --
 .../TaskLevelPolicyCheckerBuilderFactory.java   |   27 -
 .../recordaccess/AvroGenericRecordAccessor.java |  228 ----
 .../recordaccess/CoreRecordAccessProvider.java  |   34 -
 .../RecordAccessorProviderFactory.java          |   46 -
 .../main/java/gobblin/retry/RetryerFactory.java |  116 --
 .../gobblin/security/ssl/SSLContextFactory.java |  128 --
 .../source/DatePartitionedAvroFileSource.java   |   68 -
 .../source/DatePartitionedDailyAvroSource.java  |   44 -
 .../source/DatePartitionedNestedRetriever.java  |  208 ---
 .../source/PartitionAwareFileRetriever.java     |  128 --
 .../PartitionAwareFileRetrieverUtils.java       |   55 -
 .../source/PartitionedFileSourceBase.java       |  333 -----
 .../source/RegexBasedPartitionedRetriever.java  |  182 ---
 .../source/RegexPartitionedAvroFileSource.java  |   39 -
 .../DatePartitionedAvroFileExtractor.java       |   43 -
 .../source/extractor/DummyExtractor.java        |   54 -
 .../exception/ExtractPrepareException.java      |   31 -
 .../exception/HighWatermarkException.java       |   31 -
 .../extractor/exception/MetadataException.java  |   31 -
 .../exception/RecordCountException.java         |   31 -
 .../exception/RestApiClientException.java       |   31 -
 .../exception/RestApiConnectionException.java   |   31 -
 .../exception/RestApiProcessingException.java   |   31 -
 .../extractor/exception/SchemaException.java    |   31 -
 .../source/extractor/extract/Command.java       |   35 -
 .../source/extractor/extract/CommandOutput.java |   37 -
 .../source/extractor/extract/CommandType.java   |   28 -
 .../source/extractor/extract/ExtractType.java   |   22 -
 .../extract/ProtocolSpecificLayer.java          |  151 ---
 .../extractor/extract/QueryBasedExtractor.java  |  546 --------
 .../extractor/extract/QueryBasedSource.java     |  468 -------
 .../extractor/extract/SourceSpecificLayer.java  |  153 ---
 .../extract/restapi/RestApiCommand.java         |   66 -
 .../extract/restapi/RestApiCommandOutput.java   |   44 -
 .../extract/restapi/RestApiConnector.java       |  259 ----
 .../extract/restapi/RestApiExtractor.java       |  246 ----
 .../extract/restapi/RestApiSpecificLayer.java   |   32 -
 .../extractor/extract/sftp/SftpExtractor.java   |   34 -
 .../extractor/extract/sftp/SftpFsHelper.java    |  498 -------
 .../extract/sftp/SftpLightWeightFileSystem.java |  277 ----
 .../extractor/extract/sftp/SftpSource.java      |   40 -
 .../extractor/filebased/CsvFileDownloader.java  |  112 --
 .../extractor/filebased/FileBasedExtractor.java |  247 ----
 .../extractor/filebased/FileBasedHelper.java    |   34 -
 .../filebased/FileBasedHelperException.java     |   30 -
 .../extractor/filebased/FileBasedSource.java    |  259 ----
 .../extractor/filebased/FileByteIterator.java   |   62 -
 .../extractor/filebased/FileDownloader.java     |   49 -
 .../extractor/filebased/GZIPFileDownloader.java |   61 -
 .../filebased/SingleFileDownloader.java         |   59 -
 .../filebased/SizeAwareFileBasedHelper.java     |   29 -
 .../SizeAwareFileBasedHelperDecorator.java      |   68 -
 .../filebased/TextFileBasedSource.java          |   54 -
 .../TimestampAwareFileBasedHelper.java          |   29 -
 .../TokenBasedFileInputStreamExtractor.java     |   52 -
 .../filebased/TokenizedFileDownloader.java      |   98 --
 .../extractor/hadoop/AvroFileExtractor.java     |   73 --
 .../source/extractor/hadoop/AvroFileSource.java |   65 -
 .../source/extractor/hadoop/AvroFsHelper.java   |  118 --
 .../hadoop/HadoopFileInputExtractor.java        |  108 --
 .../extractor/hadoop/HadoopFileInputSource.java |  229 ----
 .../source/extractor/hadoop/HadoopFsHelper.java |  177 ---
 .../extractor/hadoop/HadoopTextInputSource.java |   51 -
 .../hadoop/OldApiHadoopFileInputExtractor.java  |  110 --
 .../hadoop/OldApiHadoopFileInputSource.java     |  185 ---
 .../hadoop/OldApiHadoopTextInputSource.java     |   53 -
 .../hadoop/OldApiWritableFileExtractor.java     |   41 -
 .../hadoop/OldApiWritableFileSource.java        |   59 -
 .../extractor/partition/AppendMaxLimitType.java |   22 -
 .../source/extractor/partition/Partition.java   |   95 --
 .../source/extractor/partition/Partitioner.java |  623 ---------
 .../source/extractor/resultset/RecordSet.java   |   36 -
 .../extractor/resultset/RecordSetList.java      |   50 -
 .../source/extractor/schema/ArrayDataType.java  |   35 -
 .../extractor/schema/ColumnAttributes.java      |   62 -
 .../source/extractor/schema/ColumnNameCase.java |   22 -
 .../source/extractor/schema/DataType.java       |   34 -
 .../source/extractor/schema/EnumDataType.java   |   48 -
 .../source/extractor/schema/MapDataType.java    |   35 -
 .../gobblin/source/extractor/schema/Schema.java |  135 --
 .../extractor/utils/InputStreamCSVReader.java   |  262 ----
 .../source/extractor/utils/ProxyFsInput.java    |   68 -
 .../gobblin/source/extractor/utils/Utils.java   |  356 -----
 .../extractor/watermark/DateWatermark.java      |  159 ---
 .../extractor/watermark/HourWatermark.java      |  156 ---
 .../source/extractor/watermark/Predicate.java   |   90 --
 .../extractor/watermark/SimpleWatermark.java    |  124 --
 .../extractor/watermark/TimestampWatermark.java |  155 ---
 .../source/extractor/watermark/Watermark.java   |   52 -
 .../extractor/watermark/WatermarkPredicate.java |   89 --
 .../extractor/watermark/WatermarkType.java      |   22 -
 .../workunit/MultiWorkUnitWeightedQueue.java    |  141 --
 .../main/java/gobblin/state/ConstructState.java |  159 ---
 .../src/main/java/gobblin/util/TestUtils.java   |   45 -
 .../gobblin/writer/AbstractAsyncDataWriter.java |   77 --
 .../gobblin/writer/AvroDataWriterBuilder.java   |   50 -
 .../java/gobblin/writer/AvroHdfsDataWriter.java |  129 --
 .../writer/CloseOnFlushWriterWrapper.java       |  146 ---
 .../main/java/gobblin/writer/ConsoleWriter.java |  112 --
 .../gobblin/writer/ConsoleWriterBuilder.java    |   30 -
 .../writer/DataWriterWrapperBuilder.java        |   57 -
 .../main/java/gobblin/writer/FsDataWriter.java  |  337 -----
 .../gobblin/writer/FsDataWriterBuilder.java     |  128 --
 .../writer/HiveWritableHdfsDataWriter.java      |   99 --
 .../HiveWritableHdfsDataWriterBuilder.java      |   68 -
 .../gobblin/writer/MetadataAwareWriter.java     |   40 -
 .../gobblin/writer/MetadataWriterWrapper.java   |  162 ---
 .../gobblin/writer/PartitionedDataWriter.java   |  338 -----
 .../src/main/java/gobblin/writer/Retriable.java |   26 -
 .../main/java/gobblin/writer/RetryWriter.java   |  213 ---
 ...SchemaBasedPartitionedDataWriterBuilder.java |   63 -
 .../java/gobblin/writer/SimpleDataWriter.java   |  138 --
 .../gobblin/writer/SimpleDataWriterBuilder.java |   79 --
 .../java/gobblin/writer/ThrottleWriter.java     |  218 ----
 .../gobblin/writer/http/AbstractHttpWriter.java |  254 ----
 .../writer/http/AbstractHttpWriterBuilder.java  |  171 ---
 .../DelegatingHttpClientConnectionManager.java  |   80 --
 .../java/gobblin/writer/http/HttpWriter.java    |   75 --
 .../gobblin/writer/http/HttpWriterBuilder.java  |   33 -
 .../writer/http/HttpWriterDecoration.java       |   77 --
 .../writer/http/HttpWriterDecorator.java        |   77 --
 .../gobblin/writer/http/RestJsonWriter.java     |   48 -
 .../writer/http/RestJsonWriterBuilder.java      |   36 -
 .../java/gobblin/writer/http/RestWriter.java    |   47 -
 .../gobblin/writer/http/RestWriterBuilder.java  |   31 -
 .../http/SalesForceRestWriterBuilder.java       |  115 --
 .../writer/http/SalesforceRestWriter.java       |  372 ------
 .../http/UnexpectedResponseException.java       |   42 -
 .../initializer/MultiWriterInitializer.java     |   47 -
 .../initializer/WriterInitializerFactory.java   |   70 -
 .../writer/objectstore/ObjectStoreClient.java   |   95 --
 .../objectstore/ObjectStoreDeleteOperation.java |   88 --
 .../objectstore/ObjectStoreOperation.java       |   47 -
 .../ObjectStoreOperationBuilder.java            |   34 -
 .../writer/objectstore/ObjectStoreWriter.java   |   84 --
 .../objectstore/response/DeleteResponse.java    |   29 -
 .../objectstore/response/GetObjectResponse.java |   37 -
 .../SchemaBasedWriterPartitioner.java           |   50 -
 .../TimeBasedAvroWriterPartitioner.java         |   92 --
 .../partitioner/TimeBasedWriterPartitioner.java |  195 ---
 .../WorkUnitStateWriterPartitioner.java         |   49 -
 .../writer/test/GobblinTestEventBusWriter.java  |  177 ---
 .../writer/test/TestingEventBusAsserter.java    |  178 ---
 .../gobblin/writer/test/TestingEventBuses.java  |   73 --
 .../gobblin/async/AsyncDataDispatcher.java      |  175 +++
 .../org/apache/gobblin/async/AsyncRequest.java  |   74 ++
 .../gobblin/async/AsyncRequestBuilder.java      |   31 +
 .../apache/gobblin/async/BufferedRecord.java    |   34 +
 .../apache/gobblin/async/DispatchException.java |   46 +
 .../gobblin/commit/FsRenameCommitStep.java      |  150 +++
 .../SpeculativeAttemptAwareConstruct.java       |   37 +
 .../apache/gobblin/config/ConfigBuilder.java    |  104 ++
 .../apache/gobblin/converter/EmptyIterable.java |   26 +
 .../GobblinMetricsPinotFlattenerConverter.java  |   87 ++
 .../gobblin/converter/StringSchemaInjector.java |   54 +
 .../avro/AvroFieldRetrieverConverter.java       |   76 ++
 .../avro/AvroRecordToAvroWritableConverter.java |   59 +
 .../avro/AvroToAvroCopyableConverter.java       |   59 +
 .../converter/avro/AvroToBytesConverter.java    |   73 ++
 .../converter/avro/BytesToAvroConverter.java    |   73 ++
 .../avro/FlattenNestedKeyConverter.java         |  145 +++
 .../avro/JsonElementConversionFactory.java      |  536 ++++++++
 .../avro/JsonIntermediateToAvroConverter.java   |  166 +++
 .../avro/UnsupportedDateTypeException.java      |   27 +
 .../converter/csv/CsvToJsonConverter.java       |   95 ++
 .../converter/csv/CsvToJsonConverterV2.java     |  210 +++
 .../filter/AvroFieldsPickConverter.java         |  212 +++
 .../converter/filter/AvroFilterConverter.java   |   92 ++
 .../http/AvroToRestJsonEntryConverter.java      |  106 ++
 .../gobblin/converter/http/RestEntry.java       |   35 +
 .../ConverterInitializerFactory.java            |   81 ++
 .../initializer/MultiConverterInitializer.java  |   44 +
 .../JsonStringToJsonIntermediateConverter.java  |  115 ++
 .../converter/json/JsonToStringConverter.java   |   49 +
 .../objectstore/ObjectStoreConverter.java       |   48 +
 .../objectstore/ObjectStoreDeleteConverter.java |   86 ++
 .../converter/serde/HiveSerDeConverter.java     |  129 ++
 .../string/ObjectToStringConverter.java         |   42 +
 .../converter/string/StringFilterConverter.java |   72 ++
 .../string/StringSplitterConverter.java         |   63 +
 .../string/StringSplitterToListConverter.java   |   70 +
 .../string/StringToBytesConverter.java          |   45 +
 .../converter/string/TextToStringConverter.java |   54 +
 .../gobblin/fork/CopyableGenericRecord.java     |   48 +
 .../org/apache/gobblin/fork/CopyableSchema.java |   41 +
 .../gobblin/fork/IdentityForkOperator.java      |   77 ++
 .../http/DefaultHttpClientConfigurator.java     |  148 +++
 .../gobblin/http/HttpClientConfigurator.java    |   53 +
 .../http/HttpClientConfiguratorLoader.java      |   79 ++
 .../gobblin/initializer/MultiInitializer.java   |   60 +
 .../java/org/apache/gobblin/net/Request.java    |   28 +
 .../policies/avro/AvroHeaderGuidPolicy.java     |   47 +
 .../avro/AvroHeaderTimestampPolicy.java         |   55 +
 .../AvroRecordTimestampLowerBoundPolicy.java    |   41 +
 .../gobblin/policies/count/RowCountPolicy.java  |   52 +
 .../policies/count/RowCountRangePolicy.java     |   54 +
 .../schema/SchemaCompatibilityPolicy.java       |   54 +
 .../policies/schema/SchemaRowCheckPolicy.java   |   34 +
 .../time/RecordTimestampLowerBoundPolicy.java   |   81 ++
 .../gobblin/publisher/BaseDataPublisher.java    |  688 ++++++++++
 .../BaseDataPublisherWithHiveRegistration.java  |   55 +
 .../publisher/CommitSequencePublisher.java      |   94 ++
 .../publisher/HiveRegistrationPublisher.java    |  167 +++
 .../apache/gobblin/publisher/NoopPublisher.java |   96 ++
 .../apache/gobblin/publisher/TaskPublisher.java |   89 ++
 .../gobblin/publisher/TaskPublisherBuilder.java |   40 +
 .../publisher/TaskPublisherBuilderFactory.java  |   29 +
 .../publisher/TimePartitionedDataPublisher.java |   68 +
 .../publisher/TimestampDataPublisher.java       |   94 ++
 ...estampDataPublisherWithHiveRegistration.java |   51 +
 .../row/RowLevelErrFileWriter.java              |   76 ++
 .../row/RowLevelPolicyCheckResults.java         |   92 ++
 .../row/RowLevelPolicyChecker.java              |  211 +++
 .../row/RowLevelPolicyCheckerBuilder.java       |   90 ++
 .../RowLevelPolicyCheckerBuilderFactory.java    |   27 +
 .../task/TaskLevelPolicyCheckResults.java       |   37 +
 .../task/TaskLevelPolicyChecker.java            |   48 +
 .../task/TaskLevelPolicyCheckerBuilder.java     |   90 ++
 .../TaskLevelPolicyCheckerBuilderFactory.java   |   27 +
 .../recordaccess/AvroGenericRecordAccessor.java |  228 ++++
 .../recordaccess/CoreRecordAccessProvider.java  |   34 +
 .../RecordAccessorProviderFactory.java          |   46 +
 .../apache/gobblin/retry/RetryerFactory.java    |  116 ++
 .../gobblin/security/ssl/SSLContextFactory.java |  128 ++
 .../source/DatePartitionedAvroFileSource.java   |   68 +
 .../source/DatePartitionedDailyAvroSource.java  |   44 +
 .../source/DatePartitionedNestedRetriever.java  |  208 +++
 .../source/PartitionAwareFileRetriever.java     |  128 ++
 .../PartitionAwareFileRetrieverUtils.java       |   55 +
 .../source/PartitionedFileSourceBase.java       |  333 +++++
 .../source/RegexBasedPartitionedRetriever.java  |  182 +++
 .../source/RegexPartitionedAvroFileSource.java  |   39 +
 .../DatePartitionedAvroFileExtractor.java       |   43 +
 .../source/extractor/DummyExtractor.java        |   54 +
 .../exception/ExtractPrepareException.java      |   31 +
 .../exception/HighWatermarkException.java       |   31 +
 .../extractor/exception/MetadataException.java  |   31 +
 .../exception/RecordCountException.java         |   31 +
 .../exception/RestApiClientException.java       |   31 +
 .../exception/RestApiConnectionException.java   |   31 +
 .../exception/RestApiProcessingException.java   |   31 +
 .../extractor/exception/SchemaException.java    |   31 +
 .../source/extractor/extract/Command.java       |   35 +
 .../source/extractor/extract/CommandOutput.java |   37 +
 .../source/extractor/extract/CommandType.java   |   28 +
 .../source/extractor/extract/ExtractType.java   |   22 +
 .../extract/ProtocolSpecificLayer.java          |  151 +++
 .../extractor/extract/QueryBasedExtractor.java  |  546 ++++++++
 .../extractor/extract/QueryBasedSource.java     |  468 +++++++
 .../extractor/extract/SourceSpecificLayer.java  |  153 +++
 .../extract/restapi/RestApiCommand.java         |   66 +
 .../extract/restapi/RestApiCommandOutput.java   |   44 +
 .../extract/restapi/RestApiConnector.java       |  259 ++++
 .../extract/restapi/RestApiExtractor.java       |  246 ++++
 .../extract/restapi/RestApiSpecificLayer.java   |   32 +
 .../extractor/extract/sftp/SftpExtractor.java   |   34 +
 .../extractor/extract/sftp/SftpFsHelper.java    |  498 +++++++
 .../extract/sftp/SftpLightWeightFileSystem.java |  277 ++++
 .../extractor/extract/sftp/SftpSource.java      |   40 +
 .../extractor/filebased/CsvFileDownloader.java  |  112 ++
 .../extractor/filebased/FileBasedExtractor.java |  247 ++++
 .../extractor/filebased/FileBasedHelper.java    |   34 +
 .../filebased/FileBasedHelperException.java     |   30 +
 .../extractor/filebased/FileBasedSource.java    |  259 ++++
 .../extractor/filebased/FileByteIterator.java   |   62 +
 .../extractor/filebased/FileDownloader.java     |   49 +
 .../extractor/filebased/GZIPFileDownloader.java |   61 +
 .../filebased/SingleFileDownloader.java         |   59 +
 .../filebased/SizeAwareFileBasedHelper.java     |   29 +
 .../SizeAwareFileBasedHelperDecorator.java      |   68 +
 .../filebased/TextFileBasedSource.java          |   54 +
 .../TimestampAwareFileBasedHelper.java          |   29 +
 .../TokenBasedFileInputStreamExtractor.java     |   52 +
 .../filebased/TokenizedFileDownloader.java      |   98 ++
 .../extractor/hadoop/AvroFileExtractor.java     |   73 ++
 .../source/extractor/hadoop/AvroFileSource.java |   65 +
 .../source/extractor/hadoop/AvroFsHelper.java   |  118 ++
 .../hadoop/HadoopFileInputExtractor.java        |  108 ++
 .../extractor/hadoop/HadoopFileInputSource.java |  229 ++++
 .../source/extractor/hadoop/HadoopFsHelper.java |  177 +++
 .../extractor/hadoop/HadoopTextInputSource.java |   51 +
 .../hadoop/OldApiHadoopFileInputExtractor.java  |  110 ++
 .../hadoop/OldApiHadoopFileInputSource.java     |  185 +++
 .../hadoop/OldApiHadoopTextInputSource.java     |   53 +
 .../hadoop/OldApiWritableFileExtractor.java     |   41 +
 .../hadoop/OldApiWritableFileSource.java        |   59 +
 .../extractor/partition/AppendMaxLimitType.java |   22 +
 .../source/extractor/partition/Partition.java   |   95 ++
 .../source/extractor/partition/Partitioner.java |  623 +++++++++
 .../source/extractor/resultset/RecordSet.java   |   36 +
 .../extractor/resultset/RecordSetList.java      |   50 +
 .../source/extractor/schema/ArrayDataType.java  |   35 +
 .../extractor/schema/ColumnAttributes.java      |   62 +
 .../source/extractor/schema/ColumnNameCase.java |   22 +
 .../source/extractor/schema/DataType.java       |   34 +
 .../source/extractor/schema/EnumDataType.java   |   48 +
 .../source/extractor/schema/MapDataType.java    |   35 +
 .../gobblin/source/extractor/schema/Schema.java |  135 ++
 .../extractor/utils/InputStreamCSVReader.java   |  262 ++++
 .../source/extractor/utils/ProxyFsInput.java    |   68 +
 .../gobblin/source/extractor/utils/Utils.java   |  356 +++++
 .../extractor/watermark/DateWatermark.java      |  159 +++
 .../extractor/watermark/HourWatermark.java      |  156 +++
 .../source/extractor/watermark/Predicate.java   |   90 ++
 .../extractor/watermark/SimpleWatermark.java    |  124 ++
 .../extractor/watermark/TimestampWatermark.java |  155 +++
 .../source/extractor/watermark/Watermark.java   |   52 +
 .../extractor/watermark/WatermarkPredicate.java |   89 ++
 .../extractor/watermark/WatermarkType.java      |   22 +
 .../workunit/MultiWorkUnitWeightedQueue.java    |  141 ++
 .../apache/gobblin/state/ConstructState.java    |  159 +++
 .../java/org/apache/gobblin/util/TestUtils.java |   45 +
 .../gobblin/writer/AbstractAsyncDataWriter.java |   77 ++
 .../gobblin/writer/AvroDataWriterBuilder.java   |   50 +
 .../gobblin/writer/AvroHdfsDataWriter.java      |  129 ++
 .../writer/CloseOnFlushWriterWrapper.java       |  146 +++
 .../apache/gobblin/writer/ConsoleWriter.java    |  112 ++
 .../gobblin/writer/ConsoleWriterBuilder.java    |   30 +
 .../writer/DataWriterWrapperBuilder.java        |   57 +
 .../org/apache/gobblin/writer/FsDataWriter.java |  337 +++++
 .../gobblin/writer/FsDataWriterBuilder.java     |  128 ++
 .../writer/HiveWritableHdfsDataWriter.java      |   99 ++
 .../HiveWritableHdfsDataWriterBuilder.java      |   68 +
 .../gobblin/writer/MetadataAwareWriter.java     |   40 +
 .../gobblin/writer/MetadataWriterWrapper.java   |  162 +++
 .../gobblin/writer/PartitionedDataWriter.java   |  338 +++++
 .../org/apache/gobblin/writer/Retriable.java    |   26 +
 .../org/apache/gobblin/writer/RetryWriter.java  |  213 +++
 ...SchemaBasedPartitionedDataWriterBuilder.java |   63 +
 .../apache/gobblin/writer/SimpleDataWriter.java |  138 ++
 .../gobblin/writer/SimpleDataWriterBuilder.java |   79 ++
 .../apache/gobblin/writer/ThrottleWriter.java   |  218 ++++
 .../gobblin/writer/http/AbstractHttpWriter.java |  254 ++++
 .../writer/http/AbstractHttpWriterBuilder.java  |  171 +++
 .../DelegatingHttpClientConnectionManager.java  |   80 ++
 .../apache/gobblin/writer/http/HttpWriter.java  |   75 ++
 .../gobblin/writer/http/HttpWriterBuilder.java  |   33 +
 .../writer/http/HttpWriterDecoration.java       |   77 ++
 .../writer/http/HttpWriterDecorator.java        |   77 ++
 .../gobblin/writer/http/RestJsonWriter.java     |   48 +
 .../writer/http/RestJsonWriterBuilder.java      |   36 +
 .../apache/gobblin/writer/http/RestWriter.java  |   47 +
 .../gobblin/writer/http/RestWriterBuilder.java  |   31 +
 .../http/SalesForceRestWriterBuilder.java       |  115 ++
 .../writer/http/SalesforceRestWriter.java       |  372 ++++++
 .../http/UnexpectedResponseException.java       |   42 +
 .../initializer/MultiWriterInitializer.java     |   47 +
 .../initializer/WriterInitializerFactory.java   |   70 +
 .../writer/objectstore/ObjectStoreClient.java   |   95 ++
 .../objectstore/ObjectStoreDeleteOperation.java |   88 ++
 .../objectstore/ObjectStoreOperation.java       |   47 +
 .../ObjectStoreOperationBuilder.java            |   34 +
 .../writer/objectstore/ObjectStoreWriter.java   |   84 ++
 .../objectstore/response/DeleteResponse.java    |   29 +
 .../objectstore/response/GetObjectResponse.java |   37 +
 .../SchemaBasedWriterPartitioner.java           |   50 +
 .../TimeBasedAvroWriterPartitioner.java         |   92 ++
 .../partitioner/TimeBasedWriterPartitioner.java |  195 +++
 .../WorkUnitStateWriterPartitioner.java         |   49 +
 .../writer/test/GobblinTestEventBusWriter.java  |  177 +++
 .../writer/test/TestingEventBusAsserter.java    |  178 +++
 .../gobblin/writer/test/TestingEventBuses.java  |   73 ++
 .../gobblin.recordaccess.RecordAccessorProvider |    1 -
 ....gobblin.recordaccess.RecordAccessorProvider |    1 +
 .../src/main/resources/mysql/mysql.properties   |    8 +-
 .../src/main/resources/sftp/sftp-example.job    |    4 +-
 .../resources/sqlserver/sqlserver.properties    |    8 +-
 .../gobblin/async/AsyncDataDispatcherTest.java  |  255 ----
 .../gobblin/commit/FsRenameCommitStepTest.java  |   74 --
 .../java/gobblin/configuration/StateTest.java   |  176 ---
 .../configuration/WorkUnitStateTest.java        |  148 ---
 .../workunit/MultiWorkUnitTest.java             |   92 --
 .../gobblin/converter/EmptyIterableTest.java    |   40 -
 ...bblinMetricsPinotFlattenerConverterTest.java |   79 --
 .../avro/AvroToBytesConverterTest.java          |   68 -
 .../avro/BytesToAvroConverterTest.java          |   61 -
 .../avro/FlattenNestedKeyConverterTest.java     |   94 --
 .../JsonIntermediateToAvroConverterTest.java    |  124 --
 .../converter/csv/CsvToJsonConverterV2Test.java |  115 --
 .../filter/AvroFieldsPickConverterTest.java     |   62 -
 .../http/AvroToRestJsonEntryConverterTest.java  |  107 --
 .../ObjectStoreDeleteConverterTest.java         |  111 --
 .../string/ObjectToStringConverterTest.java     |   78 --
 .../string/StringFilterConverterTest.java       |  109 --
 .../string/StringSplitterConverterTest.java     |   93 --
 .../StringSplitterToListConverterTest.java      |   74 --
 .../string/TextToStringConverterTest.java       |   49 -
 .../gobblin/fork/CopyableGenericRecordTest.java |   61 -
 .../java/gobblin/fork/CopyableSchemaTest.java   |   53 -
 .../gobblin/fork/IdentityForkOperatorTest.java  |   57 -
 .../TestDefaultHttpClientConfiguration.java     |  111 --
 .../http/TestHttpClientConfiguratorLoader.java  |   57 -
 .../gobblin/instrumented/InstrumentedTest.java  |   66 -
 .../policies/count/RowCountRangePolicyTest.java |   56 -
 .../publisher/BaseDataPublisherTest.java        |  559 --------
 .../RowCountTaskLevelPolicyTest.java            |  140 --
 .../RowLevelQualityCheckerTest.java             |   92 --
 .../TaskLevelQualityCheckerTest.java            |   69 -
 .../gobblin/qualitychecker/TestConstants.java   |   27 -
 .../qualitychecker/TestRowLevelPolicy.java      |   33 -
 .../qualitychecker/TestRowLevelPolicyFail.java  |   33 -
 .../qualitychecker/TestTaskLevelPolicy.java     |   33 -
 .../row/FrontLoadedSamplerTest.java             |   60 -
 .../AvroGenericRecordAccessorTest.java          |  211 ---
 .../RecordAccessorProviderFactoryTest.java      |  151 ---
 .../security/ssl/SSLContextFactoryTest.java     |   63 -
 .../test/java/gobblin/serde/HiveSerDeTest.java  |  119 --
 .../RegexBasedPartitionedRetrieverTest.java     |  134 --
 .../DatePartitionedAvroFileExtractorTest.java   |  256 ----
 .../extract/QueryBasedExtractorTest.java        |  235 ----
 .../extractor/extract/QueryBasedSourceTest.java |  205 ---
 .../filebased/FileBasedExtractorTest.java       |  118 --
 .../filebased/FileBasedSourceTest.java          |  105 --
 .../filebased/TokenizedFileDownloaderTest.java  |   56 -
 .../hadoop/HadoopFileInputSourceTest.java       |  107 --
 .../extractor/hadoop/HadoopFsHelperTest.java    |   75 --
 .../hadoop/OldApiHadoopFileInputSourceTest.java |  133 --
 .../extractor/partition/PartitionerTest.java    |  435 -------
 .../extractor/watermark/DateWatermarkTest.java  |  131 --
 .../extractor/watermark/HourWatermarkTest.java  |  130 --
 .../watermark/SimpleWatermarkTest.java          |  160 ---
 .../watermark/TimestampWatermarkTest.java       |  127 --
 .../MultiWorkUnitWeightedQueueTest.java         |   82 --
 .../java/gobblin/state/ConstructStateTest.java  |   85 --
 .../gobblin/writer/AvroHdfsDataWriterTest.java  |  158 ---
 .../writer/CloseOnFlushWriterWrapperTest.java   |  160 ---
 .../java/gobblin/writer/ConsoleWriterTest.java  |  111 --
 .../java/gobblin/writer/DestinationTest.java    |   43 -
 .../writer/MetadataWriterWrapperTest.java       |  214 ---
 .../gobblin/writer/PartitionedWriterTest.java   |  167 ---
 .../java/gobblin/writer/RetryWriterTest.java    |   83 --
 .../gobblin/writer/SimpleDataWriterTest.java    |  389 ------
 .../test/java/gobblin/writer/TestConstants.java |   63 -
 .../java/gobblin/writer/ThrottleWriterTest.java |  114 --
 .../writer/http/SalesforceRestWriterTest.java   |  399 ------
 .../objectstore/ObjectStoreWriterTest.java      |  126 --
 .../SchemaBasedWriterPartitionerTest.java       |   45 -
 .../TimeBasedAvroWriterPartitionerTest.java     |  154 ---
 .../test/GobblinTestEventBusWriterTest.java     |   79 --
 .../test/TestPartitionAwareWriterBuilder.java   |  129 --
 .../gobblin/writer/test/TestPartitioner.java    |   50 -
 .../test/TestingEventBusAsserterTest.java       |  123 --
 .../gobblin/async/AsyncDataDispatcherTest.java  |  255 ++++
 .../gobblin/commit/FsRenameCommitStepTest.java  |   74 ++
 .../apache/gobblin/configuration/StateTest.java |  176 +++
 .../configuration/WorkUnitStateTest.java        |  148 +++
 .../workunit/MultiWorkUnitTest.java             |   92 ++
 .../gobblin/converter/EmptyIterableTest.java    |   40 +
 ...bblinMetricsPinotFlattenerConverterTest.java |   79 ++
 .../avro/AvroToBytesConverterTest.java          |   68 +
 .../avro/BytesToAvroConverterTest.java          |   61 +
 .../avro/FlattenNestedKeyConverterTest.java     |   94 ++
 .../JsonIntermediateToAvroConverterTest.java    |  124 ++
 .../converter/csv/CsvToJsonConverterV2Test.java |  115 ++
 .../filter/AvroFieldsPickConverterTest.java     |   62 +
 .../http/AvroToRestJsonEntryConverterTest.java  |  107 ++
 .../ObjectStoreDeleteConverterTest.java         |  111 ++
 .../string/ObjectToStringConverterTest.java     |   78 ++
 .../string/StringFilterConverterTest.java       |  109 ++
 .../string/StringSplitterConverterTest.java     |   93 ++
 .../StringSplitterToListConverterTest.java      |   74 ++
 .../string/TextToStringConverterTest.java       |   49 +
 .../gobblin/fork/CopyableGenericRecordTest.java |   61 +
 .../apache/gobblin/fork/CopyableSchemaTest.java |   53 +
 .../gobblin/fork/IdentityForkOperatorTest.java  |   57 +
 .../TestDefaultHttpClientConfiguration.java     |  111 ++
 .../http/TestHttpClientConfiguratorLoader.java  |   57 +
 .../gobblin/instrumented/InstrumentedTest.java  |   66 +
 .../policies/count/RowCountRangePolicyTest.java |   56 +
 .../publisher/BaseDataPublisherTest.java        |  559 ++++++++
 .../RowCountTaskLevelPolicyTest.java            |  140 ++
 .../RowLevelQualityCheckerTest.java             |   92 ++
 .../TaskLevelQualityCheckerTest.java            |   69 +
 .../gobblin/qualitychecker/TestConstants.java   |   27 +
 .../qualitychecker/TestRowLevelPolicy.java      |   33 +
 .../qualitychecker/TestRowLevelPolicyFail.java  |   33 +
 .../qualitychecker/TestTaskLevelPolicy.java     |   33 +
 .../row/FrontLoadedSamplerTest.java             |   60 +
 .../AvroGenericRecordAccessorTest.java          |  211 +++
 .../RecordAccessorProviderFactoryTest.java      |  151 +++
 .../security/ssl/SSLContextFactoryTest.java     |   63 +
 .../org/apache/gobblin/serde/HiveSerDeTest.java |  119 ++
 .../RegexBasedPartitionedRetrieverTest.java     |  134 ++
 .../DatePartitionedAvroFileExtractorTest.java   |  256 ++++
 .../extract/QueryBasedExtractorTest.java        |  235 ++++
 .../extractor/extract/QueryBasedSourceTest.java |  205 +++
 .../filebased/FileBasedExtractorTest.java       |  118 ++
 .../filebased/FileBasedSourceTest.java          |  105 ++
 .../filebased/TokenizedFileDownloaderTest.java  |   56 +
 .../hadoop/HadoopFileInputSourceTest.java       |  107 ++
 .../extractor/hadoop/HadoopFsHelperTest.java    |   75 ++
 .../hadoop/OldApiHadoopFileInputSourceTest.java |  133 ++
 .../extractor/partition/PartitionerTest.java    |  435 +++++++
 .../extractor/watermark/DateWatermarkTest.java  |  131 ++
 .../extractor/watermark/HourWatermarkTest.java  |  130 ++
 .../watermark/SimpleWatermarkTest.java          |  160 +++
 .../watermark/TimestampWatermarkTest.java       |  127 ++
 .../MultiWorkUnitWeightedQueueTest.java         |   82 ++
 .../gobblin/state/ConstructStateTest.java       |   85 ++
 .../gobblin/writer/AvroHdfsDataWriterTest.java  |  158 +++
 .../writer/CloseOnFlushWriterWrapperTest.java   |  160 +++
 .../gobblin/writer/ConsoleWriterTest.java       |  111 ++
 .../apache/gobblin/writer/DestinationTest.java  |   43 +
 .../writer/MetadataWriterWrapperTest.java       |  214 +++
 .../gobblin/writer/PartitionedWriterTest.java   |  167 +++
 .../apache/gobblin/writer/RetryWriterTest.java  |   83 ++
 .../gobblin/writer/SimpleDataWriterTest.java    |  389 ++++++
 .../apache/gobblin/writer/TestConstants.java    |   63 +
 .../gobblin/writer/ThrottleWriterTest.java      |  114 ++
 .../writer/http/SalesforceRestWriterTest.java   |  399 ++++++
 .../objectstore/ObjectStoreWriterTest.java      |  126 ++
 .../SchemaBasedWriterPartitionerTest.java       |   45 +
 .../TimeBasedAvroWriterPartitionerTest.java     |  154 +++
 .../test/GobblinTestEventBusWriterTest.java     |   79 ++
 .../test/TestPartitionAwareWriterBuilder.java   |  129 ++
 .../gobblin/writer/test/TestPartitioner.java    |   50 +
 .../test/TestingEventBusAsserterTest.java       |  123 ++
 .../gobblin.recordaccess.RecordAccessorProvider |    1 -
 ....gobblin.recordaccess.RecordAccessorProvider |    1 +
 .../resources/converter/record_with_arrays.avsc |    2 +-
 .../1.0/tags/retention/hive/main.conf           |    6 +-
 .../1.0/tags/retention/timebased/main.conf      |    6 +-
 .../conversion/hive/avro/AvroSchemaManager.java |  186 ---
 .../hive/avro/SchemaNotFoundException.java      |   42 -
 .../converter/AbstractAvroToOrcConverter.java   |  810 ------------
 .../HiveAvroToFlattenedOrcConverter.java        |   56 -
 .../converter/HiveAvroToNestedOrcConverter.java |   54 -
 .../hive/dataset/ConvertibleHiveDataset.java    |  260 ----
 .../dataset/ConvertibleHiveDatasetFinder.java   |   44 -
 .../QueryBasedHiveConversionEntity.java         |   79 --
 .../entities/QueryBasedHivePublishEntity.java   |   55 -
 .../hive/entities/SchemaAwareHivePartition.java |   42 -
 .../hive/entities/SchemaAwareHiveTable.java     |   39 -
 .../conversion/hive/events/EventConstants.java  |   58 -
 .../hive/events/EventWorkunitUtils.java         |  113 --
 .../hive/extractor/HiveBaseExtractor.java       |   69 -
 .../extractor/HiveBaseExtractorFactory.java     |   33 -
 .../hive/extractor/HiveConvertExtractor.java    |  123 --
 .../extractor/HiveConvertExtractorFactory.java  |   35 -
 .../provider/DatePatternUpdateProvider.java     |   82 --
 .../hive/provider/HdfsBasedUpdateProvider.java  |   99 --
 .../HiveMetastoreBasedUpdateProvider.java       |   50 -
 .../hive/provider/HiveUnitUpdateProvider.java   |   37 -
 .../hive/provider/UpdateNotFoundException.java  |   42 -
 .../hive/provider/UpdateProviderFactory.java    |   64 -
 .../hive/publisher/HiveConvertPublisher.java    |  456 -------
 .../hive/query/HiveAvroORCQueryGenerator.java   | 1116 ----------------
 .../query/HiveValidationQueryGenerator.java     |  177 ---
 .../hive/source/BackfillHiveSource.java         |   93 --
 .../hive/source/HiveAvroToOrcSource.java        |   43 -
 .../conversion/hive/source/HiveSource.java      |  450 -------
 .../conversion/hive/source/HiveWorkUnit.java    |  192 ---
 .../hive/watermarker/HiveSourceWatermarker.java |   96 --
 .../HiveSourceWatermarkerFactory.java           |   29 -
 .../watermarker/MultiKeyValueLongWatermark.java |   89 --
 .../watermarker/PartitionLevelWatermarker.java  |  434 -------
 .../hive/watermarker/TableLevelWatermarker.java |  132 --
 .../hive/writer/HiveQueryExecutionWriter.java   |  124 --
 .../hive/writer/HiveQueryWriterBuilder.java     |   43 -
 .../management/copy/AllEqualComparator.java     |   34 -
 .../management/copy/CloseableFsCopySource.java  |   80 --
 .../copy/ConcurrentBoundedWorkUnitList.java     |  188 ---
 .../data/management/copy/CopyConfiguration.java |  137 --
 .../data/management/copy/CopyContext.java       |   78 --
 .../data/management/copy/CopyEntity.java        |  150 ---
 .../data/management/copy/CopyResourcePool.java  |  111 --
 .../data/management/copy/CopySource.java        |  441 -------
 .../data/management/copy/CopyableDataset.java   |   52 -
 .../management/copy/CopyableDatasetBase.java    |   28 -
 .../copy/CopyableDatasetMetadata.java           |   63 -
 .../data/management/copy/CopyableFile.java      |  336 -----
 .../management/copy/CopyableFileFilter.java     |   42 -
 .../copy/CopyableGlobDatasetFinder.java         |   43 -
 .../management/copy/FileAwareInputStream.java   |   40 -
 .../copy/IterableCopyableDataset.java           |   44 -
 .../copy/IterableCopyableDatasetImpl.java       |   73 --
 .../management/copy/OwnerAndPermission.java     |   69 -
 .../management/copy/PreserveAttributes.java     |  123 --
 .../copy/ReadyCopyableFileFilter.java           |   72 --
 .../copy/RecursiveCopyableDataset.java          |  181 ---
 .../management/copy/RecursivePathFinder.java    |   68 -
 .../copy/SubsetFilesCopyableDataset.java        |  119 --
 .../copy/SubsetFilesCopyableDatasetFinder.java  |  148 ---
 .../copy/TimestampBasedCopyableDataset.java     |  232 ----
 .../copy/converter/DecryptConverter.java        |  101 --
 .../copy/converter/DistcpConverter.java         |   99 --
 .../copy/converter/UnGzipConverter.java         |   64 -
 .../copy/entities/CommitStepCopyEntity.java     |   51 -
 .../copy/entities/PostPublishStep.java          |   39 -
 .../copy/entities/PrePublishStep.java           |   39 -
 ...loseableFsFileAwareInputStreamExtractor.java |   55 -
 .../copy/extractor/EmptyExtractor.java          |   53 -
 .../FileAwareInputStreamExtractor.java          |  103 --
 .../copy/hive/HiveCopyEntityHelper.java         |  763 -----------
 .../data/management/copy/hive/HiveDataset.java  |  332 -----
 .../management/copy/hive/HiveDatasetFinder.java |  343 -----
 .../data/management/copy/hive/HiveFileSet.java  |   42 -
 .../copy/hive/HiveLocationDescriptor.java       |  111 --
 .../copy/hive/HivePartitionExtendedFilter.java  |   36 -
 .../HivePartitionExtendedFilterFactory.java     |   34 -
 .../copy/hive/HivePartitionFileSet.java         |  185 ---
 .../hive/HivePartitionsDeregisterFileSet.java   |   65 -
 .../HiveTableLocationNotMatchException.java     |   46 -
 .../copy/hive/HiveTargetPathHelper.java         |  188 ---
 .../data/management/copy/hive/HiveUtils.java    |  172 ---
 .../copy/hive/PartitionFilterGenerator.java     |   27 -
 .../PathBasedHivePartitionFilterFactory.java    |   43 -
 .../copy/hive/PathBasedPartitionFilter.java     |   46 -
 .../copy/hive/UnpartitionedTableFileSet.java    |  129 --
 .../hive/avro/HiveAvroCopyEntityHelper.java     |  111 --
 .../LookbackPartitionFilterGenerator.java       |   74 --
 .../management/copy/predicates/AlwaysTrue.java  |   34 -
 ...itionTableRegistrationTimeSkipPredicate.java |   91 --
 .../RegistrationTimeSkipPredicate.java          |  100 --
 .../RootDirectoryModtimeSkipPredicate.java      |   91 --
 .../copy/prioritization/FileSetComparator.java  |   37 -
 .../PrioritizedCopyableDataset.java             |   54 -
 .../copy/publisher/CopyDataPublisher.java       |  302 -----
 .../publisher/CopyEventSubmitterHelper.java     |   83 --
 .../publisher/DeletingCopyDataPublisher.java    |   83 --
 .../copy/recovery/RecoveryHelper.java           |  188 ---
 .../ConfigBasedCopyableDatasetFinder.java       |   65 -
 .../copy/replication/ConfigBasedDataset.java    |  219 ----
 .../replication/ConfigBasedDatasetsFinder.java  |  313 -----
 .../replication/ConfigBasedMultiDatasets.java   |  219 ----
 .../management/copy/replication/CopyRoute.java  |   37 -
 .../copy/replication/CopyRouteGenerator.java    |   43 -
 .../replication/CopyRouteGeneratorBase.java     |   60 -
 .../CopyRouteGeneratorOptimizedLatency.java     |   83 --
 ...RouteGeneratorOptimizedNetworkBandwidth.java |   53 -
 ...neratorOptimizedNetworkBandwidthForTest.java |   46 -
 .../CopyRouteGeneratorOptimizer.java            |   58 -
 .../copy/replication/DataFlowTopology.java      |   80 --
 .../DataFlowTopologyPickerByHadoopFsSource.java |   40 -
 .../DataFlowTopologyPickerBySource.java         |   31 -
 .../management/copy/replication/EndPoint.java   |   70 -
 .../copy/replication/EndPointFactory.java       |   29 -
 .../copy/replication/HadoopFsEndPoint.java      |   83 --
 .../replication/HadoopFsEndPointDataset.java    |   60 -
 .../replication/HadoopFsEndPointFactory.java    |   45 -
 .../copy/replication/HadoopFsReplicaConfig.java |  120 --
 .../replication/ReplicaHadoopFsEndPoint.java    |  190 ---
 .../replication/ReplicationConfiguration.java   |  387 ------
 .../copy/replication/ReplicationCopyMode.java   |   61 -
 .../ReplicationDataValidPathPicker.java         |   92 --
 .../copy/replication/ReplicationMetaData.java   |   65 -
 .../replication/SourceHadoopFsEndPoint.java     |  156 ---
 .../WatermarkMetadataGenerationCommitStep.java  |   89 --
 .../copy/replication/WatermarkMetadataUtil.java |   67 -
 .../CopyableFileWatermarkGenerator.java         |   42 -
 .../watermark/CopyableFileWatermarkHelper.java  |   67 -
 .../FullPathCopyableFileWatermarkGenerator.java |   46 -
 .../copy/watermark/StringWatermark.java         |   57 -
 .../writer/FileAwareInputStreamDataWriter.java  |  452 -------
 .../FileAwareInputStreamDataWriterBuilder.java  |   66 -
 .../writer/TarArchiveInputStreamDataWriter.java |  112 --
 .../TarArchiveInputStreamDataWriterBuilder.java |   36 -
 .../data/management/dataset/Dataset.java        |   24 -
 .../data/management/dataset/DatasetUtils.java   |  141 --
 .../dataset/DefaultFileSystemGlobFinder.java    |   32 -
 .../data/management/dataset/DummyDataset.java   |   61 -
 .../SimpleDatasetHierarchicalPrioritizer.java   |  105 --
 .../dataset/SimpleDatasetRequest.java           |   68 -
 .../dataset/SimpleDatasetRequestor.java         |   51 -
 .../management/hive/HiveConfigClientUtils.java  |   39 -
 .../management/hive/HiveRegistrableDataset.java |   39 -
 .../partition/CopyableDatasetRequestor.java     |  121 --
 .../gobblin/data/management/partition/File.java |   33 -
 .../data/management/partition/FileSet.java      |  155 ---
 .../partition/FileSetResourceEstimator.java     |   52 -
 .../management/partition/StaticFileSet.java     |   46 -
 .../policy/CombineSelectionPolicy.java          |  201 ---
 .../EmbeddedRetentionSelectionPolicy.java       |   48 -
 .../policy/NewestKSelectionPolicy.java          |  149 ---
 .../policy/SelectAfterTimeBasedPolicy.java      |   56 -
 .../data/management/policy/SelectAllPolicy.java |   46 -
 .../policy/SelectBeforeTimeBasedPolicy.java     |   56 -
 .../policy/SelectBetweenTimeBasedPolicy.java    |  110 --
 .../management/policy/SelectNothingPolicy.java  |   48 -
 .../policy/VersionSelectionPolicy.java          |   43 -
 .../management/retention/DatasetCleaner.java    |  226 ----
 .../management/retention/RetentionEvents.java   |   39 -
 .../retention/action/AccessControlAction.java   |  168 ---
 .../action/MultiAccessControlAction.java        |  124 --
 .../retention/action/RetentionAction.java       |  123 --
 .../retention/dataset/CleanableDataset.java     |   39 -
 .../retention/dataset/CleanableDatasetBase.java |  127 --
 .../retention/dataset/CleanableHiveDataset.java |  170 ---
 .../dataset/ConfigurableCleanableDataset.java   |  250 ----
 .../retention/dataset/FsCleanableHelper.java    |  165 ---
 .../dataset/ModificationTimeDataset.java        |   72 --
 .../MultiVersionCleanableDatasetBase.java       |  378 ------
 .../retention/dataset/SnapshotDataset.java      |   74 --
 .../retention/dataset/TrackingDataset.java      |   75 --
 .../finder/CleanableHiveDatasetFinder.java      |   53 -
 .../retention/dataset/finder/DatasetFinder.java |   30 -
 .../policy/CombineRetentionPolicy.java          |  198 ---
 .../policy/DeleteAllRetentionPolicy.java        |   43 -
 .../policy/DeleteNothingRetentionPolicy.java    |   47 -
 .../policy/NewestKRetentionPolicy.java          |   92 --
 .../policy/PredicateRetentionPolicy.java        |   62 -
 .../policy/RawDatasetRetentionPolicy.java       |  104 --
 .../retention/policy/RetentionPolicy.java       |   50 -
 .../policy/TimeBasedRetentionPolicy.java        |  130 --
 .../policy/predicates/WhitelistPredicate.java   |   52 -
 .../ConfigBasedCleanabledDatasetFinder.java     |   70 -
 .../profile/ConfigurableGlobDatasetFinder.java  |  166 ---
 .../profile/GlobCleanableDatasetFinder.java     |   46 -
 .../profile/ManagedCleanableDatasetFinder.java  |   70 -
 .../profile/ModificationTimeDatasetProfile.java |   44 -
 .../profile/MultiCleanableDatasetFinder.java    |   68 -
 .../retention/profile/MultiDatasetFinder.java   |  144 ---
 .../profile/ProxyableDatasetProfile.java        |   74 --
 .../profile/SnapshotDatasetProfile.java         |   48 -
 .../profile/TrackingDatasetProfile.java         |   57 -
 .../retention/version/DatasetVersion.java       |   47 -
 .../version/FileStatusDatasetVersion.java       |   65 -
 .../version/HiveDatasetVersionCleaner.java      |  175 ---
 .../retention/version/StringDatasetVersion.java |   46 -
 .../version/TimestampedDatasetVersion.java      |   60 -
 .../retention/version/VersionCleaner.java       |   63 -
 .../version/finder/DatasetVersionFinder.java    |   42 -
 .../finder/DateTimeDatasetVersionFinder.java    |  106 --
 .../finder/FileLevelTimestampVersionFinder.java |   54 -
 .../finder/GlobModTimeDatasetVersionFinder.java |   67 -
 .../finder/ModDateTimeDatasetVersionFinder.java |   53 -
 .../version/finder/SingleVersionFinder.java     |   60 -
 .../finder/UnixTimestampVersionFinder.java      |   67 -
 .../retention/version/finder/VersionFinder.java |   30 -
 .../finder/WatermarkDatasetVersionFinder.java   |   81 --
 .../data/management/trash/AsyncTrash.java       |  157 ---
 .../management/trash/GobblinProxiedTrash.java   |   39 -
 .../data/management/trash/GobblinTrash.java     |   38 -
 .../trash/ImmediateDeletionTrash.java           |   59 -
 .../data/management/trash/MockTrash.java        |   87 --
 .../trash/NoopSnapshotCleanupPolicy.java        |   36 -
 .../data/management/trash/ProxiedTrash.java     |  166 ---
 .../management/trash/SnapshotCleanupPolicy.java |   38 -
 .../data/management/trash/TestTrash.java        |  247 ----
 .../trash/TimeBasedSnapshotCleanupPolicy.java   |   46 -
 .../gobblin/data/management/trash/Trash.java    |  301 -----
 .../data/management/trash/TrashFactory.java     |  110 --
 .../data/management/version/DatasetVersion.java |   30 -
 .../management/version/FileStatusAware.java     |   33 -
 .../version/FileStatusDatasetVersion.java       |   65 -
 .../FileStatusTimestampedDatasetVersion.java    |   45 -
 .../version/FileSystemDatasetVersion.java       |   36 -
 .../management/version/HiveDatasetVersion.java  |   31 -
 .../version/StringDatasetVersion.java           |   69 -
 .../version/TimestampedDatasetVersion.java      |   74 --
 .../version/TimestampedHiveDatasetVersion.java  |   51 -
 .../finder/AbstractDatasetVersionFinder.java    |  111 --
 .../AbstractHiveDatasetVersionFinder.java       |  100 --
 .../version/finder/DatasetVersionFinder.java    |   66 -
 .../finder/DatePartitionHiveVersionFinder.java  |  116 --
 .../finder/DateTimeDatasetVersionFinder.java    |  140 --
 .../finder/FileLevelTimestampVersionFinder.java |   83 --
 .../finder/GlobModTimeDatasetVersionFinder.java |   76 --
 .../finder/ModDateTimeDatasetVersionFinder.java |   65 -
 .../version/finder/SingleVersionFinder.java     |   65 -
 .../finder/UnixTimestampVersionFinder.java      |   84 --
 .../version/finder/VersionFinder.java           |   47 -
 .../finder/WatermarkDatasetVersionFinder.java   |  111 --
 .../runtime/embedded/EmbeddedGobblinDistcp.java |  132 --
 .../runtime/retention/DatasetCleanerCli.java    |   89 --
 .../util/commit/DeleteFileCommitStep.java       |  157 ---
 .../SimpleHiveDatasetTieringPrioritizer.java    |  119 --
 .../util/test/RetentionTestDataGenerator.java   |  234 ----
 .../gobblin/util/test/RetentionTestHelper.java  |  119 --
 .../conversion/hive/avro/AvroSchemaManager.java |  186 +++
 .../hive/avro/SchemaNotFoundException.java      |   42 +
 .../converter/AbstractAvroToOrcConverter.java   |  810 ++++++++++++
 .../HiveAvroToFlattenedOrcConverter.java        |   56 +
 .../converter/HiveAvroToNestedOrcConverter.java |   54 +
 .../hive/dataset/ConvertibleHiveDataset.java    |  260 ++++
 .../dataset/ConvertibleHiveDatasetFinder.java   |   44 +
 .../QueryBasedHiveConversionEntity.java         |   79 ++
 .../entities/QueryBasedHivePublishEntity.java   |   55 +
 .../hive/entities/SchemaAwareHivePartition.java |   42 +
 .../hive/entities/SchemaAwareHiveTable.java     |   39 +
 .../conversion/hive/events/EventConstants.java  |   58 +
 .../hive/events/EventWorkunitUtils.java         |  113 ++
 .../hive/extractor/HiveBaseExtractor.java       |   69 +
 .../extractor/HiveBaseExtractorFactory.java     |   33 +
 .../hive/extractor/HiveConvertExtractor.java    |  123 ++
 .../extractor/HiveConvertExtractorFactory.java  |   35 +
 .../provider/DatePatternUpdateProvider.java     |   82 ++
 .../hive/provider/HdfsBasedUpdateProvider.java  |   99 ++
 .../HiveMetastoreBasedUpdateProvider.java       |   50 +
 .../hive/provider/HiveUnitUpdateProvider.java   |   37 +
 .../hive/provider/UpdateNotFoundException.java  |   42 +
 .../hive/provider/UpdateProviderFactory.java    |   64 +
 .../hive/publisher/HiveConvertPublisher.java    |  456 +++++++
 .../hive/query/HiveAvroORCQueryGenerator.java   | 1116 ++++++++++++++++
 .../query/HiveValidationQueryGenerator.java     |  177 +++
 .../hive/source/BackfillHiveSource.java         |   93 ++
 .../hive/source/HiveAvroToOrcSource.java        |   43 +
 .../conversion/hive/source/HiveSource.java      |  450 +++++++
 .../conversion/hive/source/HiveWorkUnit.java    |  192 +++
 .../hive/watermarker/HiveSourceWatermarker.java |   96 ++
 .../HiveSourceWatermarkerFactory.java           |   29 +
 .../watermarker/MultiKeyValueLongWatermark.java |   89 ++
 .../watermarker/PartitionLevelWatermarker.java  |  434 +++++++
 .../hive/watermarker/TableLevelWatermarker.java |  132 ++
 .../hive/writer/HiveQueryExecutionWriter.java   |  124 ++
 .../hive/writer/HiveQueryWriterBuilder.java     |   43 +
 .../management/copy/AllEqualComparator.java     |   34 +
 .../management/copy/CloseableFsCopySource.java  |   80 ++
 .../copy/ConcurrentBoundedWorkUnitList.java     |  188 +++
 .../data/management/copy/CopyConfiguration.java |  137 ++
 .../data/management/copy/CopyContext.java       |   78 ++
 .../data/management/copy/CopyEntity.java        |  150 +++
 .../data/management/copy/CopyResourcePool.java  |  111 ++
 .../data/management/copy/CopySource.java        |  441 +++++++
 .../data/management/copy/CopyableDataset.java   |   52 +
 .../management/copy/CopyableDatasetBase.java    |   28 +
 .../copy/CopyableDatasetMetadata.java           |   63 +
 .../data/management/copy/CopyableFile.java      |  336 +++++
 .../management/copy/CopyableFileFilter.java     |   42 +
 .../copy/CopyableGlobDatasetFinder.java         |   43 +
 .../management/copy/FileAwareInputStream.java   |   40 +
 .../copy/IterableCopyableDataset.java           |   44 +
 .../copy/IterableCopyableDatasetImpl.java       |   73 ++
 .../management/copy/OwnerAndPermission.java     |   69 +
 .../management/copy/PreserveAttributes.java     |  123 ++
 .../copy/ReadyCopyableFileFilter.java           |   72 ++
 .../copy/RecursiveCopyableDataset.java          |  181 +++
 .../management/copy/RecursivePathFinder.java    |   68 +
 .../copy/SubsetFilesCopyableDataset.java        |  119 ++
 .../copy/SubsetFilesCopyableDatasetFinder.java  |  148 +++
 .../copy/TimestampBasedCopyableDataset.java     |  232 ++++
 .../copy/converter/DecryptConverter.java        |  101 ++
 .../copy/converter/DistcpConverter.java         |   99 ++
 .../copy/converter/UnGzipConverter.java         |   64 +
 .../copy/entities/CommitStepCopyEntity.java     |   51 +
 .../copy/entities/PostPublishStep.java          |   39 +
 .../copy/entities/PrePublishStep.java           |   39 +
 ...loseableFsFileAwareInputStreamExtractor.java |   55 +
 .../copy/extractor/EmptyExtractor.java          |   53 +
 .../FileAwareInputStreamExtractor.java          |  103 ++
 .../copy/hive/HiveCopyEntityHelper.java         |  763 +++++++++++
 .../data/management/copy/hive/HiveDataset.java  |  332 +++++
 .../management/copy/hive/HiveDatasetFinder.java |  343 +++++
 .../data/management/copy/hive/HiveFileSet.java  |   42 +
 .../copy/hive/HiveLocationDescriptor.java       |  111 ++
 .../copy/hive/HivePartitionExtendedFilter.java  |   36 +
 .../HivePartitionExtendedFilterFactory.java     |   34 +
 .../copy/hive/HivePartitionFileSet.java         |  185 +++
 .../hive/HivePartitionsDeregisterFileSet.java   |   65 +
 .../HiveTableLocationNotMatchException.java     |   46 +
 .../copy/hive/HiveTargetPathHelper.java         |  188 +++
 .../data/management/copy/hive/HiveUtils.java    |  172 +++
 .../copy/hive/PartitionFilterGenerator.java     |   27 +
 .../PathBasedHivePartitionFilterFactory.java    |   43 +
 .../copy/hive/PathBasedPartitionFilter.java     |   46 +
 .../copy/hive/UnpartitionedTableFileSet.java    |  129 ++
 .../hive/avro/HiveAvroCopyEntityHelper.java     |  111 ++
 .../LookbackPartitionFilterGenerator.java       |   74 ++
 .../management/copy/predicates/AlwaysTrue.java  |   34 +
 ...itionTableRegistrationTimeSkipPredicate.java |   91 ++
 .../RegistrationTimeSkipPredicate.java          |  100 ++
 .../RootDirectoryModtimeSkipPredicate.java      |   91 ++
 .../copy/prioritization/FileSetComparator.java  |   37 +
 .../PrioritizedCopyableDataset.java             |   54 +
 .../copy/publisher/CopyDataPublisher.java       |  302 +++++
 .../publisher/CopyEventSubmitterHelper.java     |   83 ++
 .../publisher/DeletingCopyDataPublisher.java    |   83 ++
 .../copy/recovery/RecoveryHelper.java           |  188 +++
 .../ConfigBasedCopyableDatasetFinder.java       |   65 +
 .../copy/replication/ConfigBasedDataset.java    |  219 ++++
 .../replication/ConfigBasedDatasetsFinder.java  |  313 +++++
 .../replication/ConfigBasedMultiDatasets.java   |  219 ++++
 .../management/copy/replication/CopyRoute.java  |   37 +
 .../copy/replication/CopyRouteGenerator.java    |   43 +
 .../replication/CopyRouteGeneratorBase.java     |   60 +
 .../CopyRouteGeneratorOptimizedLatency.java     |   83 ++
 ...RouteGeneratorOptimizedNetworkBandwidth.java |   53 +
 ...neratorOptimizedNetworkBandwidthForTest.java |   46 +
 .../CopyRouteGeneratorOptimizer.java            |   58 +
 .../copy/replication/DataFlowTopology.java      |   80 ++
 .../DataFlowTopologyPickerByHadoopFsSource.java |   40 +
 .../DataFlowTopologyPickerBySource.java         |   31 +
 .../management/copy/replication/EndPoint.java   |   70 +
 .../copy/replication/EndPointFactory.java       |   29 +
 .../copy/replication/HadoopFsEndPoint.java      |   83 ++
 .../replication/HadoopFsEndPointDataset.java    |   60 +
 .../replication/HadoopFsEndPointFactory.java    |   45 +
 .../copy/replication/HadoopFsReplicaConfig.java |  120 ++
 .../replication/ReplicaHadoopFsEndPoint.java    |  190 +++
 .../replication/ReplicationConfiguration.java   |  387 ++++++
 .../copy/replication/ReplicationCopyMode.java   |   61 +
 .../ReplicationDataValidPathPicker.java         |   92 ++
 .../copy/replication/ReplicationMetaData.java   |   65 +
 .../replication/SourceHadoopFsEndPoint.java     |  156 +++
 .../WatermarkMetadataGenerationCommitStep.java  |   89 ++
 .../copy/replication/WatermarkMetadataUtil.java |   67 +
 .../CopyableFileWatermarkGenerator.java         |   42 +
 .../watermark/CopyableFileWatermarkHelper.java  |   67 +
 .../FullPathCopyableFileWatermarkGenerator.java |   46 +
 .../copy/watermark/StringWatermark.java         |   57 +
 .../writer/FileAwareInputStreamDataWriter.java  |  452 +++++++
 .../FileAwareInputStreamDataWriterBuilder.java  |   66 +
 .../writer/TarArchiveInputStreamDataWriter.java |  112 ++
 .../TarArchiveInputStreamDataWriterBuilder.java |   36 +
 .../data/management/dataset/Dataset.java        |   24 +
 .../data/management/dataset/DatasetUtils.java   |  141 ++
 .../dataset/DefaultFileSystemGlobFinder.java    |   32 +
 .../data/management/dataset/DummyDataset.java   |   61 +
 .../SimpleDatasetHierarchicalPrioritizer.java   |  105 ++
 .../dataset/SimpleDatasetRequest.java           |   68 +
 .../dataset/SimpleDatasetRequestor.java         |   51 +
 .../management/hive/HiveConfigClientUtils.java  |   39 +
 .../management/hive/HiveRegistrableDataset.java |   39 +
 .../partition/CopyableDatasetRequestor.java     |  121 ++
 .../gobblin/data/management/partition/File.java |   33 +
 .../data/management/partition/FileSet.java      |  155 +++
 .../partition/FileSetResourceEstimator.java     |   52 +
 .../management/partition/StaticFileSet.java     |   46 +
 .../policy/CombineSelectionPolicy.java          |  201 +++
 .../EmbeddedRetentionSelectionPolicy.java       |   48 +
 .../policy/NewestKSelectionPolicy.java          |  149 +++
 .../policy/SelectAfterTimeBasedPolicy.java      |   56 +
 .../data/management/policy/SelectAllPolicy.java |   46 +
 .../policy/SelectBeforeTimeBasedPolicy.java     |   56 +
 .../policy/SelectBetweenTimeBasedPolicy.java    |  110 ++
 .../management/policy/SelectNothingPolicy.java  |   48 +
 .../policy/VersionSelectionPolicy.java          |   43 +
 .../management/retention/DatasetCleaner.java    |  226 ++++
 .../management/retention/RetentionEvents.java   |   39 +
 .../retention/action/AccessControlAction.java   |  168 +++
 .../action/MultiAccessControlAction.java        |  124 ++
 .../retention/action/RetentionAction.java       |  123 ++
 .../retention/dataset/CleanableDataset.java     |   39 +
 .../retention/dataset/CleanableDatasetBase.java |  127 ++
 .../retention/dataset/CleanableHiveDataset.java |  170 +++
 .../dataset/ConfigurableCleanableDataset.java   |  250 ++++
 .../retention/dataset/FsCleanableHelper.java    |  165 +++
 .../dataset/ModificationTimeDataset.java        |   72 ++
 .../MultiVersionCleanableDatasetBase.java       |  378 ++++++
 .../retention/dataset/SnapshotDataset.java      |   74 ++
 .../retention/dataset/TrackingDataset.java      |   75 ++
 .../finder/CleanableHiveDatasetFinder.java      |   53 +
 .../retention/dataset/finder/DatasetFinder.java |   30 +
 .../policy/CombineRetentionPolicy.java          |  198 +++
 .../policy/DeleteAllRetentionPolicy.java        |   43 +
 .../policy/DeleteNothingRetentionPolicy.java    |   47 +
 .../policy/NewestKRetentionPolicy.java          |   92 ++
 .../policy/PredicateRetentionPolicy.java        |   62 +
 .../policy/RawDatasetRetentionPolicy.java       |  104 ++
 .../retention/policy/RetentionPolicy.java       |   50 +
 .../policy/TimeBasedRetentionPolicy.java        |  130 ++
 .../policy/predicates/WhitelistPredicate.java   |   52 +
 .../ConfigBasedCleanabledDatasetFinder.java     |   70 +
 .../profile/ConfigurableGlobDatasetFinder.java  |  166 +++
 .../profile/GlobCleanableDatasetFinder.java     |   46 +
 .../profile/ManagedCleanableDatasetFinder.java  |   70 +
 .../profile/ModificationTimeDatasetProfile.java |   44 +
 .../profile/MultiCleanableDatasetFinder.java    |   68 +
 .../retention/profile/MultiDatasetFinder.java   |  144 +++
 .../profile/ProxyableDatasetProfile.java        |   74 ++
 .../profile/SnapshotDatasetProfile.java         |   48 +
 .../profile/TrackingDatasetProfile.java         |   57 +
 .../retention/version/DatasetVersion.java       |   47 +
 .../version/FileStatusDatasetVersion.java       |   65 +
 .../version/HiveDatasetVersionCleaner.java      |  175 +++
 .../retention/version/StringDatasetVersion.java |   46 +
 .../version/TimestampedDatasetVersion.java      |   60 +
 .../retention/version/VersionCleaner.java       |   63 +
 .../version/finder/DatasetVersionFinder.java    |   42 +
 .../finder/DateTimeDatasetVersionFinder.java    |  106 ++
 .../finder/FileLevelTimestampVersionFinder.java |   54 +
 .../finder/GlobModTimeDatasetVersionFinder.java |   67 +
 .../finder/ModDateTimeDatasetVersionFinder.java |   53 +
 .../version/finder/SingleVersionFinder.java     |   60 +
 .../finder/UnixTimestampVersionFinder.java      |   67 +
 .../retention/version/finder/VersionFinder.java |   30 +
 .../finder/WatermarkDatasetVersionFinder.java   |   81 ++
 .../data/management/trash/AsyncTrash.java       |  157 +++
 .../management/trash/GobblinProxiedTrash.java   |   39 +
 .../data/management/trash/GobblinTrash.java     |   38 +
 .../trash/ImmediateDeletionTrash.java           |   59 +
 .../data/management/trash/MockTrash.java        |   87 ++
 .../trash/NoopSnapshotCleanupPolicy.java        |   36 +
 .../data/management/trash/ProxiedTrash.java     |  166 +++
 .../management/trash/SnapshotCleanupPolicy.java |   38 +
 .../data/management/trash/TestTrash.java        |  247 ++++
 .../trash/TimeBasedSnapshotCleanupPolicy.java   |   46 +
 .../gobblin/data/management/trash/Trash.java    |  301 +++++
 .../data/management/trash/TrashFactory.java     |  110 ++
 .../data/management/version/DatasetVersion.java |   30 +
 .../management/version/FileStatusAware.java     |   33 +
 .../version/FileStatusDatasetVersion.java       |   65 +
 .../FileStatusTimestampedDatasetVersion.java    |   45 +
 .../version/FileSystemDatasetVersion.java       |   36 +
 .../management/version/HiveDatasetVersion.java  |   31 +
 .../version/StringDatasetVersion.java           |   69 +
 .../version/TimestampedDatasetVersion.java      |   74 ++
 .../version/TimestampedHiveDatasetVersion.java  |   51 +
 .../finder/AbstractDatasetVersionFinder.java    |  111 ++
 .../AbstractHiveDatasetVersionFinder.java       |  100 ++
 .../version/finder/DatasetVersionFinder.java    |   66 +
 .../finder/DatePartitionHiveVersionFinder.java  |  116 ++
 .../finder/DateTimeDatasetVersionFinder.java    |  140 ++
 .../finder/FileLevelTimestampVersionFinder.java |   83 ++
 .../finder/GlobModTimeDatasetVersionFinder.java |   76 ++
 .../finder/ModDateTimeDatasetVersionFinder.java |   65 +
 .../version/finder/SingleVersionFinder.java     |   65 +
 .../finder/UnixTimestampVersionFinder.java      |   84 ++
 .../version/finder/VersionFinder.java           |   47 +
 .../finder/WatermarkDatasetVersionFinder.java   |  111 ++
 .../runtime/embedded/EmbeddedGobblinDistcp.java |  132 ++
 .../runtime/retention/DatasetCleanerCli.java    |   89 ++
 .../util/commit/DeleteFileCommitStep.java       |  157 +++
 .../SimpleHiveDatasetTieringPrioritizer.java    |  119 ++
 .../util/test/RetentionTestDataGenerator.java   |  234 ++++
 .../gobblin/util/test/RetentionTestHelper.java  |  119 ++
 .../main/resources/data-management.properties   |    6 +-
 .../management/ConversionHiveTestUtils.java     |   57 -
 .../conversion/hive/BackfillHiveSourceTest.java |   64 -
 .../hive/CopyPartitionParametersTest.java       |   66 -
 .../conversion/hive/HiveSourceTest.java         |  220 ----
 .../hive/LocalHiveMetastoreTestUtils.java       |  180 ---
 .../converter/HiveAvroToOrcConverterTest.java   |  239 ----
 .../hive/converter/HiveSchemaEvolutionTest.java |  299 -----
 .../dataset/ConvertibleHiveDatasetTest.java     |  174 ---
 .../provider/DatePatternUpdateProviderTest.java |   90 --
 .../util/HiveAvroORCQueryGeneratorTest.java     |  380 ------
 .../PartitionLevelWatermarkerTest.java          |  434 -------
 .../watermarker/TableLevelWatermarkerTest.java  |  110 --
 .../copy/ConcurrentBoundedWorkUnitListTest.java |  152 ---
 .../copy/CopySourcePrioritizationTest.java      |  233 ----
 .../data/management/copy/CopySourceTest.java    |  109 --
 .../copy/CopyableDatasetMetadataTest.java       |   62 -
 .../data/management/copy/CopyableFileTest.java  |  243 ----
 .../data/management/copy/CopyableFileUtils.java |   92 --
 .../management/copy/PreserveAttributesTest.java |   86 --
 .../copy/ReadyCopyableFileFilterTest.java       |   59 -
 .../copy/RecursiveCopyableDatasetTest.java      |  339 -----
 .../management/copy/TestCopyableDataset.java    |   86 --
 .../copy/TestCopyableDatasetFinder.java         |   45 -
 .../TestCopyablePartitionableDatasedFinder.java |   44 -
 .../copy/TestCopyablePartitionableDataset.java  |   41 -
 .../copy/TimestampBasedCopyableDatasetTest.java |  344 -----
 .../copy/converter/DecryptConverterTest.java    |  145 ---
 .../copy/converter/UnGzipConverterTest.java     |  113 --
 .../extractor/InputStreamExtractorTest.java     |   66 -
 .../copy/hive/HiveCopyEntityHelperTest.java     |  372 ------
 .../copy/hive/HiveDatasetFinderTest.java        |  238 ----
 .../management/copy/hive/HiveDatasetTest.java   |  166 ---
 .../copy/hive/HiveTargetPathHelperTest.java     |  145 ---
 .../copy/hive/WhitelistBlacklistTest.java       |  173 ---
 .../LookbackPartitionFilterGeneratorTest.java   |   61 -
 .../RegistrationTimeSkipPredicateTest.java      |  114 --
 .../copy/publisher/CopyDataPublisherTest.java   |  311 -----
 .../DeletingCopyDataPublisherTest.java          |  113 --
 .../copy/recovery/RecoveryHelperTest.java       |  159 ---
 .../ConfigBasedDatasetsFinderTest.java          |  108 --
 .../replication/ConfigBasedDatasetsTest.java    |  135 --
 .../replication/CopyRouteGeneratorTest.java     |   85 --
 .../ReplicationConfigurationTest.java           |  221 ----
 .../FileAwareInputStreamDataWriterTest.java     |  265 ----
 .../TarArchiveInputStreamDataWriterTest.java    |  140 --
 .../data/management/partition/FileSetTest.java  |   73 --
 .../policy/NewestKSelectionPolicyTest.java      |  216 ----
 .../policy/TimeBasedSelectionPolicyTest.java    |  167 ---
 .../retention/CleanableDatasetBaseTest.java     |  210 ---
 .../retention/CombineRetentionPolicyTest.java   |  135 --
 .../ConfigurableCleanableDatasetTest.java       |  123 --
 .../retention/DatasetVersionFinderTest.java     |  116 --
 .../retention/FsCleanableHelperTest.java        |  155 ---
 .../retention/NewestKRetentionPolicyTest.java   |   59 -
 .../retention/TimeBasedRetentionPolicyTest.java |   95 --
 .../TimestampedDatasetVersionFinderTest.java    |  100 --
 .../UnixTimestampVersionFinderTest.java         |   50 -
 .../WatermarkDatasetVersionFinderTest.java      |   59 -
 .../retention/action/RetentionActionTest.java   |   49 -
 .../integration/HiveRetentionTest.java          |  221 ----
 .../integration/RetentionIntegrationTest.java   |  132 --
 .../retention/sql/SqlBasedRetentionPoc.java     |  174 ---
 .../data/management/retention/sql/SqlUdfs.java  |   61 -
 .../test/ContainsARetentionPolicy.java          |   34 -
 .../test/ContainsBRetentionPolicy.java          |   34 -
 .../test/ContainsCRetentionPolicy.java          |   34 -
 .../test/ContainsStringRetentionPolicy.java     |   53 -
 .../version/HiveDatasetVersionCleanerTest.java  |   98 --
 .../data/management/trash/AsyncTrashTest.java   |   78 --
 .../data/management/trash/MockTrashTest.java    |   60 -
 .../management/trash/TestCleanupPolicy.java     |   43 -
 .../data/management/trash/TestTrashTest.java    |  100 --
 .../data/management/trash/TrashFactoryTest.java |   91 --
 .../data/management/trash/TrashTest.java        |  244 ----
 .../data/management/trash/TrashTestBase.java    |   77 --
 .../data/management/util/PathUtilsTest.java     |  131 --
 .../DatePartitionedHiveVersionFinderTest.java   |  122 --
 .../embedded/EmbeddedGobblinDistcpTest.java     |   64 -
 ...SimpleHiveDatasetTieringPrioritizerTest.java |   61 -
 .../management/ConversionHiveTestUtils.java     |   57 +
 .../conversion/hive/BackfillHiveSourceTest.java |   64 +
 .../hive/CopyPartitionParametersTest.java       |   66 +
 .../conversion/hive/HiveSourceTest.java         |  220 ++++
 .../hive/LocalHiveMetastoreTestUtils.java       |  180 +++
 .../converter/HiveAvroToOrcConverterTest.java   |  239 ++++
 .../hive/converter/HiveSchemaEvolutionTest.java |  299 +++++
 .../dataset/ConvertibleHiveDatasetTest.java     |  174 +++
 .../provider/DatePatternUpdateProviderTest.java |   90 ++
 .../util/HiveAvroORCQueryGeneratorTest.java     |  380 ++++++
 .../PartitionLevelWatermarkerTest.java          |  434 +++++++
 .../watermarker/TableLevelWatermarkerTest.java  |  110 ++
 .../copy/ConcurrentBoundedWorkUnitListTest.java |  152 +++
 .../copy/CopySourcePrioritizationTest.java      |  233 ++++
 .../data/management/copy/CopySourceTest.java    |  109 ++
 .../copy/CopyableDatasetMetadataTest.java       |   62 +
 .../data/management/copy/CopyableFileTest.java  |  243 ++++
 .../data/management/copy/CopyableFileUtils.java |   92 ++
 .../management/copy/PreserveAttributesTest.java |   86 ++
 .../copy/ReadyCopyableFileFilterTest.java       |   59 +
 .../copy/RecursiveCopyableDatasetTest.java      |  339 +++++
 .../management/copy/TestCopyableDataset.java    |   86 ++
 .../copy/TestCopyableDatasetFinder.java         |   45 +
 .../TestCopyablePartitionableDatasedFinder.java |   44 +
 .../copy/TestCopyablePartitionableDataset.java  |   41 +
 .../copy/TimestampBasedCopyableDatasetTest.java |  344 +++++
 .../copy/converter/DecryptConverterTest.java    |  145 +++
 .../copy/converter/UnGzipConverterTest.java     |  113 ++
 .../extractor/InputStreamExtractorTest.java     |   66 +
 .../copy/hive/HiveCopyEntityHelperTest.java     |  372 ++++++
 .../copy/hive/HiveDatasetFinderTest.java        |  238 ++++
 .../management/copy/hive/HiveDatasetTest.java   |  166 +++
 .../copy/hive/HiveTargetPathHelperTest.java     |  145 +++
 .../copy/hive/WhitelistBlacklistTest.java       |  173 +++
 .../LookbackPartitionFilterGeneratorTest.java   |   61 +
 .../RegistrationTimeSkipPredicateTest.java      |  114 ++
 .../copy/publisher/CopyDataPublisherTest.java   |  311 +++++
 .../DeletingCopyDataPublisherTest.java          |  113 ++
 .../copy/recovery/RecoveryHelperTest.java       |  159 +++
 .../ConfigBasedDatasetsFinderTest.java          |  108 ++
 .../replication/ConfigBasedDatasetsTest.java    |  135 ++
 .../replication/CopyRouteGeneratorTest.java     |   85 ++
 .../ReplicationConfigurationTest.java           |  221 ++++
 .../FileAwareInputStreamDataWriterTest.java     |  265 ++++
 .../TarArchiveInputStreamDataWriterTest.java    |  140 ++
 .../data/management/partition/FileSetTest.java  |   73 ++
 .../policy/NewestKSelectionPolicyTest.java      |  216 ++++
 .../policy/TimeBasedSelectionPolicyTest.java    |  167 +++
 .../retention/CleanableDatasetBaseTest.java     |  210 +++
 .../retention/CombineRetentionPolicyTest.java   |  135 ++
 .../ConfigurableCleanableDatasetTest.java       |  123 ++
 .../retention/DatasetVersionFinderTest.java     |  116 ++
 .../retention/FsCleanableHelperTest.java        |  155 +++
 .../retention/NewestKRetentionPolicyTest.java   |   59 +
 .../retention/TimeBasedRetentionPolicyTest.java |   95 ++
 .../TimestampedDatasetVersionFinderTest.java    |  100 ++
 .../UnixTimestampVersionFinderTest.java         |   50 +
 .../WatermarkDatasetVersionFinderTest.java      |   59 +
 .../retention/action/RetentionActionTest.java   |   49 +
 .../integration/HiveRetentionTest.java          |  221 ++++
 .../integration/RetentionIntegrationTest.java   |  132 ++
 .../retention/sql/SqlBasedRetentionPoc.java     |  174 +++
 .../data/management/retention/sql/SqlUdfs.java  |   61 +
 .../test/ContainsARetentionPolicy.java          |   34 +
 .../test/ContainsBRetentionPolicy.java          |   34 +
 .../test/ContainsCRetentionPolicy.java          |   34 +
 .../test/ContainsStringRetentionPolicy.java     |   53 +
 .../version/HiveDatasetVersionCleanerTest.java  |   98 ++
 .../data/management/trash/AsyncTrashTest.java   |   78 ++
 .../data/management/trash/MockTrashTest.java    |   60 +
 .../management/trash/TestCleanupPolicy.java     |   43 +
 .../data/management/trash/TestTrashTest.java    |  100 ++
 .../data/management/trash/TrashFactoryTest.java |   91 ++
 .../data/management/trash/TrashTest.java        |  244 ++++
 .../data/management/trash/TrashTestBase.java    |   77 ++
 .../data/management/util/PathUtilsTest.java     |  131 ++
 .../DatePartitionedHiveVersionFinderTest.java   |  122 ++
 .../embedded/EmbeddedGobblinDistcpTest.java     |   64 +
 ...SimpleHiveDatasetTieringPrioritizerTest.java |   61 +
 .../validCompleteDataset.conf                   |   28 +-
 .../validCompleteDataset_PullMode2.conf         |   30 +-
 .../validCompleteDataset_PushMode.conf          |   28 +-
 .../validCompleteDataset_PushMode2.conf         |   28 +-
 .../testCombinePolicy/retention.job             |   10 +-
 .../testCombinePolicy/selection.conf            |   10 +-
 .../daily-retention.job                         |    4 +-
 .../hive-retention.job                          |    6 +-
 .../testHiveTimeBasedRetention/replacement.conf |    6 +-
 .../testHiveTimeBasedRetention/selection.conf   |    6 +-
 .../hourly-retention.job                        |    4 +-
 .../daily-retention-with-accessControl.conf     |    8 +-
 .../daily-hourly-retention.conf                 |    6 +-
 .../testNewestKRetention/retention.conf         |    6 +-
 .../testNewestKRetention/selection.conf         |    6 +-
 .../testTimeBasedAccessControl/selection.conf   |    6 +-
 .../testTimeBasedRetention/retention.conf       |    6 +-
 .../testTimeBasedRetention/selection.conf       |    6 +-
 .../ubuntu-gobblin-0.10.0/Dockerfile            |    2 +-
 .../ubuntu-gobblin-0.7.0/Dockerfile             |    2 +-
 .../ubuntu-gobblin-0.8.0/Dockerfile             |    2 +-
 .../ubuntu-gobblin-0.9.0/Dockerfile             |    2 +-
 .../ubuntu-gobblin-0.10.0/Dockerfile            |    2 +-
 .../ubuntu-gobblin-0.7.0/Dockerfile             |    2 +-
 .../ubuntu-gobblin-0.8.0/Dockerfile             |    2 +-
 .../ubuntu-gobblin-0.9.0/Dockerfile             |    2 +-
 .../adaptors/Hive-Avro-To-ORC-Converter.md      |   14 +-
 gobblin-docs/case-studies/Hive-Distcp.md        |   12 +-
 .../case-studies/Kafka-HDFS-Ingestion.md        |   16 +-
 gobblin-docs/data-management/DistcpNgEvents.md  |    4 +-
 .../data-management/Gobblin-Retention.md        |   46 +-
 .../metrics/Implementing-New-Reporters.md       |    2 +-
 gobblin-docs/sinks/AvroHdfsDataWriter.md        |    2 +-
 gobblin-docs/sinks/ConsoleWriter.md             |    2 +-
 gobblin-docs/sinks/Gobblin-JDBC-Writer.md       |   12 +-
 gobblin-docs/sinks/Kafka.md                     |    4 +-
 gobblin-docs/sinks/SimpleBytesWriter.md         |    2 +-
 gobblin-docs/user-guide/Compaction.md           |    6 +-
 .../Configuration-Properties-Glossary.md        |    4 +-
 gobblin-docs/user-guide/Gobblin-on-Yarn.md      |    4 +-
 gobblin-docs/user-guide/Gobblin-template.md     |   10 +-
 .../user-guide/Working-with-the-ForkOperator.md |   12 +-
 .../example/simplejson/SimpleJsonConverter.java |   75 --
 .../example/simplejson/SimpleJsonExtractor.java |  125 --
 .../example/simplejson/SimpleJsonSource.java    |   81 --
 .../wikipedia/EmbeddedWikipediaExample.java     |  104 --
 .../example/wikipedia/WikipediaConverter.java   |   77 --
 .../example/wikipedia/WikipediaExtractor.java   |  437 -------
 .../example/wikipedia/WikipediaPartitioner.java |   52 -
 .../example/wikipedia/WikipediaSource.java      |  115 --
 .../example/simplejson/SimpleJsonConverter.java |   75 ++
 .../example/simplejson/SimpleJsonExtractor.java |  125 ++
 .../example/simplejson/SimpleJsonSource.java    |   81 ++
 .../wikipedia/EmbeddedWikipediaExample.java     |  104 ++
 .../example/wikipedia/WikipediaConverter.java   |   77 ++
 .../example/wikipedia/WikipediaExtractor.java   |  437 +++++++
 .../example/wikipedia/WikipediaPartitioner.java |   52 +
 .../example/wikipedia/WikipediaSource.java      |  115 ++
 .../src/main/resources/avro-eventhub.job        |    8 +-
 .../src/main/resources/avro-to-mysql.pull       |    8 +-
 .../src/main/resources/csvToAvro.conf           |    8 +-
 .../src/main/resources/distcp-hive.pull         |   14 +-
 .../src/main/resources/distcpFromS3.job         |   18 +-
 .../src/main/resources/distcpToS3.job           |   18 +-
 ...oogle-analytics-unsampledreport-to-avro.pull |    8 +-
 .../src/main/resources/google-web-master.pull   |    8 +-
 .../resources/hdfs-monthly-to-hdfs-daily.pull   |   10 +-
 .../src/main/resources/hive-avro-to-orc.pull    |   10 +-
 .../src/main/resources/kafka-console.pull       |   12 +-
 .../src/main/resources/simplejson.pull          |   12 +-
 .../main/resources/streaming-kafka-console.pull |    6 +-
 .../main/resources/streaming-kafka-kafka.pull   |   10 +-
 .../main/resources/streaming-test-kafka.pull    |   12 +-
 .../src/main/resources/wikipedia-console.pull   |   10 +-
 .../src/main/resources/wikipedia-kafka.pull     |   14 +-
 .../src/main/resources/wikipedia-orc.pull       |    8 +-
 .../src/main/resources/wikipedia.pull           |   10 +-
 .../src/main/resources/wikipedia.template       |    8 +-
 .../main/java/gobblin/hive/HiveConstants.java   |   61 -
 .../src/main/java/gobblin/hive/HiveLock.java    |   66 -
 .../hive/HiveMetaStoreClientFactory.java        |  118 --
 .../gobblin/hive/HiveMetastoreClientPool.java   |  184 ---
 .../main/java/gobblin/hive/HivePartition.java   |   73 --
 .../gobblin/hive/HivePartitionComparator.java   |   50 -
 .../main/java/gobblin/hive/HiveRegProps.java    |  153 ---
 .../main/java/gobblin/hive/HiveRegister.java    |  384 ------
 .../java/gobblin/hive/HiveRegisterStep.java     |  112 --
 .../java/gobblin/hive/HiveRegisterUtils.java    |   55 -
 .../java/gobblin/hive/HiveRegistrationUnit.java |  462 -------
 .../hive/HiveRegistrationUnitComparator.java    |  211 ---
 .../java/gobblin/hive/HiveSerDeManager.java     |  123 --
 .../java/gobblin/hive/HiveSerDeWrapper.java     |  191 ---
 .../src/main/java/gobblin/hive/HiveTable.java   |  120 --
 .../java/gobblin/hive/HiveTableComparator.java  |   53 -
 .../gobblin/hive/PartitionDeregisterStep.java   |   65 -
 .../java/gobblin/hive/TableDeregisterStep.java  |   58 -
 .../gobblin/hive/avro/HiveAvroSerDeManager.java |  196 ---
 .../metastore/HiveMetaStoreBasedRegister.java   |  608 ---------
 .../metastore/HiveMetaStoreEventHelper.java     |  189 ---
 .../hive/metastore/HiveMetaStoreUtils.java      |  429 ------
 .../hive/policy/HiveRegistrationPolicy.java     |   43 -
 .../hive/policy/HiveRegistrationPolicyBase.java |  415 ------
 .../policy/HiveSnapshotRegistrationPolicy.java  |  133 --
 .../main/java/gobblin/hive/spec/HiveSpec.java   |   52 -
 .../hive/spec/HiveSpecWithPostActivities.java   |   36 -
 .../hive/spec/HiveSpecWithPreActivities.java    |   36 -
 .../hive/spec/HiveSpecWithPredicates.java       |   41 -
 .../java/gobblin/hive/spec/SimpleHiveSpec.java  |  120 --
 .../gobblin/hive/spec/activity/Activity.java    |   39 -
 .../spec/activity/DropPartitionActivity.java    |   47 -
 .../hive/spec/activity/DropTableActivity.java   |   43 -
 .../predicate/PartitionNotExistPredicate.java   |   52 -
 .../spec/predicate/TableNotExistPredicate.java  |   49 -
 .../org/apache/gobblin/hive/HiveConstants.java  |   61 +
 .../java/org/apache/gobblin/hive/HiveLock.java  |   66 +
 .../hive/HiveMetaStoreClientFactory.java        |  118 ++
 .../gobblin/hive/HiveMetastoreClientPool.java   |  184 +++
 .../org/apache/gobblin/hive/HivePartition.java  |   73 ++
 .../gobblin/hive/HivePartitionComparator.java   |   50 +
 .../org/apache/gobblin/hive/HiveRegProps.java   |  153 +++
 .../org/apache/gobblin/hive/HiveRegister.java   |  384 ++++++
 .../apache/gobblin/hive/HiveRegisterStep.java   |  112 ++
 .../apache/gobblin/hive/HiveRegisterUtils.java  |   55 +
 .../gobblin/hive/HiveRegistrationUnit.java      |  462 +++++++
 .../hive/HiveRegistrationUnitComparator.java    |  211 +++
 .../apache/gobblin/hive/HiveSerDeManager.java   |  123 ++
 .../apache/gobblin/hive/HiveSerDeWrapper.java   |  191 +++
 .../java/org/apache/gobblin/hive/HiveTable.java |  120 ++
 .../gobblin/hive/HiveTableComparator.java       |   53 +
 .../gobblin/hive/PartitionDeregisterStep.java   |   65 +
 .../gobblin/hive/TableDeregisterStep.java       |   58 +
 .../gobblin/hive/avro/HiveAvroSerDeManager.java |  196 +++
 .../metastore/HiveMetaStoreBasedRegister.java   |  608 +++++++++
 .../metastore/HiveMetaStoreEventHelper.java     |  189 +++
 .../hive/metastore/HiveMetaStoreUtils.java      |  429 ++++++
 .../hive/policy/HiveRegistrationPolicy.java     |   43 +
 .../hive/policy/HiveRegistrationPolicyBase.java |  415 ++++++
 .../policy/HiveSnapshotRegistrationPolicy.java  |  133 ++
 .../org/apache/gobblin/hive/spec/HiveSpec.java  |   52 +
 .../hive/spec/HiveSpecWithPostActivities.java   |   36 +
 .../hive/spec/HiveSpecWithPreActivities.java    |   36 +
 .../hive/spec/HiveSpecWithPredicates.java       |   41 +
 .../gobblin/hive/spec/SimpleHiveSpec.java       |  120 ++
 .../gobblin/hive/spec/activity/Activity.java    |   39 +
 .../spec/activity/DropPartitionActivity.java    |   47 +
 .../hive/spec/activity/DropTableActivity.java   |   43 +
 .../predicate/PartitionNotExistPredicate.java   |   52 +
 .../spec/predicate/TableNotExistPredicate.java  |   49 +
 .../hive/HiveMetaStoreClientFactoryTest.java    |   30 -
 .../HiveRegistrationUnitComparatorTest.java     |   63 -
 .../hive/metastore/HiveMetaStoreUtilsTest.java  |  114 --
 .../policy/HiveRegistrationPolicyBaseTest.java  |  109 --
 .../hive/HiveMetaStoreClientFactoryTest.java    |   30 +
 .../HiveRegistrationUnitComparatorTest.java     |   63 +
 .../hive/metastore/HiveMetaStoreUtilsTest.java  |  114 ++
 .../policy/HiveRegistrationPolicyBaseTest.java  |  109 ++
 .../metastore/DatabaseJobHistoryStore.java      |  146 ---
 .../gobblin/metastore/DatasetStateStore.java    |   43 -
 .../java/gobblin/metastore/FsStateStore.java    |  348 -----
 .../gobblin/metastore/FsStateStoreFactory.java  |   53 -
 .../metastore/JobHistoryDataSourceProvider.java |   46 -
 .../java/gobblin/metastore/JobHistoryStore.java |   53 -
 .../java/gobblin/metastore/MetaStoreModule.java |   46 -
 .../java/gobblin/metastore/MysqlStateStore.java |  413 ------
 .../metastore/MysqlStateStoreFactory.java       |   43 -
 .../main/java/gobblin/metastore/StateStore.java |  196 ---
 .../database/DatabaseJobHistoryStoreV100.java   |  915 -------------
 .../database/DatabaseJobHistoryStoreV101.java   | 1008 ---------------
 .../database/DatabaseJobHistoryStoreV102.java   |   46 -
 .../java/gobblin/metastore/database/Filter.java |   69 -
 .../database/SupportedDatabaseVersion.java      |   41 -
 .../VersionedDatabaseJobHistoryStore.java       |   34 -
 .../DatasetUrnStateStoreNameParser.java         |   47 -
 .../GuidDatasetUrnStateStoreNameParser.java     |  157 ---
 .../SimpleDatasetUrnStateStoreNameParser.java   |   45 -
 .../DatabaseJobHistoryStoreSchemaManager.java   |  170 ---
 .../gobblin/metastore/util/MySqlJdbcUrl.java    |   86 --
 .../metastore/util/StateStoreCleaner.java       |  209 ---
 .../metastore/DatabaseJobHistoryStore.java      |  146 +++
 .../gobblin/metastore/DatasetStateStore.java    |   43 +
 .../apache/gobblin/metastore/FsStateStore.java  |  348 +++++
 .../gobblin/metastore/FsStateStoreFactory.java  |   53 +
 .../metastore/JobHistoryDataSourceProvider.java |   46 +
 .../gobblin/metastore/JobHistoryStore.java      |   53 +
 .../gobblin/metastore/MetaStoreModule.java      |   46 +
 .../gobblin/metastore/MysqlStateStore.java      |  413 ++++++
 .../metastore/MysqlStateStoreFactory.java       |   43 +
 .../apache/gobblin/metastore/StateStore.java    |  196 +++
 .../database/DatabaseJobHistoryStoreV100.java   |  915 +++++++++++++
 .../database/DatabaseJobHistoryStoreV101.java   | 1008 +++++++++++++++
 .../database/DatabaseJobHistoryStoreV102.java   |   46 +
 .../gobblin/metastore/database/Filter.java      |   69 +
 .../database/SupportedDatabaseVersion.java      |   41 +
 .../VersionedDatabaseJobHistoryStore.java       |   34 +
 .../DatasetUrnStateStoreNameParser.java         |   47 +
 .../GuidDatasetUrnStateStoreNameParser.java     |  157 +++
 .../SimpleDatasetUrnStateStoreNameParser.java   |   45 +
 .../DatabaseJobHistoryStoreSchemaManager.java   |  170 +++
 .../gobblin/metastore/util/MySqlJdbcUrl.java    |   86 ++
 .../metastore/util/StateStoreCleaner.java       |  209 +++
 .../metastore/DatabaseJobHistoryStoreTest.java  |  321 -----
 .../DatabaseJobHistoryStoreV100Test.java        |   34 -
 .../DatabaseJobHistoryStoreV101Test.java        |   34 -
 .../gobblin/metastore/FsStateStoreTest.java     |  149 ---
 .../GuidDatasetUrnStateStoreNameParserTest.java |   79 --
 .../testing/ITestMetastoreDatabase.java         |   28 -
 .../metastore/testing/TestMetadataDatabase.java |   71 -
 .../testing/TestMetastoreDatabaseFactory.java   |  158 ---
 .../testing/TestMetastoreDatabaseServer.java    |  218 ----
 .../metastore/DatabaseJobHistoryStoreTest.java  |  321 +++++
 .../DatabaseJobHistoryStoreV100Test.java        |   34 +
 .../DatabaseJobHistoryStoreV101Test.java        |   34 +
 .../gobblin/metastore/FsStateStoreTest.java     |  151 +++
 .../GuidDatasetUrnStateStoreNameParserTest.java |   79 ++
 .../testing/ITestMetastoreDatabase.java         |   28 +
 .../metastore/testing/TestMetadataDatabase.java |   71 +
 .../testing/TestMetastoreDatabaseFactory.java   |  158 +++
 .../testing/TestMetastoreDatabaseServer.java    |  218 ++++
 .../src/main/avro/FlatGobblinMetric.avsc        |    2 +-
 .../src/main/avro/GobblinTrackingEvent.avsc     |    2 +-
 .../src/main/avro/MetricReport.avsc             |    2 +-
 .../gobblin/metrics/FlatGobblinMetric.java      |  433 -------
 .../gobblin/metrics/GobblinTrackingEvent.java   |  434 -------
 .../main/gen-java/gobblin/metrics/Metric.java   |  280 ----
 .../gen-java/gobblin/metrics/MetricReport.java  |  357 -----
 .../MetricsFileSystemInstrumentation.java       |  339 -----
 .../metrics/ConsoleEventReporterFactory.java    |   45 -
 .../gobblin/metrics/ConsoleReporterFactory.java |   39 -
 .../gobblin/metrics/ContextAwareCounter.java    |   62 -
 .../java/gobblin/metrics/ContextAwareGauge.java |   59 -
 .../gobblin/metrics/ContextAwareHistogram.java  |   66 -
 .../java/gobblin/metrics/ContextAwareMeter.java |   63 -
 .../gobblin/metrics/ContextAwareMetric.java     |   45 -
 .../metrics/ContextAwareMetricFactory.java      |  125 --
 .../java/gobblin/metrics/ContextAwareTimer.java |   62 -
 .../src/main/java/gobblin/metrics/Counters.java |   80 --
 .../metrics/CustomCodahaleReporterFactory.java  |   40 -
 .../gobblin/metrics/CustomReporterFactory.java  |   38 -
 .../main/java/gobblin/metrics/InnerCounter.java |   75 --
 .../main/java/gobblin/metrics/InnerGauge.java   |   55 -
 .../java/gobblin/metrics/InnerHistogram.java    |   74 --
 .../main/java/gobblin/metrics/InnerMeter.java   |   64 -
 .../gobblin/metrics/InnerMetricContext.java     |  398 ------
 .../main/java/gobblin/metrics/InnerTimer.java   |   66 -
 .../main/java/gobblin/metrics/Measurements.java |   58 -
 .../java/gobblin/metrics/MetricContext.java     |  732 -----------
 .../main/java/gobblin/metrics/MetricNames.java  |  115 --
 .../java/gobblin/metrics/RootMetricContext.java |  201 ---
 .../src/main/java/gobblin/metrics/Tag.java      |  137 --
 .../gobblin/metrics/TagBasedMetricFilter.java   |   46 -
 .../src/main/java/gobblin/metrics/Taggable.java |   74 --
 .../src/main/java/gobblin/metrics/Tagged.java   |   88 --
 .../metrics/broker/MetricContextFactory.java    |  107 --
 .../metrics/broker/MetricContextKey.java        |   33 -
 .../broker/SubTaggedMetricContextKey.java       |   97 --
 .../metrics/context/ContextWeakReference.java   |   43 -
 .../metrics/context/NameConflictException.java  |   39 -
 .../metrics/context/ReportableContext.java      |  153 ---
 .../context/filter/AllContextFilter.java        |   54 -
 .../metrics/context/filter/ContextFilter.java   |   56 -
 .../context/filter/ContextFilterFactory.java    |   63 -
 .../java/gobblin/metrics/event/EventName.java   |   66 -
 .../gobblin/metrics/event/EventSubmitter.java   |  162 ---
 .../java/gobblin/metrics/event/JobEvent.java    |   46 -
 .../gobblin/metrics/event/MultiPartEvent.java   |   94 --
 .../gobblin/metrics/event/MultiTimingEvent.java |  150 ---
 .../java/gobblin/metrics/event/TaskEvent.java   |   41 -
 .../java/gobblin/metrics/event/TimingEvent.java |  103 --
 .../gobblin/metrics/event/sla/SlaEventKeys.java |   38 -
 .../metrics/event/sla/SlaEventSubmitter.java    |  170 ---
 .../metrics/example/ReporterExampleBase.java    |  184 ---
 .../gobblin/metrics/metric/InnerMetric.java     |   36 -
 .../java/gobblin/metrics/metric/Metrics.java    |   47 -
 .../gobblin/metrics/metric/ProxyMetric.java     |   33 -
 .../metrics/metric/filter/MetricFilters.java    |   38 -
 .../metric/filter/MetricNameRegexFilter.java    |   45 -
 .../metrics/metric/filter/MetricTypeFilter.java |   69 -
 .../metrics/notification/EventNotification.java |   36 -
 .../MetricContextCleanupNotification.java       |   33 -
 .../NewMetricContextNotification.java           |   36 -
 .../metrics/notification/Notification.java      |   25 -
 .../reporter/ConfiguredScheduledReporter.java   |  252 ----
 .../metrics/reporter/ContextAwareReporter.java  |  199 ---
 .../reporter/ContextAwareScheduledReporter.java |  179 ---
 .../gobblin/metrics/reporter/EventReporter.java |  227 ----
 .../metrics/reporter/MetricReportReporter.java  |  246 ----
 .../reporter/OutputStreamEventReporter.java     |  163 ---
 .../metrics/reporter/OutputStreamReporter.java  |  352 -----
 .../RecursiveScheduledMetricReporter.java       |   72 --
 .../reporter/RecursiveScheduledReporter.java    |   80 --
 .../metrics/reporter/ScheduledReporter.java     |  239 ----
 .../reporter/util/AvroBinarySerializer.java     |   41 -
 .../reporter/util/AvroJsonSerializer.java       |   43 -
 .../metrics/reporter/util/AvroSerializer.java   |  110 --
 .../metrics/reporter/util/EventUtils.java       |  113 --
 .../reporter/util/FixedSchemaVersionWriter.java |   47 -
 .../reporter/util/MetricReportUtils.java        |  113 --
 .../reporter/util/NoopSchemaVersionWriter.java  |   45 -
 .../reporter/util/SchemaVersionWriter.java      |   57 -
 .../metrics/test/ContextStoreReporter.java      |   68 -
 .../gobblin/metrics/test/MetricsAssert.java     |  109 --
 .../gobblin/metrics/test/TestConstants.java     |   41 -
 .../gobblin/metrics/test/TimestampedValue.java  |   42 -
 .../MetricsFileSystemInstrumentation.java       |  339 +++++
 .../metrics/ConsoleEventReporterFactory.java    |   45 +
 .../gobblin/metrics/ConsoleReporterFactory.java |   39 +
 .../gobblin/metrics/ContextAwareCounter.java    |   62 +
 .../gobblin/metrics/ContextAwareGauge.java      |   59 +
 .../gobblin/metrics/ContextAwareHistogram.java  |   66 +
 .../gobblin/metrics/ContextAwareMeter.java      |   63 +
 .../gobblin/metrics/ContextAwareMetric.java     |   45 +
 .../metrics/ContextAwareMetricFactory.java      |  125 ++
 .../gobblin/metrics/ContextAwareTimer.java      |   62 +
 .../org/apache/gobblin/metrics/Counters.java    |   80 ++
 .../metrics/CustomCodahaleReporterFactory.java  |   40 +
 .../gobblin/metrics/CustomReporterFactory.java  |   38 +
 .../apache/gobblin/metrics/InnerCounter.java    |   75 ++
 .../org/apache/gobblin/metrics/InnerGauge.java  |   55 +
 .../apache/gobblin/metrics/InnerHistogram.java  |   74 ++
 .../org/apache/gobblin/metrics/InnerMeter.java  |   64 +
 .../gobblin/metrics/InnerMetricContext.java     |  398 ++++++
 .../org/apache/gobblin/metrics/InnerTimer.java  |   66 +
 .../apache/gobblin/metrics/Measurements.java    |   58 +
 .../apache/gobblin/metrics/MetricContext.java   |  732 +++++++++++
 .../org/apache/gobblin/metrics/MetricNames.java |  115 ++
 .../gobblin/metrics/RootMetricContext.java      |  201 +++
 .../java/org/apache/gobblin/metrics/Tag.java    |  137 ++
 .../gobblin/metrics/TagBasedMetricFilter.java   |   46 +
 .../org/apache/gobblin/metrics/Taggable.java    |   74 ++
 .../java/org/apache/gobblin/metrics/Tagged.java |   88 ++
 .../metrics/broker/MetricContextFactory.java    |  107 ++
 .../metrics/broker/MetricContextKey.java        |   33 +
 .../broker/SubTaggedMetricContextKey.java       |   97 ++
 .../metrics/context/ContextWeakReference.java   |   43 +
 .../metrics/context/NameConflictException.java  |   39 +
 .../metrics/context/ReportableContext.java      |  153 +++
 .../context/filter/AllContextFilter.java        |   54 +
 .../metrics/context/filter/ContextFilter.java   |   56 +
 .../context/filter/ContextFilterFactory.java    |   63 +
 .../apache/gobblin/metrics/event/EventName.java |   66 +
 .../gobblin/metrics/event/EventSubmitter.java   |  162 +++
 .../apache/gobblin/metrics/event/JobEvent.java  |   46 +
 .../gobblin/metrics/event/MultiPartEvent.java   |   94 ++
 .../gobblin/metrics/event/MultiTimingEvent.java |  150 +++
 .../apache/gobblin/metrics/event/TaskEvent.java |   41 +
 .../gobblin/metrics/event/TimingEvent.java      |  103 ++
 .../gobblin/metrics/event/sla/SlaEventKeys.java |   38 +
 .../metrics/event/sla/SlaEventSubmitter.java    |  170 +++
 .../metrics/example/ReporterExampleBase.java    |  184 +++
 .../gobblin/metrics/metric/InnerMetric.java     |   36 +
 .../apache/gobblin/metrics/metric/Metrics.java  |   47 +
 .../gobblin/metrics/metric/ProxyMetric.java     |   33 +
 .../metrics/metric/filter/MetricFilters.java    |   38 +
 .../metric/filter/MetricNameRegexFilter.java    |   45 +
 .../metrics/metric/filter/MetricTypeFilter.java |   69 +
 .../metrics/notification/EventNotification.java |   36 +
 .../MetricContextCleanupNotification.java       |   33 +
 .../NewMetricContextNotification.java           |   36 +
 .../metrics/notification/Notification.java      |   25 +
 .../reporter/ConfiguredScheduledReporter.java   |  252 ++++
 .../metrics/reporter/ContextAwareReporter.java  |  199 +++
 .../reporter/ContextAwareScheduledReporter.java |  179 +++
 .../gobblin/metrics/reporter/EventReporter.java |  227 ++++
 .../metrics/reporter/MetricReportReporter.java  |  246 ++++
 .../reporter/OutputStreamEventReporter.java     |  163 +++
 .../metrics/reporter/OutputStreamReporter.java  |  352 +++++
 .../RecursiveScheduledMetricReporter.java       |   72 ++
 .../reporter/RecursiveScheduledReporter.java    |   80 ++
 .../metrics/reporter/ScheduledReporter.java     |  239 ++++
 .../reporter/util/AvroBinarySerializer.java     |   41 +
 .../reporter/util/AvroJsonSerializer.java       |   43 +
 .../metrics/reporter/util/AvroSerializer.java   |  110 ++
 .../metrics/reporter/util/EventUtils.java       |  113 ++
 .../reporter/util/FixedSchemaVersionWriter.java |   47 +
 .../reporter/util/MetricReportUtils.java        |  113 ++
 .../reporter/util/NoopSchemaVersionWriter.java  |   45 +
 .../reporter/util/SchemaVersionWriter.java      |   57 +
 .../metrics/test/ContextStoreReporter.java      |   68 +
 .../gobblin/metrics/test/MetricsAssert.java     |  109 ++
 .../gobblin/metrics/test/TestConstants.java     |   41 +
 .../gobblin/metrics/test/TimestampedValue.java  |   42 +
 ....filesystem.FileSystemInstrumentationFactory |   18 -
 ....filesystem.FileSystemInstrumentationFactory |   18 +
 .../MetricsFileSystemInstrumentationTest.java   |  591 ---------
 .../metrics/ContextAwareMetricFactoryTest.java  |   99 --
 .../java/gobblin/metrics/MetricContextTest.java |  407 ------
 .../metrics/OutputStreamReporterTest.java       |  184 ---
 .../gobblin/metrics/RootMetricContextTest.java  |  226 ----
 .../src/test/java/gobblin/metrics/TagTest.java  |   47 -
 .../test/java/gobblin/metrics/TaggedTest.java   |   73 --
 .../broker/MetricContextFactoryTest.java        |  103 --
 .../metrics/callback/NotificationStore.java     |   52 -
 .../metric/filter/MetricFiltersTest.java        |   50 -
 .../filter/MetricNameRegexFilterTest.java       |   46 -
 .../metric/filter/MetricTypeFilterTest.java     |   67 -
 .../metrics/performance/Incrementer.java        |   72 --
 .../performance/MetricsPerformanceTest.java     |   62 -
 .../metrics/performance/MetricsUpdater.java     |  129 --
 .../metrics/performance/PerformanceUtils.java   |  213 ---
 .../MetricsFileSystemInstrumentationTest.java   |  591 +++++++++
 .../metrics/ContextAwareMetricFactoryTest.java  |   99 ++
 .../gobblin/metrics/MetricContextTest.java      |  407 ++++++
 .../metrics/OutputStreamReporterTest.java       |  184 +++
 .../gobblin/metrics/RootMetricContextTest.java  |  226 ++++
 .../org/apache/gobblin/metrics/TagTest.java     |   47 +
 .../org/apache/gobblin/metrics/TaggedTest.java  |   73 ++
 .../broker/MetricContextFactoryTest.java        |  103 ++
 .../metrics/callback/NotificationStore.java     |   52 +
 .../metric/filter/MetricFiltersTest.java        |   50 +
 .../filter/MetricNameRegexFilterTest.java       |   46 +
 .../metric/filter/MetricTypeFilterTest.java     |   67 +
 .../metrics/performance/Incrementer.java        |   72 ++
 .../performance/MetricsPerformanceTest.java     |   62 +
 .../metrics/performance/MetricsUpdater.java     |  129 ++
 .../metrics/performance/PerformanceUtils.java   |  213 +++
 .../src/test/resources/performance-testng.xml   |    2 +-
 .../src/main/java/gobblin/MetricsHelper.java    |   51 -
 .../java/gobblin/metrics/GobblinMetrics.java    |  727 -----------
 .../gobblin/metrics/GobblinMetricsRegistry.java |  152 ---
 .../java/org/apache/gobblin/MetricsHelper.java  |   51 +
 .../apache/gobblin/metrics/GobblinMetrics.java  |  727 +++++++++++
 .../gobblin/metrics/GobblinMetricsRegistry.java |  152 +++
 .../metrics/reporter/PrefixContextFilter.java   |   73 --
 .../metrics/reporter/ScheduledReporterTest.java |  173 ---
 .../metrics/reporter/PrefixContextFilter.java   |   73 ++
 .../metrics/reporter/ScheduledReporterTest.java |  173 +++
 .../avro/AvroToJsonBytesConverter.java          |   24 -
 .../AvroToJsonBytesWithMetadataConverter.java   |   56 -
 .../converter/avro/AvroToJsonConverter.java     |   74 --
 .../AvroToJsonRecordWithMetadataConverter.java  |   68 -
 .../avro/AvroToJsonStringConverter.java         |   27 -
 .../avro/AvroToJsonStringConverterBase.java     |   96 --
 .../avro/AvroToJsonBytesConverter.java          |   24 +
 .../AvroToJsonBytesWithMetadataConverter.java   |   56 +
 .../converter/avro/AvroToJsonConverter.java     |   74 ++
 .../AvroToJsonRecordWithMetadataConverter.java  |   68 +
 .../avro/AvroToJsonStringConverter.java         |   27 +
 .../avro/AvroToJsonStringConverterBase.java     |   96 ++
 ...roToJsonRecordWithMetadataConverterTest.java |   38 -
 .../avro/AvroToJsonStringConverterTest.java     |   63 -
 ...roToJsonRecordWithMetadataConverterTest.java |   38 +
 .../avro/AvroToJsonStringConverterTest.java     |   63 +
 .../azkaban/AzkabanCompactionJobLauncher.java   |   91 --
 .../gobblin/azkaban/AzkabanGobblinDaemon.java   |   63 -
 .../azkaban/AzkabanGobblinYarnAppLauncher.java  |   90 --
 .../azkaban/AzkabanIntegrationTestLauncher.java |   66 -
 .../gobblin/azkaban/AzkabanJobLauncher.java     |  307 -----
 .../azkaban/AzkabanStateStoreCleanerJob.java    |   51 -
 .../main/java/gobblin/azkaban/AzkabanTags.java  |   27 -
 .../hive/validation/ValidationJob.java          |  739 -----------
 .../retention/Avro2OrcStaleDatasetCleaner.java  |  128 --
 .../management/retention/DatasetCleanerJob.java |   81 --
 .../management/trash/TrashCollectorJob.java     |  100 --
 .../azkaban/AzkabanCompactionJobLauncher.java   |   91 ++
 .../gobblin/azkaban/AzkabanGobblinDaemon.java   |   63 +
 .../azkaban/AzkabanGobblinYarnAppLauncher.java  |   90 ++
 .../azkaban/AzkabanIntegrationTestLauncher.java |   66 +
 .../gobblin/azkaban/AzkabanJobLauncher.java     |  307 +++++
 .../azkaban/AzkabanStateStoreCleanerJob.java    |   51 +
 .../org/apache/gobblin/azkaban/AzkabanTags.java |   27 +
 .../hive/validation/ValidationJob.java          |  739 +++++++++++
 .../retention/Avro2OrcStaleDatasetCleaner.java  |  128 ++
 .../management/retention/DatasetCleanerJob.java |   81 ++
 .../management/trash/TrashCollectorJob.java     |  100 ++
 .../main/java/gobblin/codec/Base64Codec.java    |  126 --
 .../src/main/java/gobblin/codec/GzipCodec.java  |   50 -
 .../org/apache/gobblin/codec/Base64Codec.java   |  126 ++
 .../org/apache/gobblin/codec/GzipCodec.java     |   50 +
 .../compliance/ComplianceConfigurationKeys.java |  142 --
 .../gobblin/compliance/ComplianceEvents.java    |   48 -
 .../java/gobblin/compliance/ComplianceJob.java  |  117 --
 .../gobblin/compliance/DatasetDescriptor.java   |   66 -
 .../compliance/DatasetDescriptorImpl.java       |   71 -
 .../compliance/HivePartitionDataset.java        |  113 --
 .../compliance/HivePartitionDatasetPolicy.java  |   54 -
 .../gobblin/compliance/HivePartitionFinder.java |  123 --
 .../compliance/HivePartitionVersion.java        |   46 -
 .../compliance/HivePartitionVersionFinder.java  |  168 ---
 .../compliance/HivePartitionVersionPolicy.java  |   35 -
 .../compliance/HiveProxyQueryExecutor.java      |  230 ----
 .../main/java/gobblin/compliance/Policy.java    |   26 -
 .../java/gobblin/compliance/QueryExecutor.java  |   33 -
 .../azkaban/ComplianceAzkabanJob.java           |  100 --
 .../gobblin/compliance/purger/CommitPolicy.java |   24 -
 .../purger/HivePurgerCommitPolicy.java          |   38 -
 .../compliance/purger/HivePurgerConverter.java  |   44 -
 .../compliance/purger/HivePurgerExtractor.java  |  142 --
 .../compliance/purger/HivePurgerPolicy.java     |   49 -
 .../compliance/purger/HivePurgerPublisher.java  |  183 ---
 .../purger/HivePurgerQueryTemplate.java         |  215 ---
 .../compliance/purger/HivePurgerSource.java     |  342 -----
 .../compliance/purger/HivePurgerWriter.java     |   73 --
 .../purger/HivePurgerWriterBuilder.java         |   36 -
 .../gobblin/compliance/purger/PurgePolicy.java  |   24 -
 .../compliance/purger/PurgeableDataset.java     |   27 -
 .../purger/PurgeableHivePartitionDataset.java   |  186 ---
 .../PurgeableHivePartitionDatasetSchema.java    |   25 -
 .../compliance/restore/AdhocRestorePolicy.java  |   59 -
 .../restore/ComplianceRestoreJob.java           |  144 ---
 .../restore/HivePartitionRestorePolicy.java     |   34 -
 .../compliance/restore/LKGRestorePolicy.java    |   96 --
 .../compliance/restore/RestorableDataset.java   |   30 -
 .../restore/RestorableHivePartitionDataset.java |  197 ---
 .../RestorableHivePartitionDatasetFinder.java   |   69 -
 .../compliance/restore/RestorePolicy.java       |   28 -
 .../CleanableHivePartitionDataset.java          |  136 --
 .../CleanableHivePartitionDatasetFinder.java    |   63 -
 .../retention/ComplianceRetentionJob.java       |  206 ---
 .../HivePartitionRetentionVersion.java          |   75 --
 .../HivePartitionVersionRetentionCleaner.java   |  129 --
 ...ePartitionVersionRetentionCleanerPolicy.java |  117 --
 .../HivePartitionVersionRetentionReaper.java    |  210 ---
 ...vePartitionVersionRetentionReaperPolicy.java |   70 -
 .../HivePartitionVersionRetentionRunner.java    |   42 -
 .../gobblin/compliance/utils/DatasetUtils.java  |   63 -
 .../compliance/utils/PartitionUtils.java        |   82 --
 .../gobblin/compliance/utils/ProxyUtils.java    |  113 --
 .../validation/ComplianceValidationJob.java     |  141 --
 .../validation/ValidatableDataset.java          |   30 -
 .../compliance/ComplianceConfigurationKeys.java |  142 ++
 .../gobblin/compliance/ComplianceEvents.java    |   48 +
 .../gobblin/compliance/ComplianceJob.java       |  117 ++
 .../gobblin/compliance/DatasetDescriptor.java   |   66 +
 .../compliance/DatasetDescriptorImpl.java       |   71 +
 .../compliance/HivePartitionDataset.java        |  113 ++
 .../compliance/HivePartitionDatasetPolicy.java  |   54 +
 .../gobblin/compliance/HivePartitionFinder.java |  123 ++
 .../compliance/HivePartitionVersion.java        |   46 +
 .../compliance/HivePartitionVersionFinder.java  |  168 +++
 .../compliance/HivePartitionVersionPolicy.java  |   35 +
 .../compliance/HiveProxyQueryExecutor.java      |  230 ++++
 .../org/apache/gobblin/compliance/Policy.java   |   26 +
 .../gobblin/compliance/QueryExecutor.java       |   33 +
 .../azkaban/ComplianceAzkabanJob.java           |  100 ++
 .../gobblin/compliance/purger/CommitPolicy.java |   24 +
 .../purger/HivePurgerCommitPolicy.java          |   38 +
 .../compliance/purger/HivePurgerConverter.java  |   44 +
 .../compliance/purger/HivePurgerExtractor.java  |  142 ++
 .../compliance/purger/HivePurgerPolicy.java     |   49 +
 .../compliance/purger/HivePurgerPublisher.java  |  183 +++
 .../purger/HivePurgerQueryTemplate.java         |  215 +++
 .../compliance/purger/HivePurgerSource.java     |  342 +++++
 .../compliance/purger/HivePurgerWriter.java     |   73 ++
 .../purger/HivePurgerWriterBuilder.java         |   36 +
 .../gobblin/compliance/purger/PurgePolicy.java  |   24 +
 .../compliance/purger/PurgeableDataset.java     |   27 +
 .../purger/PurgeableHivePartitionDataset.java   |  186 +++
 .../PurgeableHivePartitionDatasetSchema.java    |   25 +
 .../compliance/restore/AdhocRestorePolicy.java  |   59 +
 .../restore/ComplianceRestoreJob.java           |  144 +++
 .../restore/HivePartitionRestorePolicy.java     |   34 +
 .../compliance/restore/LKGRestorePolicy.java    |   96 ++
 .../compliance/restore/RestorableDataset.java   |   30 +
 .../restore/RestorableHivePartitionDataset.java |  197 +++
 .../RestorableHivePartitionDatasetFinder.java   |   69 +
 .../compliance/restore/RestorePolicy.java       |   28 +
 .../CleanableHivePartitionDataset.java          |  136 ++
 .../CleanableHivePartitionDatasetFinder.java    |   63 +
 .../retention/ComplianceRetentionJob.java       |  206 +++
 .../HivePartitionRetentionVersion.java          |   75 ++
 .../HivePartitionVersionRetentionCleaner.java   |  129 ++
 ...ePartitionVersionRetentionCleanerPolicy.java |  117 ++
 .../HivePartitionVersionRetentionReaper.java    |  210 +++
 ...vePartitionVersionRetentionReaperPolicy.java |   70 +
 .../HivePartitionVersionRetentionRunner.java    |   42 +
 .../gobblin/compliance/utils/DatasetUtils.java  |   63 +
 .../compliance/utils/PartitionUtils.java        |   82 ++
 .../gobblin/compliance/utils/ProxyUtils.java    |  113 ++
 .../validation/ComplianceValidationJob.java     |  141 ++
 .../validation/ValidatableDataset.java          |   30 +
 .../purger/HivePurgerConverterTest.java         |   49 -
 .../purger/HivePurgerExtractorTest.java         |   53 -
 .../compliance/purger/HivePurgerWriterTest.java |   20 -
 .../purger/HivePurgerConverterTest.java         |   49 +
 .../purger/HivePurgerExtractorTest.java         |   53 +
 .../compliance/purger/HivePurgerWriterTest.java |   20 +
 .../gobblin/couchbase/common/TupleDocument.java |   49 -
 .../converter/AnyToCouchbaseJsonConverter.java  |  108 --
 .../AvroToCouchbaseTupleConverter.java          |   63 -
 .../writer/CouchbaseEnvironmentFactory.java     |   51 -
 .../couchbase/writer/CouchbaseWriter.java       |  277 ----
 .../writer/CouchbaseWriterBuilder.java          |   54 -
 .../CouchbaseWriterConfigurationKeys.java       |   40 -
 .../gobblin/couchbase/common/TupleDocument.java |   49 +
 .../converter/AnyToCouchbaseJsonConverter.java  |  108 ++
 .../AvroToCouchbaseTupleConverter.java          |   63 +
 .../writer/CouchbaseEnvironmentFactory.java     |   51 +
 .../couchbase/writer/CouchbaseWriter.java       |  277 ++++
 .../writer/CouchbaseWriterBuilder.java          |   54 +
 .../CouchbaseWriterConfigurationKeys.java       |   40 +
 .../gobblin/couchbase/CouchbaseTestServer.java  |  206 ---
 .../AnyToCouchbaseJsonConverterTest.java        |  140 --
 .../AvroToCouchbaseTupleConverterTest.java      |   75 --
 .../couchbase/writer/CouchbaseWriterTest.java   |  582 ---------
 .../gobblin/couchbase/CouchbaseTestServer.java  |  206 +++
 .../AnyToCouchbaseJsonConverterTest.java        |  140 ++
 .../AvroToCouchbaseTupleConverterTest.java      |   75 ++
 .../couchbase/writer/CouchbaseWriterTest.java   |  582 +++++++++
 .../AvroStringFieldEncryptorConverter.java      |   36 -
 ...ordToEncryptedSerializedRecordConverter.java |   44 -
 .../StringFieldEncryptorConverter.java          |  106 --
 .../crypto/GobblinEncryptionProvider.java       |  140 --
 .../crypto/JCEKSKeystoreCredentialStoreCli.java |  294 -----
 .../AvroStringFieldEncryptorConverter.java      |   36 +
 ...ordToEncryptedSerializedRecordConverter.java |   44 +
 .../StringFieldEncryptorConverter.java          |  106 ++
 .../crypto/GobblinEncryptionProvider.java       |  140 ++
 .../crypto/JCEKSKeystoreCredentialStoreCli.java |  294 +++++
 .../gobblin.crypto.CredentialStoreProvider      |    1 -
 .../services/gobblin.crypto.EncryptionProvider  |    1 -
 ...pache.gobblin.crypto.CredentialStoreProvider |    1 +
 ...org.apache.gobblin.crypto.EncryptionProvider |    1 +
 .../AvroStringFieldEncryptorConverterTest.java  |  110 --
 ...oEncryptedSerializedRecordConverterTest.java |   95 --
 .../crypto/GobblinEncryptionProviderTest.java   |   37 -
 .../AvroStringFieldEncryptorConverterTest.java  |  110 ++
 ...oEncryptedSerializedRecordConverterTest.java |   95 ++
 .../crypto/GobblinEncryptionProviderTest.java   |   37 +
 .../src/test/resources/record_with_arrays.avsc  |    2 +-
 .../java/gobblin/crypto/EncodingBenchmark.java  |  126 --
 .../gobblin/crypto/EncodingBenchmark.java       |  126 ++
 ...oEncryptedSerializedRecordConverterBase.java |   81 --
 .../gobblin/crypto/Base64KeyToStringCodec.java  |   37 -
 .../src/main/java/gobblin/crypto/GPGCodec.java  |   54 -
 .../java/gobblin/crypto/GPGFileDecryptor.java   |   81 --
 .../gobblin/crypto/HexKeyToStringCodec.java     |   37 -
 .../crypto/JCEKSKeystoreCredentialStore.java    |  188 ---
 .../gobblin/crypto/JsonCredentialStore.java     |   93 --
 .../java/gobblin/crypto/KeyToStringCodec.java   |   34 -
 .../java/gobblin/crypto/RotatingAESCodec.java   |  328 -----
 ...oEncryptedSerializedRecordConverterBase.java |   81 ++
 .../gobblin/crypto/Base64KeyToStringCodec.java  |   37 +
 .../org/apache/gobblin/crypto/GPGCodec.java     |   54 +
 .../apache/gobblin/crypto/GPGFileDecryptor.java |   81 ++
 .../gobblin/crypto/HexKeyToStringCodec.java     |   37 +
 .../crypto/JCEKSKeystoreCredentialStore.java    |  188 +++
 .../gobblin/crypto/JsonCredentialStore.java     |   93 ++
 .../apache/gobblin/crypto/KeyToStringCodec.java |   34 +
 .../apache/gobblin/crypto/RotatingAESCodec.java |  328 +++++
 .../JCEKSKeystoreCredentialStoreTest.java       |   64 -
 .../gobblin/crypto/JsonCredentialStoreTest.java |   54 -
 .../gobblin/crypto/KeyToStringCodecTest.java    |   46 -
 .../gobblin/crypto/RotatingAESCodecTest.java    |  191 ---
 .../JCEKSKeystoreCredentialStoreTest.java       |   64 +
 .../gobblin/crypto/JsonCredentialStoreTest.java |   54 +
 .../gobblin/crypto/KeyToStringCodecTest.java    |   46 +
 .../gobblin/crypto/RotatingAESCodecTest.java    |  191 +++
 .../gobblin/eventhub/EventhubMetricNames.java   |   36 -
 .../writer/BatchedEventhubDataWriter.java       |   27 -
 .../writer/EventhubBatchAccumulator.java        |   18 -
 .../eventhub/writer/EventhubDataWriter.java     |  286 ----
 .../writer/EventhubDataWriterBuilder.java       |   73 --
 .../eventhub/writer/EventhubRequest.java        |   38 -
 .../gobblin/eventhub/EventhubMetricNames.java   |   36 +
 .../writer/BatchedEventhubDataWriter.java       |   27 +
 .../writer/EventhubBatchAccumulator.java        |   18 +
 .../eventhub/writer/EventhubDataWriter.java     |  286 ++++
 .../writer/EventhubDataWriterBuilder.java       |   73 ++
 .../eventhub/writer/EventhubRequest.java        |   38 +
 .../writer/BatchedEventhubDataWriterTest.java   |   94 --
 .../writer/EventhubAccumulatorTest.java         |  169 ---
 .../eventhub/writer/EventhubBatchTest.java      |   52 -
 .../eventhub/writer/EventhubDataWriterTest.java |   94 --
 .../writer/BatchedEventhubDataWriterTest.java   |   94 ++
 .../writer/EventhubAccumulatorTest.java         |  169 +++
 .../eventhub/writer/EventhubBatchTest.java      |   52 +
 .../eventhub/writer/EventhubDataWriterTest.java |   94 ++
 .../java/gobblin/metastore/ZkStateStore.java    |  324 -----
 .../ZkStateStoreConfigurationKeys.java          |   22 -
 .../gobblin/metastore/ZkStateStoreFactory.java  |   43 -
 .../gobblin/runtime/ZkDatasetStateStore.java    |  127 --
 .../runtime/ZkDatasetStateStoreFactory.java     |   42 -
 .../apache/gobblin/metastore/ZkStateStore.java  |  324 +++++
 .../ZkStateStoreConfigurationKeys.java          |   22 +
 .../gobblin/metastore/ZkStateStoreFactory.java  |   43 +
 .../gobblin/runtime/ZkDatasetStateStore.java    |  127 ++
 .../runtime/ZkDatasetStateStoreFactory.java     |   42 +
 .../runtime/StateStoreWatermarkStorageTest.java |   85 --
 .../runtime/ZkDatasetStateStoreTest.java        |  231 ----
 .../runtime/StateStoreWatermarkStorageTest.java |   85 ++
 .../runtime/ZkDatasetStateStoreTest.java        |  231 ++++
 .../src/main/avro/HttpOperation.avsc            |    2 +-
 .../src/main/avro/HttpRequestReponse.avsc       |    2 +-
 .../converter/AsyncHttpJoinConverter.java       |  182 ---
 .../converter/AvroApacheHttpJoinConverter.java  |   62 -
 .../converter/AvroHttpJoinConverter.java        |  121 --
 .../gobblin/converter/AvroR2JoinConverter.java  |   76 --
 .../gobblin/converter/HttpJoinConverter.java    |  120 --
 .../gobblin/http/ApacheHttpAsyncClient.java     |  184 ---
 .../java/gobblin/http/ApacheHttpClient.java     |  111 --
 .../java/gobblin/http/ApacheHttpRequest.java    |   58 -
 .../gobblin/http/ApacheHttpRequestBuilder.java  |  120 --
 .../gobblin/http/ApacheHttpResponseHandler.java |   77 --
 .../gobblin/http/ApacheHttpResponseStatus.java  |   15 -
 .../src/main/java/gobblin/http/HttpClient.java  |   43 -
 .../main/java/gobblin/http/ResponseHandler.java |   31 -
 .../main/java/gobblin/http/ResponseStatus.java  |   34 -
 .../src/main/java/gobblin/http/StatusType.java  |   33 -
 .../java/gobblin/http/ThrottledHttpClient.java  |   84 --
 .../src/main/java/gobblin/r2/D2ClientProxy.java |  146 ---
 .../src/main/java/gobblin/r2/R2Client.java      |   89 --
 .../main/java/gobblin/r2/R2ClientFactory.java   |  179 ---
 .../main/java/gobblin/r2/R2HttpClientProxy.java |   48 -
 .../src/main/java/gobblin/r2/R2Request.java     |   52 -
 .../main/java/gobblin/r2/R2ResponseStatus.java  |   19 -
 .../java/gobblin/r2/R2RestRequestBuilder.java   |  114 --
 .../java/gobblin/r2/R2RestResponseHandler.java  |   70 -
 .../main/java/gobblin/utils/HttpConstants.java  |   27 -
 .../src/main/java/gobblin/utils/HttpUtils.java  |  207 ---
 .../java/gobblin/writer/AsyncHttpWriter.java    |  171 ---
 .../gobblin/writer/AsyncHttpWriterBuilder.java  |  119 --
 .../gobblin/writer/AvroHttpWriterBuilder.java   |   46 -
 .../gobblin/writer/R2RestWriterBuilder.java     |   73 --
 .../converter/AsyncHttpJoinConverter.java       |  182 +++
 .../converter/AvroApacheHttpJoinConverter.java  |   62 +
 .../converter/AvroHttpJoinConverter.java        |  121 ++
 .../gobblin/converter/AvroR2JoinConverter.java  |   76 ++
 .../gobblin/converter/HttpJoinConverter.java    |  120 ++
 .../gobblin/http/ApacheHttpAsyncClient.java     |  184 +++
 .../apache/gobblin/http/ApacheHttpClient.java   |  111 ++
 .../apache/gobblin/http/ApacheHttpRequest.java  |   58 +
 .../gobblin/http/ApacheHttpRequestBuilder.java  |  120 ++
 .../gobblin/http/ApacheHttpResponseHandler.java |   77 ++
 .../gobblin/http/ApacheHttpResponseStatus.java  |   15 +
 .../org/apache/gobblin/http/HttpClient.java     |   43 +
 .../apache/gobblin/http/ResponseHandler.java    |   31 +
 .../org/apache/gobblin/http/ResponseStatus.java |   34 +
 .../org/apache/gobblin/http/StatusType.java     |   33 +
 .../gobblin/http/ThrottledHttpClient.java       |   84 ++
 .../org/apache/gobblin/r2/D2ClientProxy.java    |  146 +++
 .../java/org/apache/gobblin/r2/R2Client.java    |   89 ++
 .../org/apache/gobblin/r2/R2ClientFactory.java  |  179 +++
 .../apache/gobblin/r2/R2HttpClientProxy.java    |   48 +
 .../java/org/apache/gobblin/r2/R2Request.java   |   52 +
 .../org/apache/gobblin/r2/R2ResponseStatus.java |   19 +
 .../apache/gobblin/r2/R2RestRequestBuilder.java |  114 ++
 .../gobblin/r2/R2RestResponseHandler.java       |   70 +
 .../org/apache/gobblin/utils/HttpConstants.java |   27 +
 .../org/apache/gobblin/utils/HttpUtils.java     |  207 +++
 .../apache/gobblin/writer/AsyncHttpWriter.java  |  171 +++
 .../gobblin/writer/AsyncHttpWriterBuilder.java  |  119 ++
 .../gobblin/writer/AvroHttpWriterBuilder.java   |   46 +
 .../gobblin/writer/R2RestWriterBuilder.java     |   73 ++
 .../src/test/java/gobblin/HttpTestUtils.java    |   98 --
 .../test/java/gobblin/MockGenericRecord.java    |   41 -
 .../http/ApacheHttpRequestBuilderTest.java      |   51 -
 .../java/gobblin/r2/R2ClientFactoryTest.java    |  102 --
 .../gobblin/r2/R2RestRequestBuilderTest.java    |   61 -
 .../test/java/gobblin/util/HttpUtilsTest.java   |   71 -
 .../gobblin/writer/AsyncHttpWriterTest.java     |  337 -----
 .../java/org/apache/gobblin/HttpTestUtils.java  |   98 ++
 .../org/apache/gobblin/MockGenericRecord.java   |   41 +
 .../http/ApacheHttpRequestBuilderTest.java      |   51 +
 .../apache/gobblin/r2/R2ClientFactoryTest.java  |  102 ++
 .../gobblin/r2/R2RestRequestBuilderTest.java    |   61 +
 .../org/apache/gobblin/util/HttpUtilsTest.java  |   71 +
 .../gobblin/writer/AsyncHttpWriterTest.java     |  337 +++++
 .../resource/job-props/testKafkaIngest.pull     |   12 +-
 .../kafka/client/Kafka08ConsumerClient.java     |  376 ------
 .../kafka/serialize/LiAvroDeserializer.java     |   67 -
 .../kafka/serialize/LiAvroSerializer.java       |   44 -
 .../gobblin/kafka/tool/KafkaCheckpoint.java     |   73 --
 .../gobblin/kafka/tool/SimpleKafkaConsumer.java |  201 ---
 .../gobblin/kafka/writer/Kafka08DataWriter.java |  131 --
 .../kafka/writer/KafkaDataWriterBuilder.java    |   45 -
 .../gobblin/metrics/KafkaReportingFormats.java  |   83 --
 .../metrics/kafka/KafkaAvroEventReporter.java   |  122 --
 .../metrics/kafka/KafkaAvroReporter.java        |  105 --
 .../metrics/kafka/KafkaEventReporter.java       |  151 ---
 .../java/gobblin/metrics/kafka/KafkaPusher.java |   85 --
 .../gobblin/metrics/kafka/KafkaReporter.java    |  147 ---
 .../metrics/kafka/KafkaReporterFactory.java     |  104 --
 .../metrics/kafka/ProducerCloseable.java        |   33 -
 .../SimpleKafkaSpecExecutorInstance.java        |  131 --
 ...SimpleKafkaSpecExecutorInstanceConsumer.java |  261 ----
 ...SimpleKafkaSpecExecutorInstanceProducer.java |  139 --
 ...eamingKafkaSpecExecutorInstanceConsumer.java |  171 ---
 .../kafka/KafkaDeserializerExtractor.java       |  219 ----
 .../extract/kafka/KafkaDeserializerSource.java  |   39 -
 .../extract/kafka/KafkaGsonDeserializer.java    |   56 -
 .../extractor/extract/kafka/KafkaWrapper.java   |  546 --------
 .../kafka/client/Kafka08ConsumerClient.java     |  376 ++++++
 .../kafka/serialize/LiAvroDeserializer.java     |   67 +
 .../kafka/serialize/LiAvroSerializer.java       |   44 +
 .../gobblin/kafka/tool/KafkaCheckpoint.java     |   73 ++
 .../gobblin/kafka/tool/SimpleKafkaConsumer.java |  201 +++
 .../gobblin/kafka/writer/Kafka08DataWriter.java |  131 ++
 .../kafka/writer/KafkaDataWriterBuilder.java    |   45 +
 .../gobblin/metrics/KafkaReportingFormats.java  |   83 ++
 .../metrics/kafka/KafkaAvroEventReporter.java   |  122 ++
 .../metrics/kafka/KafkaAvroReporter.java        |  105 ++
 .../metrics/kafka/KafkaEventReporter.java       |  151 +++
 .../gobblin/metrics/kafka/KafkaPusher.java      |   85 ++
 .../gobblin/metrics/kafka/KafkaReporter.java    |  147 +++
 .../metrics/kafka/KafkaReporterFactory.java     |  104 ++
 .../metrics/kafka/ProducerCloseable.java        |   33 +
 .../SimpleKafkaSpecExecutorInstance.java        |  131 ++
 ...SimpleKafkaSpecExecutorInstanceConsumer.java |  261 ++++
 ...SimpleKafkaSpecExecutorInstanceProducer.java |  139 ++
 ...eamingKafkaSpecExecutorInstanceConsumer.java |  171 +++
 .../kafka/KafkaDeserializerExtractor.java       |  219 ++++
 .../extract/kafka/KafkaDeserializerSource.java  |   39 +
 .../extract/kafka/KafkaGsonDeserializer.java    |   56 +
 .../extractor/extract/kafka/KafkaWrapper.java   |  546 ++++++++
 .../test/java/StandaloneTestKafkaServer.java    |    2 +-
 .../java/gobblin/kafka/FlakyKafkaProducer.java  |  104 --
 .../test/java/gobblin/kafka/KafkaTestBase.java  |  251 ----
 .../Kafka08DataWriterIntegrationTest.java       |  181 ---
 .../kafka/writer/Kafka08DataWriterTest.java     |  172 ---
 .../kafka/writer/Kafka08DataWriterUnitTest.java |   51 -
 .../reporter/KafkaAvroEventReporterTest.java    |   50 -
 ...AvroEventReporterWithSchemaRegistryTest.java |  111 --
 .../metrics/reporter/KafkaAvroReporterTest.java |   68 -
 .../reporter/KafkaEventReporterTest.java        |  150 ---
 .../metrics/reporter/KafkaPusherTest.java       |   82 --
 .../metrics/reporter/KafkaReporterTest.java     |  242 ----
 .../gobblin/metrics/reporter/KafkaTestBase.java |  133 --
 .../metrics/reporter/MockKafkaPusher.java       |   64 -
 .../SimpleKafkaSpecExecutorInstanceTest.java    |  180 ---
 .../StreamingKafkaSpecExecutorInstanceTest.java |  192 ---
 .../kafka/ConfluentKafkaSchemaRegistryTest.java |   98 --
 .../kafka/KafkaDeserializerExtractorTest.java   |  286 ----
 .../kafka/KafkaGsonDeserializerTest.java        |   43 -
 .../extract/kafka/KafkaWrapperTest.java         |   56 -
 .../kafka/SimpleKafkaSchemaRegistryTest.java    |   36 -
 .../gobblin/kafka/FlakyKafkaProducer.java       |  104 ++
 .../org/apache/gobblin/kafka/KafkaTestBase.java |  251 ++++
 .../Kafka08DataWriterIntegrationTest.java       |  181 +++
 .../kafka/writer/Kafka08DataWriterTest.java     |  172 +++
 .../kafka/writer/Kafka08DataWriterUnitTest.java |   51 +
 .../reporter/KafkaAvroEventReporterTest.java    |   50 +
 ...AvroEventReporterWithSchemaRegistryTest.java |  111 ++
 .../metrics/reporter/KafkaAvroReporterTest.java |   68 +
 .../reporter/KafkaEventReporterTest.java        |  150 +++
 .../metrics/reporter/KafkaPusherTest.java       |   82 ++
 .../metrics/reporter/KafkaReporterTest.java     |  242 ++++
 .../gobblin/metrics/reporter/KafkaTestBase.java |  133 ++
 .../metrics/reporter/MockKafkaPusher.java       |   64 +
 .../SimpleKafkaSpecExecutorInstanceTest.java    |  180 +++
 .../StreamingKafkaSpecExecutorInstanceTest.java |  192 +++
 .../kafka/ConfluentKafkaSchemaRegistryTest.java |   98 ++
 .../kafka/KafkaDeserializerExtractorTest.java   |  286 ++++
 .../kafka/KafkaGsonDeserializerTest.java        |   43 +
 .../extract/kafka/KafkaWrapperTest.java         |   56 +
 .../kafka/SimpleKafkaSchemaRegistryTest.java    |   36 +
 .../kafka/client/Kafka09ConsumerClient.java     |  197 ---
 .../kafka/serialize/LiAvroDeserializer.java     |   67 -
 .../kafka/serialize/LiAvroSerializer.java       |   44 -
 .../gobblin/kafka/writer/Kafka09DataWriter.java |  129 --
 .../kafka/writer/KafkaDataWriterBuilder.java    |   35 -
 .../kafka/KafkaSimpleStreamingExtractor.java    |  247 ----
 .../kafka/KafkaSimpleStreamingSource.java       |  145 ---
 .../kafka/client/Kafka09ConsumerClient.java     |  197 +++
 .../kafka/serialize/LiAvroDeserializer.java     |   67 +
 .../kafka/serialize/LiAvroSerializer.java       |   44 +
 .../gobblin/kafka/writer/Kafka09DataWriter.java |  129 ++
 .../kafka/writer/KafkaDataWriterBuilder.java    |   35 +
 .../kafka/KafkaSimpleStreamingExtractor.java    |  247 ++++
 .../kafka/KafkaSimpleStreamingSource.java       |  145 +++
 .../test/java/gobblin/kafka/KafkaTestBase.java  |  269 ----
 .../kafka/client/Kafka09ConsumerClientTest.java |   76 --
 .../extract/kafka/KafkaSimpleStreamingTest.java |  267 ----
 .../kafka/writer/Kafka09DataWriterTest.java     |  184 ---
 .../org/apache/gobblin/kafka/KafkaTestBase.java |  269 ++++
 .../kafka/client/Kafka09ConsumerClientTest.java |   76 ++
 .../extract/kafka/KafkaSimpleStreamingTest.java |  267 ++++
 .../kafka/writer/Kafka09DataWriterTest.java     |  184 +++
 .../converter/EnvelopeSchemaConverter.java      |  152 ---
 .../LiKafkaByteArrayMsgToAvroConverter.java     |   78 --
 .../client/AbstractBaseKafkaConsumerClient.java |   89 --
 .../kafka/client/BaseKafkaConsumerRecord.java   |   49 -
 .../kafka/client/ByteArrayBasedKafkaRecord.java |   28 -
 .../kafka/client/DecodeableKafkaRecord.java     |   36 -
 .../client/GobblinKafkaConsumerClient.java      |   99 --
 .../kafka/client/KafkaConsumerRecord.java       |   41 -
 .../schemareg/CachingKafkaSchemaRegistry.java   |  124 --
 .../ConfigDrivenMd5SchemaRegistry.java          |  152 ---
 .../kafka/schemareg/HttpClientFactory.java      |   43 -
 .../kafka/schemareg/KafkaSchemaRegistry.java    |   74 --
 .../KafkaSchemaRegistryConfigurationKeys.java   |   27 -
 .../schemareg/KafkaSchemaRegistryFactory.java   |   63 -
 .../kafka/schemareg/LiKafkaSchemaRegistry.java  |  251 ----
 .../schemareg/SchemaRegistryException.java      |   33 -
 .../kafka/serialize/LiAvroDeserializerBase.java |  124 --
 .../kafka/serialize/LiAvroSerDeHelper.java      |   38 -
 .../kafka/serialize/LiAvroSerializerBase.java   |   84 --
 .../java/gobblin/kafka/serialize/MD5Digest.java |  115 --
 .../kafka/serialize/SerializationException.java |   32 -
 .../writer/BaseKafkaDataWriterBuilder.java      |   73 --
 .../writer/KafkaWriterConfigurationKeys.java    |   51 -
 .../gobblin/kafka/writer/KafkaWriterHelper.java |   92 --
 .../kafka/writer/KafkaWriterMetricNames.java    |   42 -
 .../metrics/kafka/KafkaAvroSchemaRegistry.java  |  267 ----
 .../kafka/KafkaAvroSchemaRegistryFactory.java   |   29 -
 .../metrics/kafka/KafkaSchemaRegistry.java      |  204 ---
 .../kafka/KafkaSchemaRegistryFactory.java       |   35 -
 .../metrics/kafka/SchemaRegistryException.java  |   36 -
 .../util/SchemaRegistryVersionWriter.java       |  140 --
 .../extract/kafka/ConfigStoreUtils.java         |  216 ----
 .../kafka/ConfluentKafkaSchemaRegistry.java     |  114 --
 .../kafka/FixedSchemaKafkaAvroExtractor.java    |   62 -
 .../extract/kafka/KafkaAvroExtractor.java       |  144 ---
 .../extractor/extract/kafka/KafkaExtractor.java |  367 ------
 .../KafkaOffsetRetrievalFailureException.java   |   27 -
 .../extractor/extract/kafka/KafkaPartition.java |  150 ---
 .../extractor/extract/kafka/KafkaRecord.java    |   89 --
 .../extract/kafka/KafkaSimpleExtractor.java     |   62 -
 .../extract/kafka/KafkaSimpleJsonExtractor.java |   55 -
 .../extract/kafka/KafkaSimpleSource.java        |   48 -
 .../extractor/extract/kafka/KafkaSource.java    |  644 ---------
 .../extractor/extract/kafka/KafkaTopic.java     |   52 -
 .../extractor/extract/kafka/KafkaUtils.java     |  171 ---
 .../extract/kafka/MultiLongWatermark.java       |  133 --
 .../kafka/PreviousOffsetNotFoundException.java  |   27 -
 .../kafka/SimpleKafkaSchemaRegistry.java        |   57 -
 .../kafka/StartOffsetOutOfRangeException.java   |   27 -
 .../extract/kafka/UniversalKafkaSource.java     |   54 -
 ...AvgRecordSizeBasedWorkUnitSizeEstimator.java |   91 --
 ...AvgRecordTimeBasedWorkUnitSizeEstimator.java |  152 ---
 .../packer/KafkaBiLevelWorkUnitPacker.java      |  153 ---
 .../packer/KafkaSingleLevelWorkUnitPacker.java  |   73 --
 .../workunit/packer/KafkaWorkUnitPacker.java    |  352 -----
 .../packer/KafkaWorkUnitSizeEstimator.java      |   32 -
 .../converter/EnvelopeSchemaConverter.java      |  152 +++
 .../LiKafkaByteArrayMsgToAvroConverter.java     |   78 ++
 .../client/AbstractBaseKafkaConsumerClient.java |   89 ++
 .../kafka/client/BaseKafkaConsumerRecord.java   |   49 +
 .../kafka/client/ByteArrayBasedKafkaRecord.java |   28 +
 .../kafka/client/DecodeableKafkaRecord.java     |   36 +
 .../client/GobblinKafkaConsumerClient.java      |   99 ++
 .../kafka/client/KafkaConsumerRecord.java       |   41 +
 .../schemareg/CachingKafkaSchemaRegistry.java   |  124 ++
 .../ConfigDrivenMd5SchemaRegistry.java          |  152 +++
 .../kafka/schemareg/HttpClientFactory.java      |   43 +
 .../kafka/schemareg/KafkaSchemaRegistry.java    |   74 ++
 .../KafkaSchemaRegistryConfigurationKeys.java   |   27 +
 .../schemareg/KafkaSchemaRegistryFactory.java   |   63 +
 .../kafka/schemareg/LiKafkaSchemaRegistry.java  |  251 ++++
 .../schemareg/SchemaRegistryException.java      |   33 +
 .../kafka/serialize/LiAvroDeserializerBase.java |  124 ++
 .../kafka/serialize/LiAvroSerDeHelper.java      |   38 +
 .../kafka/serialize/LiAvroSerializerBase.java   |   84 ++
 .../gobblin/kafka/serialize/MD5Digest.java      |  115 ++
 .../kafka/serialize/SerializationException.java |   32 +
 .../writer/BaseKafkaDataWriterBuilder.java      |   73 ++
 .../writer/KafkaWriterConfigurationKeys.java    |   51 +
 .../gobblin/kafka/writer/KafkaWriterHelper.java |   92 ++
 .../kafka/writer/KafkaWriterMetricNames.java    |   42 +
 .../metrics/kafka/KafkaAvroSchemaRegistry.java  |  267 ++++
 .../kafka/KafkaAvroSchemaRegistryFactory.java   |   29 +
 .../metrics/kafka/KafkaSchemaRegistry.java      |  204 +++
 .../kafka/KafkaSchemaRegistryFactory.java       |   35 +
 .../metrics/kafka/SchemaRegistryException.java  |   36 +
 .../util/SchemaRegistryVersionWriter.java       |  140 ++
 .../extract/kafka/ConfigStoreUtils.java         |  216 ++++
 .../kafka/ConfluentKafkaSchemaRegistry.java     |  114 ++
 .../kafka/FixedSchemaKafkaAvroExtractor.java    |   62 +
 .../extract/kafka/KafkaAvroExtractor.java       |  144 +++
 .../extractor/extract/kafka/KafkaExtractor.java |  367 ++++++
 .../KafkaOffsetRetrievalFailureException.java   |   27 +
 .../extractor/extract/kafka/KafkaPartition.java |  150 +++
 .../extractor/extract/kafka/KafkaRecord.java    |   89 ++
 .../extract/kafka/KafkaSimpleExtractor.java     |   62 +
 .../extract/kafka/KafkaSimpleJsonExtractor.java |   55 +
 .../extract/kafka/KafkaSimpleSource.java        |   48 +
 .../extractor/extract/kafka/KafkaSource.java    |  644 +++++++++
 .../extractor/extract/kafka/KafkaTopic.java     |   52 +
 .../extractor/extract/kafka/KafkaUtils.java     |  171 +++
 .../extract/kafka/MultiLongWatermark.java       |  133 ++
 .../kafka/PreviousOffsetNotFoundException.java  |   27 +
 .../kafka/SimpleKafkaSchemaRegistry.java        |   57 +
 .../kafka/StartOffsetOutOfRangeException.java   |   27 +
 .../extract/kafka/UniversalKafkaSource.java     |   54 +
 ...AvgRecordSizeBasedWorkUnitSizeEstimator.java |   91 ++
 ...AvgRecordTimeBasedWorkUnitSizeEstimator.java |  152 +++
 .../packer/KafkaBiLevelWorkUnitPacker.java      |  153 +++
 .../packer/KafkaSingleLevelWorkUnitPacker.java  |   73 ++
 .../workunit/packer/KafkaWorkUnitPacker.java    |  352 +++++
 .../packer/KafkaWorkUnitSizeEstimator.java      |   32 +
 .../converter/EnvelopeSchemaConverterTest.java  |   73 --
 .../KafkaAvroSchemaRegistryForTest.java         |   50 -
 .../CachingKafkaSchemaRegistryTest.java         |  133 --
 .../gobblin/kafka/serialize/MD5DigestTest.java  |   66 -
 .../converter/EnvelopeSchemaConverterTest.java  |   73 ++
 .../KafkaAvroSchemaRegistryForTest.java         |   50 +
 .../CachingKafkaSchemaRegistryTest.java         |  133 ++
 .../gobblin/kafka/serialize/MD5DigestTest.java  |   66 +
 .../converter/MetadataConverterWrapper.java     |  111 --
 ...thMetadataToEnvelopedRecordWithMetadata.java |  123 --
 .../metadata/GlobalMetadataCollector.java       |  123 --
 .../DatasetAwareFsMetadataProvider.java         |   35 -
 .../provider/DatasetAwareMetadataProvider.java  |   34 -
 .../DatasetAwareMetadataProviderFactory.java    |   28 -
 .../provider/MetadataAwareFileSystem.java       |  127 --
 .../provider/PermissionMetadataParser.java      |   46 -
 .../provider/SimpleConfigMetadataProvider.java  |   50 -
 .../provider/SimpleMetadataProviderFactory.java |   47 -
 .../gobblin/metadata/types/GlobalMetadata.java  |  393 ------
 .../types/GlobalMetadataJsonMerger.java         |   71 -
 .../java/gobblin/metadata/types/Metadata.java   |   27 -
 .../types/StaticStringMetadataMerger.java       |   54 -
 .../java/gobblin/type/ContentTypeUtils.java     |   98 --
 .../java/gobblin/type/RecordWithMetadata.java   |   40 -
 .../type/SerializedRecordWithMetadata.java      |   38 -
 .../converter/MetadataConverterWrapper.java     |  111 ++
 ...thMetadataToEnvelopedRecordWithMetadata.java |  123 ++
 .../metadata/GlobalMetadataCollector.java       |  123 ++
 .../DatasetAwareFsMetadataProvider.java         |   35 +
 .../provider/DatasetAwareMetadataProvider.java  |   34 +
 .../DatasetAwareMetadataProviderFactory.java    |   28 +
 .../provider/MetadataAwareFileSystem.java       |  127 ++
 .../provider/PermissionMetadataParser.java      |   46 +
 .../provider/SimpleConfigMetadataProvider.java  |   50 +
 .../provider/SimpleMetadataProviderFactory.java |   47 +
 .../gobblin/metadata/types/GlobalMetadata.java  |  393 ++++++
 .../types/GlobalMetadataJsonMerger.java         |   71 +
 .../apache/gobblin/metadata/types/Metadata.java |   27 +
 .../types/StaticStringMetadataMerger.java       |   54 +
 .../apache/gobblin/type/ContentTypeUtils.java   |   98 ++
 .../apache/gobblin/type/RecordWithMetadata.java |   40 +
 .../type/SerializedRecordWithMetadata.java      |   38 +
 ....filesystem.FileSystemInstrumentationFactory |   17 -
 ....filesystem.FileSystemInstrumentationFactory |   17 +
 .../converter/MetadataConverterWrapperTest.java |  117 --
 ...tadataToEnvelopedRecordWithMetadataTest.java |  135 --
 .../metadata/GlobalMetadataCollectorTest.java   |  118 --
 .../metadata/types/GlobalMetadataTest.java      |  131 --
 .../converter/MetadataConverterWrapperTest.java |  117 ++
 ...tadataToEnvelopedRecordWithMetadataTest.java |  135 ++
 .../metadata/GlobalMetadataCollectorTest.java   |  118 ++
 .../metadata/types/GlobalMetadataTest.java      |  131 ++
 .../graphite/GraphiteConnectionType.java        |   51 -
 .../metrics/graphite/GraphiteEventReporter.java |  245 ----
 .../metrics/graphite/GraphitePusher.java        |   66 -
 .../metrics/graphite/GraphiteReporter.java      |  267 ----
 .../graphite/GraphiteConnectionType.java        |   51 +
 .../metrics/graphite/GraphiteEventReporter.java |  245 ++++
 .../metrics/graphite/GraphitePusher.java        |   66 +
 .../metrics/graphite/GraphiteReporter.java      |  267 ++++
 .../graphite/GraphiteEventReporterTest.java     |  152 ---
 .../metrics/graphite/GraphiteReporterTest.java  |  198 ---
 .../metrics/graphite/TestGraphiteSender.java    |   79 --
 .../graphite/GraphiteEventReporterTest.java     |  152 +++
 .../metrics/graphite/GraphiteReporterTest.java  |  198 +++
 .../metrics/graphite/TestGraphiteSender.java    |   79 ++
 .../hadoop/AbstractHadoopCounterReporter.java   |  118 --
 .../metrics/hadoop/HadoopCounterReporter.java   |   97 --
 .../hadoop/NewAPIHadoopCounterReporter.java     |  120 --
 .../hadoop/AbstractHadoopCounterReporter.java   |  118 ++
 .../metrics/hadoop/HadoopCounterReporter.java   |   97 ++
 .../hadoop/NewAPIHadoopCounterReporter.java     |  120 ++
 .../hadoop/HadoopCounterReporterTest.java       |  165 ---
 .../hadoop/NewAPIHadoopCounterReporterTest.java |  165 ---
 .../hadoop/HadoopCounterReporterTest.java       |  165 +++
 .../hadoop/NewAPIHadoopCounterReporterTest.java |  165 +++
 .../influxdb/InfluxDBConnectionType.java        |   41 -
 .../metrics/influxdb/InfluxDBEventReporter.java |  225 ----
 .../metrics/influxdb/InfluxDBPusher.java        |  104 --
 .../metrics/influxdb/InfluxDBReporter.java      |  275 ----
 .../influxdb/InfluxDBConnectionType.java        |   41 +
 .../metrics/influxdb/InfluxDBEventReporter.java |  225 ++++
 .../metrics/influxdb/InfluxDBPusher.java        |  104 ++
 .../metrics/influxdb/InfluxDBReporter.java      |  275 ++++
 .../influxdb/InfluxDBEventReporterTest.java     |  158 ---
 .../metrics/influxdb/InfluxDBReporterTest.java  |  201 ---
 .../gobblin/metrics/influxdb/TestInfluxDB.java  |  150 ---
 .../influxdb/InfluxDBEventReporterTest.java     |  158 +++
 .../metrics/influxdb/InfluxDBReporterTest.java  |  201 +++
 .../gobblin/metrics/influxdb/TestInfluxDB.java  |  150 +++
 .../AvroToJdbcEntryConverterInitializer.java    |  116 --
 .../jdbc/AvroToJdbcEntryConverter.java          |  344 -----
 .../gobblin/converter/jdbc/JdbcEntryData.java   |   61 -
 .../gobblin/converter/jdbc/JdbcEntryDatum.java  |   44 -
 .../converter/jdbc/JdbcEntryMetaDatum.java      |   44 -
 .../gobblin/converter/jdbc/JdbcEntrySchema.java |   68 -
 .../java/gobblin/converter/jdbc/JdbcType.java   |   51 -
 .../java/gobblin/publisher/JdbcPublisher.java   |  208 ---
 .../extractor/extract/jdbc/MysqlSource.java     |   53 -
 .../extractor/extract/jdbc/OracleSource.java    |   53 -
 .../extractor/extract/jdbc/SqlServerSource.java |   53 -
 .../extractor/extract/jdbc/TeradataSource.java  |   51 -
 .../java/gobblin/source/jdbc/JdbcCommand.java   |   77 --
 .../source/jdbc/JdbcCommandFormatException.java |   36 -
 .../gobblin/source/jdbc/JdbcCommandOutput.java  |   53 -
 .../java/gobblin/source/jdbc/JdbcExtractor.java | 1222 ------------------
 .../java/gobblin/source/jdbc/JdbcProvider.java  |   92 --
 .../gobblin/source/jdbc/JdbcSpecificLayer.java  |   55 -
 .../gobblin/source/jdbc/MysqlExtractor.java     |  290 -----
 .../gobblin/source/jdbc/OracleExtractor.java    |  325 -----
 .../java/gobblin/source/jdbc/SqlQueryUtils.java |   91 --
 .../gobblin/source/jdbc/SqlServerExtractor.java |  304 -----
 .../gobblin/source/jdbc/TeradataExtractor.java  |  346 -----
 .../main/java/gobblin/writer/JdbcWriter.java    |  202 ---
 .../java/gobblin/writer/JdbcWriterBuilder.java  |   46 -
 .../commands/BaseJdbcBufferedInserter.java      |  189 ---
 .../commands/GenericJdbcBufferedInserter.java   |  114 --
 .../writer/commands/JdbcBufferedInserter.java   |   62 -
 .../writer/commands/JdbcWriterCommands.java     |   96 --
 .../commands/JdbcWriterCommandsFactory.java     |   63 -
 .../writer/commands/MySqlBufferedInserter.java  |   89 --
 .../writer/commands/MySqlWriterCommands.java    |  165 ---
 .../commands/TeradataBufferedInserter.java      |  121 --
 .../writer/commands/TeradataWriterCommands.java |  185 ---
 .../initializer/JdbcWriterInitializer.java      |  331 -----
 .../AvroToJdbcEntryConverterInitializer.java    |  116 ++
 .../jdbc/AvroToJdbcEntryConverter.java          |  344 +++++
 .../gobblin/converter/jdbc/JdbcEntryData.java   |   61 +
 .../gobblin/converter/jdbc/JdbcEntryDatum.java  |   44 +
 .../converter/jdbc/JdbcEntryMetaDatum.java      |   44 +
 .../gobblin/converter/jdbc/JdbcEntrySchema.java |   68 +
 .../apache/gobblin/converter/jdbc/JdbcType.java |   51 +
 .../apache/gobblin/publisher/JdbcPublisher.java |  208 +++
 .../extractor/extract/jdbc/MysqlSource.java     |   53 +
 .../extractor/extract/jdbc/OracleSource.java    |   53 +
 .../extractor/extract/jdbc/SqlServerSource.java |   53 +
 .../extractor/extract/jdbc/TeradataSource.java  |   51 +
 .../apache/gobblin/source/jdbc/JdbcCommand.java |   77 ++
 .../source/jdbc/JdbcCommandFormatException.java |   36 +
 .../gobblin/source/jdbc/JdbcCommandOutput.java  |   53 +
 .../gobblin/source/jdbc/JdbcExtractor.java      | 1222 ++++++++++++++++++
 .../gobblin/source/jdbc/JdbcProvider.java       |   92 ++
 .../gobblin/source/jdbc/JdbcSpecificLayer.java  |   55 +
 .../gobblin/source/jdbc/MysqlExtractor.java     |  290 +++++
 .../gobblin/source/jdbc/OracleExtractor.java    |  325 +++++
 .../gobblin/source/jdbc/SqlQueryUtils.java      |   91 ++
 .../gobblin/source/jdbc/SqlServerExtractor.java |  304 +++++
 .../gobblin/source/jdbc/TeradataExtractor.java  |  346 +++++
 .../org/apache/gobblin/writer/JdbcWriter.java   |  202 +++
 .../gobblin/writer/JdbcWriterBuilder.java       |   46 +
 .../commands/BaseJdbcBufferedInserter.java      |  189 +++
 .../commands/GenericJdbcBufferedInserter.java   |  114 ++
 .../writer/commands/JdbcBufferedInserter.java   |   62 +
 .../writer/commands/JdbcWriterCommands.java     |   96 ++
 .../commands/JdbcWriterCommandsFactory.java     |   63 +
 .../writer/commands/MySqlBufferedInserter.java  |   89 ++
 .../writer/commands/MySqlWriterCommands.java    |  165 +++
 .../commands/TeradataBufferedInserter.java      |  121 ++
 .../writer/commands/TeradataWriterCommands.java |  185 +++
 .../initializer/JdbcWriterInitializer.java      |  331 +++++
 .../jdbc/AvroToJdbcEntryConverterTest.java      |  133 --
 .../gobblin/source/TimestampWatermarkTest.java  |   64 -
 .../gobblin/source/jdbc/JdbcExtractorTest.java  |  148 ---
 .../gobblin/source/jdbc/MockJdbcColumn.java     |   31 -
 .../source/jdbc/OracleExtractorTest.java        |  124 --
 .../gobblin/source/jdbc/SqlQueryUtilsTest.java  |   96 --
 .../writer/JdbcBufferedInserterTestBase.java    |   77 --
 .../java/gobblin/writer/JdbcPublisherTest.java  |  130 --
 .../gobblin/writer/JdbcWriterCommandsTest.java  |  130 --
 .../writer/JdbcWriterInitializerTest.java       |  162 ---
 .../java/gobblin/writer/JdbcWriterTest.java     |   83 --
 .../writer/MySqlBufferedInserterTest.java       |  108 --
 .../writer/TeradataBufferedInserterTest.java    |   87 --
 .../jdbc/AvroToJdbcEntryConverterTest.java      |  133 ++
 .../gobblin/source/TimestampWatermarkTest.java  |   64 +
 .../gobblin/source/jdbc/JdbcExtractorTest.java  |  148 +++
 .../gobblin/source/jdbc/MockJdbcColumn.java     |   31 +
 .../source/jdbc/OracleExtractorTest.java        |  124 ++
 .../gobblin/source/jdbc/SqlQueryUtilsTest.java  |   96 ++
 .../writer/JdbcBufferedInserterTestBase.java    |   77 ++
 .../gobblin/writer/JdbcPublisherTest.java       |  130 ++
 .../gobblin/writer/JdbcWriterCommandsTest.java  |  130 ++
 .../writer/JdbcWriterInitializerTest.java       |  162 +++
 .../apache/gobblin/writer/JdbcWriterTest.java   |   83 ++
 .../writer/MySqlBufferedInserterTest.java       |  108 ++
 .../writer/TeradataBufferedInserterTest.java    |   87 ++
 .../google/AsyncIteratorWithDataSink.java       |   93 --
 .../ingestion/google/DayPartitioner.java        |  122 --
 .../GoggleIngestionConfigurationKeys.java       |   37 -
 .../ingestion/google/util/SchemaUtil.java       |   21 -
 .../google/webmaster/GoogleWebMasterSource.java |  253 ----
 .../webmaster/GoogleWebMasterSourceDaily.java   |  105 --
 .../google/webmaster/GoogleWebmasterClient.java |   68 -
 .../webmaster/GoogleWebmasterClientImpl.java    |  104 --
 .../webmaster/GoogleWebmasterDataFetcher.java   |  114 --
 .../GoogleWebmasterDataFetcherImpl.java         |  381 ------
 .../GoogleWebmasterDayPartitioner.java          |   29 -
 .../webmaster/GoogleWebmasterExtractor.java     |  252 ----
 .../GoogleWebmasterExtractorIterator.java       |  478 -------
 .../google/webmaster/GoogleWebmasterFilter.java |  101 --
 .../ingestion/google/webmaster/ProducerJob.java |   78 --
 .../google/webmaster/SimpleProducerJob.java     |  103 --
 .../google/webmaster/TrieBasedProducerJob.java  |  111 --
 .../ingestion/google/webmaster/UrlGrouper.java  |   38 -
 .../ingestion/google/webmaster/UrlTrie.java     |   88 --
 .../ingestion/google/webmaster/UrlTrieNode.java |   95 --
 .../webmaster/UrlTriePostOrderIterator.java     |  142 --
 .../google/webmaster/UrlTriePrefixGrouper.java  |  106 --
 .../webmaster/WebmasterPerformanceTuningMetrics |   41 -
 .../GoogleAnalyticsUnsampledExtractor.java      |  387 ------
 .../google/GoogleAnalyticsUnsampledSource.java  |   58 -
 .../extractor/extract/google/GoogleCommon.java  |  241 ----
 .../extract/google/GoogleCommonKeys.java        |   25 -
 .../extract/google/GoogleDriveExtractor.java    |   35 -
 .../extract/google/GoogleDriveFileSystem.java   |  324 -----
 .../extract/google/GoogleDriveFsHelper.java     |  164 ---
 .../extract/google/GoogleDriveSource.java       |  130 --
 .../google/AsyncIteratorWithDataSink.java       |   93 ++
 .../ingestion/google/DayPartitioner.java        |  122 ++
 .../GoggleIngestionConfigurationKeys.java       |   37 +
 .../ingestion/google/util/SchemaUtil.java       |   21 +
 .../google/webmaster/GoogleWebMasterSource.java |  253 ++++
 .../webmaster/GoogleWebMasterSourceDaily.java   |  105 ++
 .../google/webmaster/GoogleWebmasterClient.java |   68 +
 .../webmaster/GoogleWebmasterClientImpl.java    |  104 ++
 .../webmaster/GoogleWebmasterDataFetcher.java   |  114 ++
 .../GoogleWebmasterDataFetcherImpl.java         |  381 ++++++
 .../GoogleWebmasterDayPartitioner.java          |   29 +
 .../webmaster/GoogleWebmasterExtractor.java     |  252 ++++
 .../GoogleWebmasterExtractorIterator.java       |  478 +++++++
 .../google/webmaster/GoogleWebmasterFilter.java |  101 ++
 .../ingestion/google/webmaster/ProducerJob.java |   78 ++
 .../google/webmaster/SimpleProducerJob.java     |  103 ++
 .../google/webmaster/TrieBasedProducerJob.java  |  111 ++
 .../ingestion/google/webmaster/UrlGrouper.java  |   38 +
 .../ingestion/google/webmaster/UrlTrie.java     |   88 ++
 .../ingestion/google/webmaster/UrlTrieNode.java |   95 ++
 .../webmaster/UrlTriePostOrderIterator.java     |  142 ++
 .../google/webmaster/UrlTriePrefixGrouper.java  |  106 ++
 .../webmaster/WebmasterPerformanceTuningMetrics |   41 +
 .../GoogleAnalyticsUnsampledExtractor.java      |  387 ++++++
 .../google/GoogleAnalyticsUnsampledSource.java  |   58 +
 .../extractor/extract/google/GoogleCommon.java  |  241 ++++
 .../extract/google/GoogleCommonKeys.java        |   25 +
 .../extract/google/GoogleDriveExtractor.java    |   35 +
 .../extract/google/GoogleDriveFileSystem.java   |  324 +++++
 .../extract/google/GoogleDriveFsHelper.java     |  164 +++
 .../extract/google/GoogleDriveSource.java       |  130 ++
 .../GoogleWebmasterDataFetcherImplTest.java     |   96 --
 .../GoogleWebmasterExtractorIteratorTest.java   |  126 --
 .../webmaster/GoogleWebmasterExtractorTest.java |   89 --
 .../google/webmaster/ProducerJobTest.java       |   56 -
 .../google/webmaster/SimpleProducerJobTest.java |   54 -
 .../webmaster/TrieBasedProducerJobTest.java     |  118 --
 .../webmaster/UrlTriePostOrderIteratorTest.java |  216 ----
 .../webmaster/UrlTriePrefixGrouperTest.java     |  194 ---
 .../ingestion/google/webmaster/UrlTrieTest.java |  163 ---
 .../GoogleAnalyticsUnsampledExtractorTest.java  |  140 --
 .../google/GoogleDriveFileSystemTest.java       |   67 -
 .../extract/google/GoogleDriveFsHelperTest.java |  192 ---
 .../filebased/GoogleDriveSourceTest.java        |   71 -
 .../GoogleWebmasterDataFetcherImplTest.java     |   96 ++
 .../GoogleWebmasterExtractorIteratorTest.java   |  126 ++
 .../webmaster/GoogleWebmasterExtractorTest.java |   89 ++
 .../google/webmaster/ProducerJobTest.java       |   56 +
 .../google/webmaster/SimpleProducerJobTest.java |   54 +
 .../webmaster/TrieBasedProducerJobTest.java     |  118 ++
 .../webmaster/UrlTriePostOrderIteratorTest.java |  216 ++++
 .../webmaster/UrlTriePrefixGrouperTest.java     |  194 +++
 .../ingestion/google/webmaster/UrlTrieTest.java |  163 +++
 .../GoogleAnalyticsUnsampledExtractorTest.java  |  140 ++
 .../google/GoogleDriveFileSystemTest.java       |   67 +
 .../extract/google/GoogleDriveFsHelperTest.java |  192 +++
 .../filebased/GoogleDriveSourceTest.java        |   71 +
 .../gobblin-oozie-example-system.properties     |    2 +-
 ...obblin-oozie-mr-example-sysconfig.properties |    2 +-
 .../gobblin.rest.jobExecutions.restspec.json    |   23 -
 ...che.gobblin.rest.jobExecutions.restspec.json |   23 +
 .../pegasus/gobblin/rest/JobExecutionInfo.pdsc  |   99 --
 .../pegasus/gobblin/rest/JobExecutionQuery.pdsc |   84 --
 .../gobblin/rest/JobExecutionQueryResult.pdsc   |   16 -
 .../src/main/pegasus/gobblin/rest/Metric.pdsc   |   32 -
 .../src/main/pegasus/gobblin/rest/Table.pdsc    |   29 -
 .../pegasus/gobblin/rest/TaskExecutionInfo.pdsc |   87 --
 .../main/pegasus/gobblin/rest/TimeRange.pdsc    |   25 -
 .../apache/gobblin/rest/JobExecutionInfo.pdsc   |   99 ++
 .../apache/gobblin/rest/JobExecutionQuery.pdsc  |   84 ++
 .../gobblin/rest/JobExecutionQueryResult.pdsc   |   16 +
 .../pegasus/org/apache/gobblin/rest/Metric.pdsc |   32 +
 .../pegasus/org/apache/gobblin/rest/Table.pdsc  |   29 +
 .../apache/gobblin/rest/TaskExecutionInfo.pdsc  |   87 ++
 .../org/apache/gobblin/rest/TimeRange.pdsc      |   25 +
 .../gobblin.rest.jobExecutions.snapshot.json    |  350 -----
 ...che.gobblin.rest.jobExecutions.snapshot.json |  350 +++++
 .../gobblin/rest/JobExecutionInfoClient.java    |  112 --
 .../gobblin/rest/JobExecutionInfoClient.java    |  112 ++
 .../gobblin/rest/JobExecutionInfoResource.java  |   95 --
 .../gobblin/rest/JobExecutionInfoServer.java    |  134 --
 .../gobblin/rest/JobExecutionInfoResource.java  |   95 ++
 .../gobblin/rest/JobExecutionInfoServer.java    |  134 ++
 .../rest/JobExecutionInfoServerTest.java        |  299 -----
 .../rest/JobExecutionInfoServerTest.java        |  299 +++++
 .../gobblin.service.flowconfigs.restspec.json   |   31 -
 .../gobblin.service.flowstatuses.restspec.json  |   29 -
 ...he.gobblin.service.flowconfigs.restspec.json |   31 +
 ...e.gobblin.service.flowstatuses.restspec.json |   29 +
 .../gobblin/service/ExecutionStatus.pdsc        |    9 -
 .../pegasus/gobblin/service/FlowConfig.pdsc     |   31 -
 .../main/pegasus/gobblin/service/FlowId.pdsc    |   29 -
 .../pegasus/gobblin/service/FlowStatistics.pdsc |   18 -
 .../pegasus/gobblin/service/FlowStatus.pdsc     |   33 -
 .../pegasus/gobblin/service/FlowStatusId.pdsc   |   23 -
 .../src/main/pegasus/gobblin/service/JobId.pdsc |   29 -
 .../main/pegasus/gobblin/service/JobState.pdsc  |   18 -
 .../pegasus/gobblin/service/JobStatistics.pdsc  |   23 -
 .../main/pegasus/gobblin/service/JobStatus.pdsc |   38 -
 .../main/pegasus/gobblin/service/Schedule.pdsc  |   22 -
 .../apache/gobblin/service/ExecutionStatus.pdsc |    9 +
 .../org/apache/gobblin/service/FlowConfig.pdsc  |   31 +
 .../org/apache/gobblin/service/FlowId.pdsc      |   29 +
 .../apache/gobblin/service/FlowStatistics.pdsc  |   18 +
 .../org/apache/gobblin/service/FlowStatus.pdsc  |   33 +
 .../apache/gobblin/service/FlowStatusId.pdsc    |   23 +
 .../org/apache/gobblin/service/JobId.pdsc       |   29 +
 .../org/apache/gobblin/service/JobState.pdsc    |   18 +
 .../apache/gobblin/service/JobStatistics.pdsc   |   23 +
 .../org/apache/gobblin/service/JobStatus.pdsc   |   38 +
 .../org/apache/gobblin/service/Schedule.pdsc    |   22 +
 .../gobblin.service.flowconfigs.snapshot.json   |  116 --
 .../gobblin.service.flowstatuses.snapshot.json  |  227 ----
 ...he.gobblin.service.flowconfigs.snapshot.json |  116 ++
 ...e.gobblin.service.flowstatuses.snapshot.json |  227 ++++
 .../java/gobblin/service/FlowConfigClient.java  |  170 ---
 .../java/gobblin/service/FlowStatusClient.java  |  138 --
 .../gobblin/service/FlowConfigClient.java       |  170 +++
 .../gobblin/service/FlowStatusClient.java       |  138 ++
 .../java/gobblin/service/FlowConfigTest.java    |  316 -----
 .../java/gobblin/service/FlowStatusTest.java    |  281 ----
 .../apache/gobblin/service/FlowConfigTest.java  |  316 +++++
 .../apache/gobblin/service/FlowStatusTest.java  |  281 ++++
 .../gobblin/service/FlowConfigsResource.java    |  296 -----
 .../gobblin/service/FlowStatusResource.java     |  208 ---
 .../service/validator/CronValidator.java        |   51 -
 .../service/validator/TemplateUriValidator.java |   70 -
 .../gobblin/service/FlowConfigsResource.java    |  296 +++++
 .../gobblin/service/FlowStatusResource.java     |  208 +++
 .../service/validator/CronValidator.java        |   51 +
 .../service/validator/TemplateUriValidator.java |   70 +
 .../gobblin/restli/EmbeddedRestliServer.java    |  176 ---
 .../gobblin/restli/SharedRestClientFactory.java |  146 ---
 .../gobblin/restli/SharedRestClientKey.java     |   39 -
 .../java/gobblin/restli/UriRestClientKey.java   |   78 --
 .../gobblin/restli/EmbeddedRestliServer.java    |  176 +++
 .../gobblin/restli/SharedRestClientFactory.java |  146 +++
 .../gobblin/restli/SharedRestClientKey.java     |   39 +
 .../apache/gobblin/restli/UriRestClientKey.java |   78 ++
 ...blin.restli.throttling.permits.restspec.json |   22 -
 ...blin.restli.throttling.permits.restspec.json |   22 +
 ...lin.restli.throttling.policies.restspec.json |   20 +
 .../restli/throttling/PermitAllocation.pdsc     |   11 -
 .../restli/throttling/PermitRequest.pdsc        |   12 -
 .../gobblin/restli/throttling/Policy.pdsc       |   13 -
 .../restli/throttling/PermitAllocation.pdsc     |   11 +
 .../restli/throttling/PermitRequest.pdsc        |   12 +
 .../gobblin/restli/throttling/Policy.pdsc       |   13 +
 ...blin.restli.throttling.permits.snapshot.json |   76 --
 ...blin.restli.throttling.permits.snapshot.json |   76 ++
 ...lin.restli.throttling.policies.snapshot.json |   55 +
 .../util/limiter/BatchedPermitsRequester.java   |  525 --------
 .../RedirectAwareRestClientRequestSender.java   |  203 ---
 .../gobblin/util/limiter/RequestSender.java     |   42 -
 .../util/limiter/RestClientRequestSender.java   |   60 -
 .../util/limiter/RestliLimiterFactory.java      |   89 --
 .../util/limiter/RestliServiceBasedLimiter.java |  101 --
 .../stressTest/FixedOperationsStressor.java     |   50 -
 .../util/limiter/stressTest/MRStressTest.java   |  363 ------
 .../stressTest/RandomDelayStartStressor.java    |   48 -
 .../stressTest/RandomRuntimeStressor.java       |   49 -
 .../RateComputingLimiterContainer.java          |  143 --
 .../limiter/stressTest/StressTestUtils.java     |   84 --
 .../util/limiter/stressTest/Stressor.java       |   32 -
 .../util/limiter/BatchedPermitsRequester.java   |  525 ++++++++
 .../RedirectAwareRestClientRequestSender.java   |  203 +++
 .../gobblin/util/limiter/RequestSender.java     |   42 +
 .../util/limiter/RestClientRequestSender.java   |   60 +
 .../util/limiter/RestliLimiterFactory.java      |   89 ++
 .../util/limiter/RestliServiceBasedLimiter.java |  101 ++
 .../stressTest/FixedOperationsStressor.java     |   50 +
 .../util/limiter/stressTest/MRStressTest.java   |  363 ++++++
 .../stressTest/RandomDelayStartStressor.java    |   48 +
 .../stressTest/RandomRuntimeStressor.java       |   49 +
 .../RateComputingLimiterContainer.java          |  143 ++
 .../limiter/stressTest/StressTestUtils.java     |   84 ++
 .../util/limiter/stressTest/Stressor.java       |   32 +
 .../restli/throttling/LocalStressTest.java      |  204 ---
 .../restli/throttling/ThrottlingClientTest.java |  130 --
 .../limiter/BatchedPermitsRequesterTest.java    |  260 ----
 .../gobblin/util/limiter/MockRequester.java     |  137 --
 .../util/limiter/RestliLimiterFactoryTest.java  |  121 --
 .../limiter/RestliServiceBasedLimiterTest.java  |  188 ---
 .../restli/throttling/LocalStressTest.java      |  204 +++
 .../restli/throttling/ThrottlingClientTest.java |  130 ++
 .../limiter/BatchedPermitsRequesterTest.java    |  260 ++++
 .../gobblin/util/limiter/MockRequester.java     |  137 ++
 .../util/limiter/RestliLimiterFactoryTest.java  |  121 ++
 .../limiter/RestliServiceBasedLimiterTest.java  |  188 +++
 .../ConfigClientBasedPolicyFactory.java         |   76 --
 .../restli/throttling/CountBasedPolicy.java     |   95 --
 .../restli/throttling/DynamicTokenBucket.java   |  101 --
 .../gobblin/restli/throttling/LeaderFinder.java |   65 -
 .../throttling/LimiterServerResource.java       |  163 ---
 .../gobblin/restli/throttling/NoopPolicy.java   |   68 -
 .../restli/throttling/PoliciesResource.java     |   78 --
 .../gobblin/restli/throttling/QPSPolicy.java    |  114 --
 .../ThrottlingGuiceServletConfig.java           |  202 ---
 .../restli/throttling/ThrottlingPolicy.java     |   41 -
 .../throttling/ThrottlingPolicyFactory.java     |   95 --
 .../throttling/ThrottlingServerScopes.java      |   74 --
 .../gobblin/restli/throttling/TokenBucket.java  |  140 --
 .../gobblin/restli/throttling/URIMetadata.java  |   38 -
 .../throttling/ZookeeperLeaderElection.java     |  223 ----
 .../ConfigClientBasedPolicyFactory.java         |   76 ++
 .../restli/throttling/CountBasedPolicy.java     |   95 ++
 .../restli/throttling/DynamicTokenBucket.java   |  101 ++
 .../gobblin/restli/throttling/LeaderFinder.java |   65 +
 .../throttling/LimiterServerResource.java       |  163 +++
 .../gobblin/restli/throttling/NoopPolicy.java   |   68 +
 .../restli/throttling/PoliciesResource.java     |   78 ++
 .../gobblin/restli/throttling/QPSPolicy.java    |  114 ++
 .../ThrottlingGuiceServletConfig.java           |  202 +++
 .../restli/throttling/ThrottlingPolicy.java     |   41 +
 .../throttling/ThrottlingPolicyFactory.java     |   95 ++
 .../throttling/ThrottlingServerScopes.java      |   74 ++
 .../gobblin/restli/throttling/TokenBucket.java  |  140 ++
 .../gobblin/restli/throttling/URIMetadata.java  |   38 +
 .../throttling/ZookeeperLeaderElection.java     |  223 ++++
 .../throttling/ConfigStoreBasedPolicyTest.java  |   62 -
 .../throttling/DynamicTokenBucketTest.java      |   52 -
 .../throttling/LimiterServerResourceTest.java   |  144 ---
 .../restli/throttling/PoliciesResourceTest.java |   61 -
 .../gobblin/restli/throttling/TestFailover.java |   98 --
 .../restli/throttling/TokenBucketTest.java      |  109 --
 .../throttling/ConfigStoreBasedPolicyTest.java  |   62 +
 .../throttling/DynamicTokenBucketTest.java      |   52 +
 .../throttling/LimiterServerResourceTest.java   |  144 +++
 .../restli/throttling/PoliciesResourceTest.java |   61 +
 .../gobblin/restli/throttling/TestFailover.java |   98 ++
 .../restli/throttling/TokenBucketTest.java      |  109 ++
 .../1/ConfigBasedPolicyTest/resource2/main.conf |    2 +-
 ...adoopKerberosKeytabAuthenticationPlugin.java |  124 --
 ...adoopKerberosKeytabAuthenticationPlugin.java |  124 ++
 ...adoopKerberosKeytabAuthenticationPlugin.java |   85 --
 ...adoopKerberosKeytabAuthenticationPlugin.java |   85 ++
 gobblin-runtime/build.gradle                    |    2 +-
 .../gobblin/runtime/fork/MockTaskContext.java   |   94 --
 .../gobblin/runtime/fork/MockTaskContext.java   |   94 ++
 gobblin-runtime/src/main/avro/AvroJobSpec.avsc  |    2 +-
 .../gobblin/runtime/AbstractJobLauncher.java    |  932 -------------
 .../runtime/AbstractTaskStateTracker.java       |  122 --
 .../java/gobblin/runtime/BaseLimiterType.java   |   78 --
 .../runtime/BoundedBlockingRecordQueue.java     |  317 -----
 .../runtime/CheckpointableWatermarkState.java   |   50 -
 .../java/gobblin/runtime/CountBasedLimiter.java |   32 -
 .../gobblin/runtime/CountUpAndDownLatch.java    |   75 --
 .../gobblin/runtime/DefaultLimiterFactory.java  |   27 -
 .../gobblin/runtime/EventMetadataUtils.java     |   75 --
 .../java/gobblin/runtime/ExecutionModel.java    |   32 -
 .../runtime/ForkBranchMismatchException.java    |   34 -
 .../java/gobblin/runtime/ForkException.java     |   33 -
 .../gobblin/runtime/FsDatasetStateStore.java    |  388 ------
 .../runtime/FsDatasetStateStoreFactory.java     |   34 -
 .../runtime/GobblinMultiTaskAttempt.java        |  381 ------
 .../main/java/gobblin/runtime/JobContext.java   |  527 --------
 .../main/java/gobblin/runtime/JobException.java |   35 -
 .../runtime/JobExecutionEventSubmitter.java     |  118 --
 .../main/java/gobblin/runtime/JobLauncher.java  |   70 -
 .../gobblin/runtime/JobLauncherFactory.java     |  123 --
 .../src/main/java/gobblin/runtime/JobState.java |  809 ------------
 .../src/main/java/gobblin/runtime/Limiter.java  |   28 -
 .../runtime/LimitingExtractorDecorator.java     |  159 ---
 .../java/gobblin/runtime/MultiConverter.java    |  215 ---
 .../gobblin/runtime/MysqlDatasetStateStore.java |  131 --
 .../runtime/MysqlDatasetStateStoreFactory.java  |   45 -
 .../gobblin/runtime/NewTaskCompletionEvent.java |   51 -
 .../gobblin/runtime/NonRefillableLimiter.java   |   28 -
 .../gobblin/runtime/NoopDatasetStateStore.java  |  106 --
 .../runtime/NoopEventMetadataGenerator.java     |   33 -
 .../java/gobblin/runtime/PoolBasedLimiter.java  |   31 -
 .../java/gobblin/runtime/RateBasedLimiter.java  |   38 -
 .../java/gobblin/runtime/SafeDatasetCommit.java |  356 -----
 .../java/gobblin/runtime/SourceDecorator.java   |  119 --
 .../StateStoreBasedWatermarkStorage.java        |  142 --
 .../StateStoreBasedWatermarkStorageCli.java     |  184 ---
 .../gobblin/runtime/StreamModelTaskRunner.java  |  142 --
 .../src/main/java/gobblin/runtime/Task.java     |  905 -------------
 .../gobblin/runtime/TaskConfigurationKeys.java  |   40 -
 .../main/java/gobblin/runtime/TaskContext.java  |  334 -----
 .../main/java/gobblin/runtime/TaskExecutor.java |  203 ---
 .../runtime/TaskInstantiationException.java     |   30 -
 .../main/java/gobblin/runtime/TaskState.java    |  463 -------
 .../runtime/TaskStateCollectorService.java      |  166 ---
 .../java/gobblin/runtime/TaskStateTracker.java  |   50 -
 .../java/gobblin/runtime/TimeBasedLimiter.java  |   37 -
 .../java/gobblin/runtime/api/Configurable.java  |   32 -
 .../runtime/api/EventMetadataGenerator.java     |   35 -
 .../main/java/gobblin/runtime/api/FlowSpec.java |  352 -----
 .../runtime/api/GobblinInstanceDriver.java      |  108 --
 .../runtime/api/GobblinInstanceEnvironment.java |   44 -
 .../runtime/api/GobblinInstanceLauncher.java    |   74 --
 .../runtime/api/GobblinInstancePlugin.java      |   31 -
 .../api/GobblinInstancePluginFactory.java       |   30 -
 .../java/gobblin/runtime/api/JobCatalog.java    |  115 --
 .../gobblin/runtime/api/JobCatalogListener.java |   94 --
 .../api/JobCatalogListenersContainer.java       |   45 -
 .../runtime/api/JobCatalogWithTemplates.java    |   38 -
 .../java/gobblin/runtime/api/JobExecution.java  |   36 -
 .../gobblin/runtime/api/JobExecutionDriver.java |   58 -
 .../runtime/api/JobExecutionLauncher.java       |   78 --
 .../gobblin/runtime/api/JobExecutionResult.java |   77 --
 .../gobblin/runtime/api/JobExecutionState.java  |  285 ----
 .../runtime/api/JobExecutionStateListener.java  |  116 --
 .../api/JobExecutionStateListenerContainer.java |   35 -
 .../gobblin/runtime/api/JobExecutionStatus.java |   28 -
 .../runtime/api/JobLifecycleListener.java       |   50 -
 .../api/JobLifecycleListenersContainer.java     |   36 -
 .../main/java/gobblin/runtime/api/JobSpec.java  |  320 -----
 .../gobblin/runtime/api/JobSpecMonitor.java     |   26 -
 .../runtime/api/JobSpecMonitorFactory.java      |   35 -
 .../runtime/api/JobSpecNotFoundException.java   |   28 -
 .../gobblin/runtime/api/JobSpecSchedule.java    |   32 -
 .../gobblin/runtime/api/JobSpecScheduler.java   |   62 -
 .../runtime/api/JobSpecSchedulerListener.java   |   77 --
 .../api/JobSpecSchedulerListenersContainer.java |   38 -
 .../java/gobblin/runtime/api/JobTemplate.java   |   64 -
 .../gobblin/runtime/api/MutableJobCatalog.java  |   40 -
 .../gobblin/runtime/api/MutableSpecCatalog.java |   40 -
 .../java/gobblin/runtime/api/SpecCatalog.java   |  112 --
 .../runtime/api/SpecCatalogListener.java        |   89 --
 .../api/SpecCatalogListenersContainer.java      |   41 -
 .../java/gobblin/runtime/api/SpecCompiler.java  |   44 -
 .../runtime/api/SpecNotFoundException.java      |   43 -
 .../java/gobblin/runtime/api/SpecSerDe.java     |   37 -
 .../java/gobblin/runtime/api/SpecStore.java     |  108 --
 .../java/gobblin/runtime/api/TopologySpec.java  |  340 -----
 .../java/gobblin/runtime/api/package-info.java  |   75 --
 .../runtime/app/ApplicationException.java       |   34 -
 .../runtime/app/ApplicationLauncher.java        |   51 -
 .../runtime/app/ServiceBasedAppLauncher.java    |  296 -----
 .../gobblin/runtime/cli/CliEmbeddedGobblin.java |  121 --
 .../java/gobblin/runtime/cli/CliOptions.java    |   94 --
 ...ructorAndPublicMethodsGobblinCliFactory.java |   42 -
 .../runtime/cli/EmbeddedGobblinCliFactory.java  |   28 -
 .../gobblin/runtime/cli/PasswordManagerCLI.java |   38 -
 .../cli/PublicMethodsGobblinCliFactory.java     |   57 -
 .../runtime/commit/DatasetStateCommitStep.java  |  125 --
 .../runtime/commit/FsCommitSequenceStore.java   |  148 ---
 .../java/gobblin/runtime/crypto/DecryptCli.java |   90 --
 .../runtime/embedded/EmbeddedGobblin.java       |  624 ---------
 .../gobblin/runtime/fork/AsynchronousFork.java  |  106 --
 .../main/java/gobblin/runtime/fork/Fork.java    |  663 ----------
 .../gobblin/runtime/fork/SynchronousFork.java   |   71 -
 .../DefaultGobblinInstanceDriverImpl.java       |  305 -----
 .../SimpleGobblinInstanceEnvironment.java       |   77 --
 .../instance/StandardGobblinInstanceDriver.java |  476 -------
 .../StandardGobblinInstanceLauncher.java        |  307 -----
 .../instance/hadoop/HadoopConfigLoader.java     |   70 -
 .../instance/plugin/BaseIdlePluginImpl.java     |   50 -
 .../runtime/job_catalog/CachingJobCatalog.java  |  152 ---
 .../runtime/job_catalog/FSJobCatalog.java       |  227 ----
 .../FSPathAlterationListenerAdaptor.java        |   87 --
 .../job_catalog/ImmutableFSJobCatalog.java      |  320 -----
 .../runtime/job_catalog/InMemoryJobCatalog.java |   91 --
 .../runtime/job_catalog/JobCatalogBase.java     |  143 --
 .../job_catalog/JobCatalogListenersList.java    |  121 --
 .../job_catalog/MutableCachingJobCatalog.java   |   49 -
 .../job_catalog/MutableJobCatalogBase.java      |   97 --
 .../job_catalog/NonObservingFSJobCatalog.java   |  124 --
 .../PackagedTemplatesJobCatalogDecorator.java   |  106 --
 .../runtime/job_catalog/StaticJobCatalog.java   |   96 --
 .../job_exec/JobLauncherExecutionDriver.java    |  574 --------
 .../gobblin/runtime/job_exec/package-info.java  |   21 -
 .../job_monitor/AvroJobSpecKafkaJobMonitor.java |  156 ---
 .../job_monitor/KafkaAvroJobMonitor.java        |  129 --
 .../runtime/job_monitor/KafkaJobMonitor.java    |  108 --
 .../job_monitor/SLAEventKafkaJobMonitor.java    |  220 ----
 .../runtime/job_spec/ResolvedJobSpec.java       |   92 --
 .../runtime/kafka/HighLevelConsumer.java        |  207 ---
 .../listeners/AbstractCloseableJobListener.java |   27 -
 .../runtime/listeners/AbstractJobListener.java  |   82 --
 .../runtime/listeners/CloseableJobListener.java |   30 -
 .../listeners/EmailNotificationJobListener.java |   86 --
 .../JobExecutionEventSubmitterListener.java     |   45 -
 .../gobblin/runtime/listeners/JobListener.java  |   62 -
 .../gobblin/runtime/listeners/JobListeners.java |  201 ---
 .../runtime/listeners/RunOnceJobListener.java   |   58 -
 .../runtime/local/CliLocalJobLauncher.java      |  102 --
 .../gobblin/runtime/local/LocalJobLauncher.java |  166 ---
 .../runtime/local/LocalTaskStateTracker.java    |  131 --
 .../locks/AbstractJobLockFactoryManager.java    |   70 -
 .../gobblin/runtime/locks/FileBasedJobLock.java |  116 --
 .../runtime/locks/FileBasedJobLockFactory.java  |  249 ----
 .../locks/FileBasedJobLockFactoryManager.java   |   57 -
 .../java/gobblin/runtime/locks/JobLock.java     |   73 --
 .../runtime/locks/JobLockEventListener.java     |   31 -
 .../gobblin/runtime/locks/JobLockException.java |   38 -
 .../gobblin/runtime/locks/JobLockFactory.java   |   33 -
 .../runtime/locks/JobLockFactoryManager.java    |   36 -
 .../locks/LegacyJobLockFactoryManager.java      |   68 -
 .../runtime/locks/ListenableJobLock.java        |   27 -
 .../runtime/locks/ZookeeperBasedJobLock.java    |  258 ----
 .../runtime/mapreduce/CliMRJobLauncher.java     |  113 --
 .../mapreduce/GobblinOutputCommitter.java       |  180 ---
 .../runtime/mapreduce/GobblinOutputFormat.java  |   35 -
 .../mapreduce/GobblinWorkUnitsInputFormat.java  |  214 ---
 .../runtime/mapreduce/MRJobLauncher.java        |  700 ----------
 .../java/gobblin/runtime/mapreduce/MRTask.java  |  128 --
 .../runtime/mapreduce/MRTaskFactory.java        |   43 -
 .../runtime/mapreduce/MRTaskStateTracker.java   |  158 ---
 .../gobblin/runtime/metrics/RuntimeMetrics.java |   38 -
 .../plugins/GobblinInstancePluginUtils.java     |   63 -
 .../runtime/plugins/PluginStaticKeys.java       |   35 -
 .../plugins/email/EmailNotificationPlugin.java  |  162 ---
 .../plugins/metrics/GobblinMetricsPlugin.java   |   58 -
 .../scheduler/AbstractJobSpecScheduler.java     |  194 ---
 .../DefaultJobSpecSchedulerListenerImpl.java    |   72 --
 .../scheduler/ImmediateJobSpecScheduler.java    |   64 -
 .../scheduler/JobSpecSchedulerListeners.java    |  103 --
 .../scheduler/QuartzJobSpecScheduler.java       |  202 ---
 .../runtime/services/JMXReportingService.java   |   82 --
 .../services/MetricsReportingService.java       |   49 -
 .../runtime/spec_catalog/FlowCatalog.java       |  267 ----
 .../spec_catalog/SpecCatalogListenersList.java  |  152 ---
 .../runtime/spec_catalog/TopologyCatalog.java   |  268 ----
 .../InMemorySpecExecutorInstanceProducer.java   |  147 ---
 .../gobblin/runtime/spec_store/FSSpecStore.java |  293 -----
 .../runtime/std/DefaultConfigurableImpl.java    |   58 -
 .../std/DefaultJobCatalogListenerImpl.java      |   73 --
 .../DefaultJobExecutionStateListenerImpl.java   |   79 --
 .../std/DefaultJobLifecycleListenerImpl.java    |  126 --
 .../runtime/std/DefaultJobSpecScheduleImpl.java |   49 -
 .../std/FilteredJobLifecycleListener.java       |  115 --
 .../runtime/std/JobExecutionStateListeners.java |  108 --
 .../runtime/std/JobExecutionUpdatable.java      |   52 -
 .../runtime/std/JobLifecycleListenersList.java  |  139 --
 .../java/gobblin/runtime/std/JobSpecFilter.java |   98 --
 .../gobblin/runtime/task/BaseAbstractTask.java  |   91 --
 .../java/gobblin/runtime/task/FailedTask.java   |   47 -
 .../java/gobblin/runtime/task/TaskFactory.java  |   41 -
 .../java/gobblin/runtime/task/TaskIFace.java    |   83 --
 .../gobblin/runtime/task/TaskIFaceWrapper.java  |  170 ---
 .../java/gobblin/runtime/task/TaskUtils.java    |   53 -
 .../template/HOCONInputStreamJobTemplate.java   |   60 -
 .../runtime/template/InheritingJobTemplate.java |  175 ---
 .../template/PullFileToConfigConverter.java     |  179 ---
 .../template/ResourceBasedJobTemplate.java      |   90 --
 .../runtime/template/StaticJobTemplate.java     |  107 --
 .../gobblin/runtime/util/ClustersNames.java     |   44 -
 .../java/gobblin/runtime/util/JobMetrics.java   |  139 --
 .../runtime/util/JobStateToJsonConverter.java   |  225 ----
 .../java/gobblin/runtime/util/MetricGroup.java  |   25 -
 .../util/MultiWorkUnitUnpackingIterator.java    |   63 -
 .../gobblin/runtime/util/RuntimeConstructs.java |   43 -
 .../java/gobblin/runtime/util/StateStores.java  |   86 --
 .../java/gobblin/runtime/util/TaskMetrics.java  |  100 --
 .../java/gobblin/scheduler/BaseGobblinJob.java  |   82 --
 .../java/gobblin/scheduler/JobScheduler.java    |  531 --------
 ...PathAlterationListenerAdaptorForMonitor.java |  258 ----
 .../java/gobblin/scheduler/SchedulerDaemon.java |   77 --
 .../gobblin/scheduler/SchedulerService.java     |   79 --
 .../gobblin/service/monitoring/FlowStatus.java  |   39 -
 .../service/monitoring/FlowStatusGenerator.java |   75 --
 .../gobblin/service/monitoring/JobStatus.java   |   46 -
 .../service/monitoring/JobStatusRetriever.java  |   46 -
 .../LatestFlowExecutionIdTracker.java           |   31 -
 .../main/java/gobblin/util/SchedulerUtils.java  |  187 ---
 .../gobblin/runtime/AbstractJobLauncher.java    |  932 +++++++++++++
 .../runtime/AbstractTaskStateTracker.java       |  122 ++
 .../apache/gobblin/runtime/BaseLimiterType.java |   78 ++
 .../runtime/BoundedBlockingRecordQueue.java     |  317 +++++
 .../runtime/CheckpointableWatermarkState.java   |   50 +
 .../gobblin/runtime/CountBasedLimiter.java      |   32 +
 .../gobblin/runtime/CountUpAndDownLatch.java    |   75 ++
 .../gobblin/runtime/DefaultLimiterFactory.java  |   27 +
 .../gobblin/runtime/EventMetadataUtils.java     |   75 ++
 .../apache/gobblin/runtime/ExecutionModel.java  |   32 +
 .../runtime/ForkBranchMismatchException.java    |   34 +
 .../apache/gobblin/runtime/ForkException.java   |   33 +
 .../gobblin/runtime/FsDatasetStateStore.java    |  388 ++++++
 .../runtime/FsDatasetStateStoreFactory.java     |   34 +
 .../runtime/GobblinMultiTaskAttempt.java        |  381 ++++++
 .../org/apache/gobblin/runtime/JobContext.java  |  527 ++++++++
 .../apache/gobblin/runtime/JobException.java    |   35 +
 .../runtime/JobExecutionEventSubmitter.java     |  118 ++
 .../org/apache/gobblin/runtime/JobLauncher.java |   70 +
 .../gobblin/runtime/JobLauncherFactory.java     |  123 ++
 .../org/apache/gobblin/runtime/JobState.java    |  809 ++++++++++++
 .../org/apache/gobblin/runtime/Limiter.java     |   28 +
 .../runtime/LimitingExtractorDecorator.java     |  159 +++
 .../apache/gobblin/runtime/MultiConverter.java  |  215 +++
 .../gobblin/runtime/MysqlDatasetStateStore.java |  131 ++
 .../runtime/MysqlDatasetStateStoreFactory.java  |   45 +
 .../gobblin/runtime/NewTaskCompletionEvent.java |   51 +
 .../gobblin/runtime/NonRefillableLimiter.java   |   28 +
 .../gobblin/runtime/NoopDatasetStateStore.java  |  106 ++
 .../runtime/NoopEventMetadataGenerator.java     |   33 +
 .../gobblin/runtime/PoolBasedLimiter.java       |   31 +
 .../gobblin/runtime/RateBasedLimiter.java       |   38 +
 .../gobblin/runtime/SafeDatasetCommit.java      |  356 +++++
 .../apache/gobblin/runtime/SourceDecorator.java |  119 ++
 .../StateStoreBasedWatermarkStorage.java        |  142 ++
 .../StateStoreBasedWatermarkStorageCli.java     |  184 +++
 .../gobblin/runtime/StreamModelTaskRunner.java  |  142 ++
 .../java/org/apache/gobblin/runtime/Task.java   |  905 +++++++++++++
 .../gobblin/runtime/TaskConfigurationKeys.java  |   40 +
 .../org/apache/gobblin/runtime/TaskContext.java |  334 +++++
 .../apache/gobblin/runtime/TaskExecutor.java    |  203 +++
 .../runtime/TaskInstantiationException.java     |   30 +
 .../org/apache/gobblin/runtime/TaskState.java   |  463 +++++++
 .../runtime/TaskStateCollectorService.java      |  166 +++
 .../gobblin/runtime/TaskStateTracker.java       |   50 +
 .../gobblin/runtime/TimeBasedLimiter.java       |   37 +
 .../gobblin/runtime/api/Configurable.java       |   32 +
 .../runtime/api/EventMetadataGenerator.java     |   35 +
 .../apache/gobblin/runtime/api/FlowSpec.java    |  352 +++++
 .../runtime/api/GobblinInstanceDriver.java      |  108 ++
 .../runtime/api/GobblinInstanceEnvironment.java |   44 +
 .../runtime/api/GobblinInstanceLauncher.java    |   74 ++
 .../runtime/api/GobblinInstancePlugin.java      |   31 +
 .../api/GobblinInstancePluginFactory.java       |   30 +
 .../apache/gobblin/runtime/api/JobCatalog.java  |  115 ++
 .../gobblin/runtime/api/JobCatalogListener.java |   94 ++
 .../api/JobCatalogListenersContainer.java       |   45 +
 .../runtime/api/JobCatalogWithTemplates.java    |   38 +
 .../gobblin/runtime/api/JobExecution.java       |   36 +
 .../gobblin/runtime/api/JobExecutionDriver.java |   58 +
 .../runtime/api/JobExecutionLauncher.java       |   78 ++
 .../gobblin/runtime/api/JobExecutionResult.java |   77 ++
 .../gobblin/runtime/api/JobExecutionState.java  |  285 ++++
 .../runtime/api/JobExecutionStateListener.java  |  116 ++
 .../api/JobExecutionStateListenerContainer.java |   35 +
 .../gobblin/runtime/api/JobExecutionStatus.java |   28 +
 .../runtime/api/JobLifecycleListener.java       |   50 +
 .../api/JobLifecycleListenersContainer.java     |   36 +
 .../org/apache/gobblin/runtime/api/JobSpec.java |  320 +++++
 .../gobblin/runtime/api/JobSpecMonitor.java     |   26 +
 .../runtime/api/JobSpecMonitorFactory.java      |   35 +
 .../runtime/api/JobSpecNotFoundException.java   |   28 +
 .../gobblin/runtime/api/JobSpecSchedule.java    |   32 +
 .../gobblin/runtime/api/JobSpecScheduler.java   |   62 +
 .../runtime/api/JobSpecSchedulerListener.java   |   77 ++
 .../api/JobSpecSchedulerListenersContainer.java |   38 +
 .../apache/gobblin/runtime/api/JobTemplate.java |   64 +
 .../gobblin/runtime/api/MutableJobCatalog.java  |   40 +
 .../gobblin/runtime/api/MutableSpecCatalog.java |   40 +
 .../apache/gobblin/runtime/api/SpecCatalog.java |  112 ++
 .../runtime/api/SpecCatalogListener.java        |   89 ++
 .../api/SpecCatalogListenersContainer.java      |   41 +
 .../gobblin/runtime/api/SpecCompiler.java       |   44 +
 .../runtime/api/SpecNotFoundException.java      |   43 +
 .../apache/gobblin/runtime/api/SpecSerDe.java   |   37 +
 .../apache/gobblin/runtime/api/SpecStore.java   |  108 ++
 .../gobblin/runtime/api/TopologySpec.java       |  340 +++++
 .../gobblin/runtime/api/package-info.java       |   75 ++
 .../runtime/app/ApplicationException.java       |   34 +
 .../runtime/app/ApplicationLauncher.java        |   51 +
 .../runtime/app/ServiceBasedAppLauncher.java    |  296 +++++
 .../gobblin/runtime/cli/CliEmbeddedGobblin.java |  121 ++
 .../apache/gobblin/runtime/cli/CliOptions.java  |   94 ++
 ...ructorAndPublicMethodsGobblinCliFactory.java |   42 +
 .../runtime/cli/EmbeddedGobblinCliFactory.java  |   28 +
 .../gobblin/runtime/cli/PasswordManagerCLI.java |   38 +
 .../cli/PublicMethodsGobblinCliFactory.java     |   57 +
 .../runtime/commit/DatasetStateCommitStep.java  |  125 ++
 .../runtime/commit/FsCommitSequenceStore.java   |  148 +++
 .../gobblin/runtime/crypto/DecryptCli.java      |   90 ++
 .../runtime/embedded/EmbeddedGobblin.java       |  624 +++++++++
 .../gobblin/runtime/fork/AsynchronousFork.java  |  106 ++
 .../org/apache/gobblin/runtime/fork/Fork.java   |  663 ++++++++++
 .../gobblin/runtime/fork/SynchronousFork.java   |   71 +
 .../DefaultGobblinInstanceDriverImpl.java       |  305 +++++
 .../SimpleGobblinInstanceEnvironment.java       |   77 ++
 .../instance/StandardGobblinInstanceDriver.java |  476 +++++++
 .../StandardGobblinInstanceLauncher.java        |  307 +++++
 .../instance/hadoop/HadoopConfigLoader.java     |   70 +
 .../instance/plugin/BaseIdlePluginImpl.java     |   50 +
 .../runtime/job_catalog/CachingJobCatalog.java  |  152 +++
 .../runtime/job_catalog/FSJobCatalog.java       |  227 ++++
 .../FSPathAlterationListenerAdaptor.java        |   87 ++
 .../job_catalog/ImmutableFSJobCatalog.java      |  320 +++++
 .../runtime/job_catalog/InMemoryJobCatalog.java |   91 ++
 .../runtime/job_catalog/JobCatalogBase.java     |  143 ++
 .../job_catalog/JobCatalogListenersList.java    |  121 ++
 .../job_catalog/MutableCachingJobCatalog.java   |   49 +
 .../job_catalog/MutableJobCatalogBase.java      |   97 ++
 .../job_catalog/NonObservingFSJobCatalog.java   |  124 ++
 .../PackagedTemplatesJobCatalogDecorator.java   |  106 ++
 .../runtime/job_catalog/StaticJobCatalog.java   |   96 ++
 .../job_exec/JobLauncherExecutionDriver.java    |  574 ++++++++
 .../gobblin/runtime/job_exec/package-info.java  |   21 +
 .../job_monitor/AvroJobSpecKafkaJobMonitor.java |  156 +++
 .../job_monitor/KafkaAvroJobMonitor.java        |  129 ++
 .../runtime/job_monitor/KafkaJobMonitor.java    |  108 ++
 .../job_monitor/SLAEventKafkaJobMonitor.java    |  220 ++++
 .../runtime/job_spec/ResolvedJobSpec.java       |   92 ++
 .../runtime/kafka/HighLevelConsumer.java        |  207 +++
 .../listeners/AbstractCloseableJobListener.java |   27 +
 .../runtime/listeners/AbstractJobListener.java  |   82 ++
 .../runtime/listeners/CloseableJobListener.java |   30 +
 .../listeners/EmailNotificationJobListener.java |   86 ++
 .../JobExecutionEventSubmitterListener.java     |   45 +
 .../gobblin/runtime/listeners/JobListener.java  |   62 +
 .../gobblin/runtime/listeners/JobListeners.java |  201 +++
 .../runtime/listeners/RunOnceJobListener.java   |   58 +
 .../runtime/local/CliLocalJobLauncher.java      |  102 ++
 .../gobblin/runtime/local/LocalJobLauncher.java |  166 +++
 .../runtime/local/LocalTaskStateTracker.java    |  131 ++
 .../locks/AbstractJobLockFactoryManager.java    |   70 +
 .../gobblin/runtime/locks/FileBasedJobLock.java |  116 ++
 .../runtime/locks/FileBasedJobLockFactory.java  |  249 ++++
 .../locks/FileBasedJobLockFactoryManager.java   |   57 +
 .../apache/gobblin/runtime/locks/JobLock.java   |   73 ++
 .../runtime/locks/JobLockEventListener.java     |   31 +
 .../gobblin/runtime/locks/JobLockException.java |   38 +
 .../gobblin/runtime/locks/JobLockFactory.java   |   33 +
 .../runtime/locks/JobLockFactoryManager.java    |   36 +
 .../locks/LegacyJobLockFactoryManager.java      |   68 +
 .../runtime/locks/ListenableJobLock.java        |   27 +
 .../runtime/locks/ZookeeperBasedJobLock.java    |  258 ++++
 .../runtime/mapreduce/CliMRJobLauncher.java     |  113 ++
 .../mapreduce/GobblinOutputCommitter.java       |  180 +++
 .../runtime/mapreduce/GobblinOutputFormat.java  |   35 +
 .../mapreduce/GobblinWorkUnitsInputFormat.java  |  214 +++
 .../runtime/mapreduce/MRJobLauncher.java        |  700 ++++++++++
 .../gobblin/runtime/mapreduce/MRTask.java       |  128 ++
 .../runtime/mapreduce/MRTaskFactory.java        |   43 +
 .../runtime/mapreduce/MRTaskStateTracker.java   |  158 +++
 .../gobblin/runtime/metrics/RuntimeMetrics.java |   38 +
 .../plugins/GobblinInstancePluginUtils.java     |   63 +
 .../runtime/plugins/PluginStaticKeys.java       |   35 +
 .../plugins/email/EmailNotificationPlugin.java  |  162 +++
 .../plugins/metrics/GobblinMetricsPlugin.java   |   58 +
 .../scheduler/AbstractJobSpecScheduler.java     |  194 +++
 .../DefaultJobSpecSchedulerListenerImpl.java    |   72 ++
 .../scheduler/ImmediateJobSpecScheduler.java    |   64 +
 .../scheduler/JobSpecSchedulerListeners.java    |  103 ++
 .../scheduler/QuartzJobSpecScheduler.java       |  202 +++
 .../runtime/services/JMXReportingService.java   |   82 ++
 .../services/MetricsReportingService.java       |   49 +
 .../runtime/spec_catalog/FlowCatalog.java       |  267 ++++
 .../spec_catalog/SpecCatalogListenersList.java  |  152 +++
 .../runtime/spec_catalog/TopologyCatalog.java   |  268 ++++
 .../InMemorySpecExecutorInstanceProducer.java   |  147 +++
 .../gobblin/runtime/spec_store/FSSpecStore.java |  293 +++++
 .../runtime/std/DefaultConfigurableImpl.java    |   58 +
 .../std/DefaultJobCatalogListenerImpl.java      |   73 ++
 .../DefaultJobExecutionStateListenerImpl.java   |   79 ++
 .../std/DefaultJobLifecycleListenerImpl.java    |  126 ++
 .../runtime/std/DefaultJobSpecScheduleImpl.java |   49 +
 .../std/FilteredJobLifecycleListener.java       |  115 ++
 .../runtime/std/JobExecutionStateListeners.java |  108 ++
 .../runtime/std/JobExecutionUpdatable.java      |   52 +
 .../runtime/std/JobLifecycleListenersList.java  |  139 ++
 .../gobblin/runtime/std/JobSpecFilter.java      |   98 ++
 .../gobblin/runtime/task/BaseAbstractTask.java  |   91 ++
 .../apache/gobblin/runtime/task/FailedTask.java |   47 +
 .../gobblin/runtime/task/TaskFactory.java       |   41 +
 .../apache/gobblin/runtime/task/TaskIFace.java  |   83 ++
 .../gobblin/runtime/task/TaskIFaceWrapper.java  |  170 +++
 .../apache/gobblin/runtime/task/TaskUtils.java  |   53 +
 .../template/HOCONInputStreamJobTemplate.java   |   60 +
 .../runtime/template/InheritingJobTemplate.java |  175 +++
 .../template/PullFileToConfigConverter.java     |  179 +++
 .../template/ResourceBasedJobTemplate.java      |   90 ++
 .../runtime/template/StaticJobTemplate.java     |  107 ++
 .../gobblin/runtime/util/ClustersNames.java     |   44 +
 .../apache/gobblin/runtime/util/JobMetrics.java |  139 ++
 .../runtime/util/JobStateToJsonConverter.java   |  225 ++++
 .../gobblin/runtime/util/MetricGroup.java       |   25 +
 .../util/MultiWorkUnitUnpackingIterator.java    |   63 +
 .../gobblin/runtime/util/RuntimeConstructs.java |   43 +
 .../gobblin/runtime/util/StateStores.java       |   86 ++
 .../gobblin/runtime/util/TaskMetrics.java       |  100 ++
 .../gobblin/scheduler/BaseGobblinJob.java       |   82 ++
 .../apache/gobblin/scheduler/JobScheduler.java  |  531 ++++++++
 ...PathAlterationListenerAdaptorForMonitor.java |  258 ++++
 .../gobblin/scheduler/SchedulerDaemon.java      |   77 ++
 .../gobblin/scheduler/SchedulerService.java     |   79 ++
 .../gobblin/service/monitoring/FlowStatus.java  |   39 +
 .../service/monitoring/FlowStatusGenerator.java |   75 ++
 .../gobblin/service/monitoring/JobStatus.java   |   46 +
 .../service/monitoring/JobStatusRetriever.java  |   46 +
 .../LatestFlowExecutionIdTracker.java           |   31 +
 .../org/apache/gobblin/util/SchedulerUtils.java |  187 +++
 .../runtime/api/GobblinInstanceLauncher.conf    |   21 -
 .../runtime/api/GobblinInstanceLauncher.conf    |   21 +
 .../main/resources/templates/distcp.template    |   12 +-
 .../resources/templates/gobblin-kafka.template  |    8 +-
 .../resources/templates/hello-world.template    |    6 +-
 .../resources/templates/kafka-to-kafka.template |    8 +-
 .../gobblin/performance/PerformanceTest.java    |  134 --
 .../runtime/BoundedBlockingRecordQueueTest.java |  135 --
 .../runtime/CountUpAndDownLatchTest.java        |   91 --
 .../gobblin/runtime/DatasetStateStoreTest.java  |  293 -----
 .../java/gobblin/runtime/DummyJobContext.java   |   98 --
 .../runtime/FsDatasetStateStoreTest.java        |  182 ---
 .../gobblin/runtime/JobBrokerInjectionTest.java |  238 ----
 .../java/gobblin/runtime/JobContextTest.java    |  279 ----
 .../runtime/JobExecutionEventSubmitterTest.java |   75 --
 .../gobblin/runtime/JobLauncherTestHelper.java  |  358 -----
 .../java/gobblin/runtime/JobListenersTest.java  |   54 -
 .../test/java/gobblin/runtime/JobStateTest.java |  210 ---
 .../gobblin/runtime/LimiterStopEventTest.java   |  144 ---
 .../gobblin/runtime/MultiConverterTest.java     |  394 ------
 .../runtime/MysqlDatasetStateStoreTest.java     |  239 ----
 .../java/gobblin/runtime/TaskContextTest.java   |   82 --
 .../gobblin/runtime/TaskContinuousTest.java     |  543 --------
 .../runtime/TaskStateCollectorServiceTest.java  |  118 --
 .../java/gobblin/runtime/TaskStateTest.java     |  119 --
 .../src/test/java/gobblin/runtime/TaskTest.java |  562 --------
 .../java/gobblin/runtime/TestRecordStream.java  |  348 -----
 .../runtime/TestWorkUnitStreamSource.java       |  191 ---
 .../runtime/api/TestJobExecutionState.java      |  269 ----
 .../java/gobblin/runtime/api/TestJobSpec.java   |  152 ---
 .../runtime/commit/CommitSequenceTest.java      |  100 --
 .../commit/FsCommitSequenceStoreTest.java       |  103 --
 .../runtime/embedded/EmbeddedGobblinTest.java   |   95 --
 .../TestDefaultGobblinInstanceDriverImpl.java   |  140 --
 .../TestStandardGobblinInstanceDriver.java      |   99 --
 .../TestStandardGobblinInstanceLauncher.java    |  282 ----
 .../instance/hadoop/TestHadoopConfigLoader.java |   53 -
 .../job_catalog/FSJobCatalogHelperTest.java     |  307 -----
 ...ackagedTemplatesJobCatalogDecoratorTest.java |  103 --
 .../runtime/job_catalog/TestFSJobCatalog.java   |  164 ---
 .../job_catalog/TestImmutableFSJobCatalog.java  |   74 --
 .../job_catalog/TestInMemoryJobCatalog.java     |  195 ---
 .../TestJobCatalogListenersList.java            |   75 --
 .../TestMutableCachingJobCatalog.java           |  106 --
 .../TestNonObservingFSJobCatalog.java           |  145 ---
 .../TestJobLauncherExecutionDriver.java         |  114 --
 .../job_monitor/KafkaAvroJobMonitorTest.java    |  140 --
 .../job_monitor/KafkaJobMonitorTest.java        |   63 -
 .../runtime/job_monitor/MockKafkaStream.java    |  106 --
 .../job_monitor/MockedKafkaJobMonitor.java      |  152 ---
 .../SLAEventKafkaJobMonitorTest.java            |  174 ---
 .../runtime/kafka/HighLevelConsumerTest.java    |   75 --
 .../runtime/kafka/MockedHighLevelConsumer.java  |   83 --
 .../runtime/local/LocalJobLauncherTest.java     |  225 ----
 .../FileBasedJobLockFactoryManagerTest.java     |   82 --
 .../runtime/locks/FileBasedJobLockTest.java     |   70 -
 .../java/gobblin/runtime/locks/JobLockTest.java |  149 ---
 .../locks/LegacyJobLockFactoryManagerTest.java  |  126 --
 .../locks/ZookeeperBasedJobLockTest.java        |   66 -
 .../mapreduce/GobblinOutputCommitterTest.java   |  145 ---
 .../GobblinWorkUnitsInputFormatTest.java        |  182 ---
 .../runtime/mapreduce/MRJobLauncherTest.java    |  309 -----
 .../runtime/mapreduce/MRTaskFactoryTest.java    |  196 ---
 .../TestImmediateJobSpecScheduler.java          |  103 --
 .../scheduler/TestQuartzJobSpecScheduler.java   |  165 ---
 .../std/TestDefaultConfigurableImpl.java        |   66 -
 .../std/TestFilteredJobLifecycleListener.java   |   96 --
 .../std/TestJobExecutionStateListeners.java     |   88 --
 .../std/TestJobLifecycleListenersList.java      |   68 -
 .../gobblin/runtime/std/TestJobSpecFilter.java  |   49 -
 .../template/InheritingJobTemplateTest.java     |  232 ----
 .../template/ResourceBasedJobTemplateTest.java  |   52 -
 .../runtime/template/StaticJobTemplateTest.java |   77 --
 .../gobblin/runtime/util/JobMetricsTest.java    |  103 --
 .../util/JobStateToJsonConverterTest.java       |   55 -
 .../MultiWorkUnitUnpackingIteratorTest.java     |   71 -
 .../gobblin/runtime/util/TaskMetricsTest.java   |   64 -
 .../scheduler/JobConfigFileMonitorTest.java     |  195 ---
 .../filebased/TextFileBasedSourceTest.java      |  132 --
 .../gobblin/spec_catalog/FlowCatalogTest.java   |  174 ---
 .../spec_catalog/TopologyCatalogTest.java       |  169 ---
 .../test/java/gobblin/task/CustomTaskTest.java  |  105 --
 .../task/EventBusPublishingTaskFactory.java     |  220 ----
 .../test/java/gobblin/test/TestConverter.java   |   64 -
 .../test/java/gobblin/test/TestConverter2.java  |   85 --
 .../java/gobblin/test/TestDataPublisher.java    |   72 --
 .../test/java/gobblin/test/TestExtractor.java   |  119 --
 .../src/test/java/gobblin/test/TestSource.java  |   83 --
 .../java/gobblin/util/SchedulerUtilsTest.java   |  314 -----
 .../test/java/gobblin/util/TemplateTest.java    |  123 --
 .../gobblin/performance/PerformanceTest.java    |  134 ++
 .../runtime/BoundedBlockingRecordQueueTest.java |  135 ++
 .../runtime/CountUpAndDownLatchTest.java        |   91 ++
 .../gobblin/runtime/DatasetStateStoreTest.java  |  293 +++++
 .../apache/gobblin/runtime/DummyJobContext.java |   98 ++
 .../runtime/FsDatasetStateStoreTest.java        |  182 +++
 .../gobblin/runtime/JobBrokerInjectionTest.java |  238 ++++
 .../apache/gobblin/runtime/JobContextTest.java  |  279 ++++
 .../runtime/JobExecutionEventSubmitterTest.java |   75 ++
 .../gobblin/runtime/JobLauncherTestHelper.java  |  358 +++++
 .../gobblin/runtime/JobListenersTest.java       |   54 +
 .../apache/gobblin/runtime/JobStateTest.java    |  210 +++
 .../gobblin/runtime/LimiterStopEventTest.java   |  144 +++
 .../gobblin/runtime/MultiConverterTest.java     |  394 ++++++
 .../runtime/MysqlDatasetStateStoreTest.java     |  239 ++++
 .../apache/gobblin/runtime/TaskContextTest.java |   82 ++
 .../gobblin/runtime/TaskContinuousTest.java     |  543 ++++++++
 .../runtime/TaskStateCollectorServiceTest.java  |  118 ++
 .../apache/gobblin/runtime/TaskStateTest.java   |  119 ++
 .../org/apache/gobblin/runtime/TaskTest.java    |  562 ++++++++
 .../gobblin/runtime/TestRecordStream.java       |  348 +++++
 .../runtime/TestWorkUnitStreamSource.java       |  191 +++
 .../runtime/api/TestJobExecutionState.java      |  269 ++++
 .../apache/gobblin/runtime/api/TestJobSpec.java |  152 +++
 .../runtime/commit/CommitSequenceTest.java      |  100 ++
 .../commit/FsCommitSequenceStoreTest.java       |  103 ++
 .../runtime/embedded/EmbeddedGobblinTest.java   |   95 ++
 .../TestDefaultGobblinInstanceDriverImpl.java   |  140 ++
 .../TestStandardGobblinInstanceDriver.java      |   99 ++
 .../TestStandardGobblinInstanceLauncher.java    |  282 ++++
 .../instance/hadoop/TestHadoopConfigLoader.java |   53 +
 .../job_catalog/FSJobCatalogHelperTest.java     |  307 +++++
 ...ackagedTemplatesJobCatalogDecoratorTest.java |  103 ++
 .../runtime/job_catalog/TestFSJobCatalog.java   |  164 +++
 .../job_catalog/TestImmutableFSJobCatalog.java  |   74 ++
 .../job_catalog/TestInMemoryJobCatalog.java     |  195 +++
 .../TestJobCatalogListenersList.java            |   75 ++
 .../TestMutableCachingJobCatalog.java           |  106 ++
 .../TestNonObservingFSJobCatalog.java           |  145 +++
 .../TestJobLauncherExecutionDriver.java         |  114 ++
 .../job_monitor/KafkaAvroJobMonitorTest.java    |  140 ++
 .../job_monitor/KafkaJobMonitorTest.java        |   63 +
 .../runtime/job_monitor/MockKafkaStream.java    |  106 ++
 .../job_monitor/MockedKafkaJobMonitor.java      |  152 +++
 .../SLAEventKafkaJobMonitorTest.java            |  174 +++
 .../runtime/kafka/HighLevelConsumerTest.java    |   75 ++
 .../runtime/kafka/MockedHighLevelConsumer.java  |   83 ++
 .../runtime/local/LocalJobLauncherTest.java     |  225 ++++
 .../FileBasedJobLockFactoryManagerTest.java     |   82 ++
 .../runtime/locks/FileBasedJobLockTest.java     |   70 +
 .../gobblin/runtime/locks/JobLockTest.java      |  149 +++
 .../locks/LegacyJobLockFactoryManagerTest.java  |  126 ++
 .../locks/ZookeeperBasedJobLockTest.java        |   66 +
 .../mapreduce/GobblinOutputCommitterTest.java   |  145 +++
 .../GobblinWorkUnitsInputFormatTest.java        |  182 +++
 .../runtime/mapreduce/MRJobLauncherTest.java    |  309 +++++
 .../runtime/mapreduce/MRTaskFactoryTest.java    |  196 +++
 .../TestImmediateJobSpecScheduler.java          |  103 ++
 .../scheduler/TestQuartzJobSpecScheduler.java   |  165 +++
 .../std/TestDefaultConfigurableImpl.java        |   66 +
 .../std/TestFilteredJobLifecycleListener.java   |   96 ++
 .../std/TestJobExecutionStateListeners.java     |   88 ++
 .../std/TestJobLifecycleListenersList.java      |   68 +
 .../gobblin/runtime/std/TestJobSpecFilter.java  |   49 +
 .../template/InheritingJobTemplateTest.java     |  232 ++++
 .../template/ResourceBasedJobTemplateTest.java  |   52 +
 .../runtime/template/StaticJobTemplateTest.java |   77 ++
 .../gobblin/runtime/util/JobMetricsTest.java    |  103 ++
 .../util/JobStateToJsonConverterTest.java       |   56 +
 .../MultiWorkUnitUnpackingIteratorTest.java     |   71 +
 .../gobblin/runtime/util/TaskMetricsTest.java   |   64 +
 .../scheduler/JobConfigFileMonitorTest.java     |  195 +++
 .../filebased/TextFileBasedSourceTest.java      |  132 ++
 .../gobblin/spec_catalog/FlowCatalogTest.java   |  174 +++
 .../spec_catalog/TopologyCatalogTest.java       |  169 +++
 .../org/apache/gobblin/task/CustomTaskTest.java |  105 ++
 .../task/EventBusPublishingTaskFactory.java     |  220 ++++
 .../org/apache/gobblin/test/TestConverter.java  |   64 +
 .../org/apache/gobblin/test/TestConverter2.java |   85 ++
 .../apache/gobblin/test/TestDataPublisher.java  |   72 ++
 .../org/apache/gobblin/test/TestExtractor.java  |  119 ++
 .../org/apache/gobblin/test/TestSource.java     |   83 ++
 .../apache/gobblin/util/SchedulerUtilsTest.java |  314 +++++
 .../org/apache/gobblin/util/TemplateTest.java   |  123 ++
 .../brokerTest/SimpleHelloWorldJob.jobconf      |    8 +-
 .../instance/SimpleHelloWorldJob.jobconf        |    6 +-
 .../instance/SimpleHelloWorldJob.template       |    6 +-
 .../resources/mr-job-conf/GobblinMRTest.pull    |   12 +-
 .../resources/templates/distcp-ng-hive.template |    8 +-
 .../test/resources/templates/distcp-ng.template |   14 +-
 .../templates/performanceTest.template          |    4 +-
 .../templates/textFileBasedSourceTest.template  |    4 +-
 .../salesforce/SalesforceConfigurationKeys.java |   25 -
 .../gobblin/salesforce/SalesforceConnector.java |   97 --
 .../gobblin/salesforce/SalesforceExtractor.java |  804 ------------
 .../gobblin/salesforce/SalesforceSource.java    |  331 -----
 .../salesforce/SalesforceConfigurationKeys.java |   25 +
 .../gobblin/salesforce/SalesforceConnector.java |   97 ++
 .../gobblin/salesforce/SalesforceExtractor.java |  804 ++++++++++++
 .../gobblin/salesforce/SalesforceSource.java    |  331 +++++
 .../main/resources/azkaban/common.properties    |   10 +-
 .../resources/stand-alone/common.properties     |    8 +-
 .../salesforce/SalesforceSourceTest.java        |   25 -
 .../salesforce/SalesforceSourceTest.java        |   25 +
 .../main/java/gobblin/service/HelixUtils.java   |  110 --
 .../java/gobblin/service/ServiceConfigKeys.java |   77 --
 .../gobblin/service/ServiceMetricNames.java     |   31 -
 .../modules/core/GobblinServiceManager.java     |  552 --------
 .../flow/IdentityFlowToJobSpecCompiler.java     |  260 ----
 .../modules/orchestration/Orchestrator.java     |  237 ----
 .../scheduler/GobblinServiceJobScheduler.java   |  314 -----
 .../ConfigBasedTopologySpecFactory.java         |  104 --
 .../modules/topology/TopologySpecFactory.java   |   36 -
 .../org/apache/gobblin/service/HelixUtils.java  |  110 ++
 .../gobblin/service/ServiceConfigKeys.java      |   77 ++
 .../gobblin/service/ServiceMetricNames.java     |   31 +
 .../modules/core/GobblinServiceManager.java     |  552 ++++++++
 .../flow/IdentityFlowToJobSpecCompiler.java     |  260 ++++
 .../modules/orchestration/Orchestrator.java     |  237 ++++
 .../scheduler/GobblinServiceJobScheduler.java   |  314 +++++
 .../ConfigBasedTopologySpecFactory.java         |  104 ++
 .../modules/topology/TopologySpecFactory.java   |   36 +
 .../modules/core/GobblinServiceHATest.java      |  509 --------
 .../modules/core/GobblinServiceManagerTest.java |  298 -----
 .../core/IdentityFlowToJobSpecCompilerTest.java |  253 ----
 .../modules/orchestration/OrchestratorTest.java |  290 -----
 .../ConfigBasedTopologySpecFactoryTest.java     |   97 --
 .../modules/core/GobblinServiceHATest.java      |  509 ++++++++
 .../modules/core/GobblinServiceManagerTest.java |  298 +++++
 .../core/IdentityFlowToJobSpecCompilerTest.java |  253 ++++
 .../modules/orchestration/OrchestratorTest.java |  290 +++++
 .../ConfigBasedTopologySpecFactoryTest.java     |   97 ++
 .../dataManagement/copy/job-props/copy.pull     |   10 +-
 .../test/execution/operator/SetupOperator.java  |   32 -
 .../execution/validator/ValidationRule.java     |   34 -
 .../test/setup/config/ConfigStepsGenerator.java |   36 -
 .../java/gobblin/test/setup/config/Step.java    |   41 -
 .../test/setup/config/TestHarnessLauncher.java  |   37 -
 .../test/execution/operator/SetupOperator.java  |   32 +
 .../execution/validator/ValidationRule.java     |   34 +
 .../test/setup/config/ConfigStepsGenerator.java |   36 +
 .../apache/gobblin/test/setup/config/Step.java  |   41 +
 .../test/setup/config/TestHarnessLauncher.java  |   37 +
 .../gobblin/GobblinLocalJobLauncherUtils.java   |   67 -
 .../gobblin/SkipWorkUnitsIntegrationTest.java   |   80 --
 .../test/java/gobblin/TestAvroExtractor.java    |   87 --
 .../src/test/java/gobblin/TestAvroSource.java   |   36 -
 .../gobblin/TestSkipWorkUnitsPublisher.java     |   59 -
 .../java/gobblin/TestSkipWorkUnitsSource.java   |   78 --
 .../WriterOutputFormatIntegrationTest.java      |   63 -
 .../data/management/CopyIntegrationTest.java    |   86 --
 .../gobblin/GobblinLocalJobLauncherUtils.java   |   67 +
 .../gobblin/SkipWorkUnitsIntegrationTest.java   |   80 ++
 .../org/apache/gobblin/TestAvroExtractor.java   |   87 ++
 .../java/org/apache/gobblin/TestAvroSource.java |   36 +
 .../gobblin/TestSkipWorkUnitsPublisher.java     |   59 +
 .../apache/gobblin/TestSkipWorkUnitsSource.java |   78 ++
 .../WriterOutputFormatIntegrationTest.java      |   63 +
 .../data/management/CopyIntegrationTest.java    |   86 ++
 .../runtime_test/skip_workunits_test.properties |    2 +-
 .../writer_output_format_test.properties        |    6 +-
 .../java/gobblin/test/ConstantTimingType.java   |   33 -
 .../main/java/gobblin/test/ErrorManager.java    |  106 --
 .../main/java/gobblin/test/NthTimingType.java   |   45 -
 .../src/main/java/gobblin/test/TestUtils.java   |  103 --
 .../main/java/gobblin/test/TimingManager.java   |   36 -
 .../main/java/gobblin/test/TimingResult.java    |   31 -
 .../src/main/java/gobblin/test/TimingType.java  |   25 -
 .../gobblin/test/crypto/InsecureShiftCodec.java |  107 --
 .../test/crypto/TestEncryptionProvider.java     |   57 -
 .../test/crypto/TestRandomCredentialStore.java  |   56 -
 .../apache/gobblin/test/ConstantTimingType.java |   33 +
 .../org/apache/gobblin/test/ErrorManager.java   |  106 ++
 .../org/apache/gobblin/test/NthTimingType.java  |   45 +
 .../java/org/apache/gobblin/test/TestUtils.java |  103 ++
 .../org/apache/gobblin/test/TimingManager.java  |   36 +
 .../org/apache/gobblin/test/TimingResult.java   |   31 +
 .../org/apache/gobblin/test/TimingType.java     |   25 +
 .../gobblin/test/crypto/InsecureShiftCodec.java |  107 ++
 .../test/crypto/TestEncryptionProvider.java     |   57 +
 .../test/crypto/TestRandomCredentialStore.java  |   56 +
 .../gobblin.crypto.CredentialStoreProvider      |    1 -
 .../services/gobblin.crypto.EncryptionProvider  |    1 -
 ...pache.gobblin.crypto.CredentialStoreProvider |    1 +
 ...org.apache.gobblin.crypto.EncryptionProvider |    1 +
 .../java/gobblin/test/ErrorManagerTest.java     |   65 -
 .../gobblin/test/RandomCredentialStoreTest.java |   57 -
 .../apache/gobblin/test/ErrorManagerTest.java   |   65 +
 .../gobblin/test/RandomCredentialStoreTest.java |   57 +
 .../resource/job-conf/GobblinTest1.pull         |   12 +-
 .../resource/job-conf/GobblinTest2.pull         |   12 +-
 .../resource/job-conf/GobblinTest3.pull         |   12 +-
 .../resource/mr-job-conf/GobblinMRTest.pull     |   12 +-
 .../main/java/gobblin/tunnel/AcceptHandler.java |   58 -
 .../src/main/java/gobblin/tunnel/Config.java    |   49 -
 .../main/java/gobblin/tunnel/HandlerState.java  |   28 -
 .../java/gobblin/tunnel/ProxySetupHandler.java  |  189 ---
 .../java/gobblin/tunnel/ReadWriteHandler.java   |  205 ---
 .../src/main/java/gobblin/tunnel/Tunnel.java    |  190 ---
 .../apache/gobblin/tunnel/AcceptHandler.java    |   58 +
 .../java/org/apache/gobblin/tunnel/Config.java  |   49 +
 .../org/apache/gobblin/tunnel/HandlerState.java |   28 +
 .../gobblin/tunnel/ProxySetupHandler.java       |  189 +++
 .../apache/gobblin/tunnel/ReadWriteHandler.java |  205 +++
 .../java/org/apache/gobblin/tunnel/Tunnel.java  |  190 +++
 .../java/gobblin/tunnel/ConnectProxyServer.java |  173 ---
 .../java/gobblin/tunnel/DoubleEchoServer.java   |   59 -
 .../test/java/gobblin/tunnel/EasyThread.java    |   53 -
 .../test/java/gobblin/tunnel/MockServer.java    |  130 --
 .../tunnel/TalkFirstDoubleEchoServer.java       |   62 -
 .../java/gobblin/tunnel/TalkPastServer.java     |   94 --
 .../TestTunnelWithArbitraryTCPTraffic.java      |  625 ---------
 .../test/java/gobblin/tunnel/TunnelTest.java    |  256 ----
 .../gobblin/tunnel/ConnectProxyServer.java      |  173 +++
 .../apache/gobblin/tunnel/DoubleEchoServer.java |   59 +
 .../org/apache/gobblin/tunnel/EasyThread.java   |   53 +
 .../org/apache/gobblin/tunnel/MockServer.java   |  130 ++
 .../tunnel/TalkFirstDoubleEchoServer.java       |   62 +
 .../apache/gobblin/tunnel/TalkPastServer.java   |   94 ++
 .../TestTunnelWithArbitraryTCPTraffic.java      |  625 +++++++++
 .../org/apache/gobblin/tunnel/TunnelTest.java   |  256 ++++
 .../src/main/bash/gobblin_password_encryptor.sh |    2 +-
 .../broker/BrokerConfigurationKeyGenerator.java |   59 -
 .../java/gobblin/broker/DefaultBrokerCache.java |  212 ---
 .../src/main/java/gobblin/broker/EmptyKey.java  |   34 -
 .../broker/KeyedScopedConfigViewImpl.java       |   91 --
 .../gobblin/broker/NonExtendableBrokerView.java |   41 -
 .../java/gobblin/broker/ResourceInstance.java   |   41 -
 .../main/java/gobblin/broker/ScopeWrapper.java  |   37 -
 .../broker/SharedResourcesBrokerFactory.java    |  155 ---
 .../broker/SharedResourcesBrokerImpl.java       |  328 -----
 .../broker/SharedResourcesBrokerUtils.java      |   89 --
 .../java/gobblin/broker/TTLResourceEntry.java   |   50 -
 .../main/java/gobblin/broker/package-info.java  |   46 -
 .../copy/hive/WhitelistBlacklist.java           |  149 ---
 .../gobblin/runtime/cli/CliApplication.java     |   30 -
 .../gobblin/runtime/cli/CliObjectFactory.java   |   61 -
 .../gobblin/runtime/cli/CliObjectOption.java    |   41 -
 .../gobblin/runtime/cli/CliObjectSupport.java   |   40 -
 ...tructorAndPublicMethodsCliObjectFactory.java |  134 --
 .../java/gobblin/runtime/cli/GobblinCli.java    |   61 -
 .../main/java/gobblin/runtime/cli/NotOnCli.java |   31 -
 .../cli/PublicMethodsCliObjectFactory.java      |  221 ----
 .../gobblin/util/ApplicationLauncherUtils.java  |   35 -
 .../java/gobblin/util/AutoCloseableLock.java    |   49 -
 .../java/gobblin/util/AutoReturnableObject.java |   70 -
 .../main/java/gobblin/util/AvroFlattener.java   |  489 -------
 .../src/main/java/gobblin/util/AvroUtils.java   |  655 ----------
 .../src/main/java/gobblin/util/AzkabanTags.java |   71 -
 .../java/gobblin/util/CLIPasswordEncryptor.java |  149 ---
 .../main/java/gobblin/util/ClusterNameTags.java |   62 -
 .../main/java/gobblin/util/ClustersNames.java   |  180 ---
 .../src/main/java/gobblin/util/ConfigUtils.java |  507 --------
 .../java/gobblin/util/DatasetFilterUtils.java   |  100 --
 .../java/gobblin/util/DatePartitionType.java    |  100 --
 .../src/main/java/gobblin/util/Either.java      |  109 --
 .../src/main/java/gobblin/util/EmailUtils.java  |  125 --
 .../main/java/gobblin/util/EmptyIterable.java   |   38 -
 .../main/java/gobblin/util/ExecutorsUtils.java  |  305 -----
 .../java/gobblin/util/ExponentialBackoff.java   |  130 --
 .../main/java/gobblin/util/FileListUtils.java   |  175 ---
 .../java/gobblin/util/ForkOperatorUtils.java    |  114 --
 .../src/main/java/gobblin/util/HadoopUtils.java |  923 -------------
 .../java/gobblin/util/HeapDumpForTaskUtils.java |   87 --
 .../java/gobblin/util/HiveJdbcConnector.java    |  288 -----
 .../src/main/java/gobblin/util/HostUtils.java   |   19 -
 .../src/main/java/gobblin/util/Id.java          |  187 ---
 .../java/gobblin/util/ImmutableProperties.java  |   71 -
 .../gobblin/util/JobConfigurationUtils.java     |  131 --
 .../java/gobblin/util/JobLauncherUtils.java     |  279 ----
 .../src/main/java/gobblin/util/JvmUtils.java    |   40 -
 .../util/LoggingUncaughtExceptionHandler.java   |   47 -
 .../main/java/gobblin/util/NoopCloseable.java   |   34 -
 .../main/java/gobblin/util/ParallelRunner.java  |  380 ------
 .../src/main/java/gobblin/util/PathUtils.java   |  199 ---
 .../src/main/java/gobblin/util/PortUtils.java   |  165 ---
 .../main/java/gobblin/util/PropertiesUtils.java |   82 --
 .../gobblin/util/ProxiedFileSystemCache.java    |  345 -----
 .../gobblin/util/ProxiedFileSystemUtils.java    |  255 ----
 .../gobblin/util/ProxiedFileSystemWrapper.java  |  166 ---
 .../main/java/gobblin/util/PublisherUtils.java  |  111 --
 .../main/java/gobblin/util/PullFileLoader.java  |  271 ----
 .../gobblin/util/RateControlledFileSystem.java  |  114 --
 .../java/gobblin/util/SerializationUtils.java   |  193 ---
 .../java/gobblin/util/StringParsingUtils.java   |   83 --
 .../main/java/gobblin/util/TemplateUtils.java   |   44 -
 .../java/gobblin/util/TimeRangeChecker.java     |  108 --
 .../gobblin/util/WritableShimSerialization.java |  133 --
 .../src/main/java/gobblin/util/WriterUtils.java |  335 -----
 .../gobblin/util/binpacking/FieldWeighter.java  |   38 -
 .../WorstFitDecreasingBinPacking.java           |  168 ---
 .../java/gobblin/util/callbacks/Callback.java   |   35 -
 .../gobblin/util/callbacks/CallbackResult.java  |   76 --
 .../util/callbacks/CallbacksDispatcher.java     |  204 ---
 .../gobblin/util/callbacks/package-info.java    |   63 -
 .../gobblin/util/concurrent/AutoResetEvent.java |   71 -
 .../util/concurrent/CancellableTask.java        |   44 -
 .../HashedWheelTimerTaskScheduler.java          |  170 ---
 .../ScheduledExecutorServiceTaskScheduler.java  |  153 ---
 .../gobblin/util/concurrent/ScheduledTask.java  |   37 -
 .../gobblin/util/concurrent/TaskScheduler.java  |  169 ---
 .../util/concurrent/TaskSchedulerFactory.java   |   66 -
 .../util/concurrent/TaskSchedulerType.java      |   63 -
 .../java/gobblin/util/dataset/DatasetUtils.java |  134 --
 .../util/deprecation/DeprecationUtils.java      |   56 -
 .../util/executors/ForceQueuePolicy.java        |   39 -
 .../util/executors/IteratorExecutor.java        |  172 ---
 .../util/executors/MDCPropagatingCallable.java  |   49 -
 .../MDCPropagatingExecutorService.java          |   58 -
 .../util/executors/MDCPropagatingRunnable.java  |   44 -
 .../MDCPropagatingScheduledExecutorService.java |  145 ---
 .../gobblin/util/executors/ScalingQueue.java    |   77 --
 .../executors/ScalingThreadPoolExecutor.java    |   67 -
 ...CatchingPathAlterationListenerDecorator.java |  114 --
 .../util/filesystem/FileStatusEntry.java        |  185 ---
 .../util/filesystem/FileSystemDecorator.java    |  533 --------
 .../util/filesystem/FileSystemFactory.java      |  113 --
 .../filesystem/FileSystemInstrumentation.java   |   62 -
 .../FileSystemInstrumentationFactory.java       |   43 -
 .../gobblin/util/filesystem/FileSystemKey.java  |   91 --
 .../util/filesystem/FileSystemLimiterKey.java   |   66 -
 .../util/filesystem/InstrumentedFileSystem.java |   54 -
 .../filesystem/InstrumentedFileSystemUtils.java |  102 --
 .../filesystem/InstrumentedHDFSFileSystem.java  |   36 -
 .../filesystem/InstrumentedLocalFileSystem.java |   37 -
 .../InstrumentedWebHDFSFileSystem.java          |   36 -
 .../util/filesystem/PathAlterationListener.java |   46 -
 .../PathAlterationListenerAdaptor.java          |   47 -
 .../util/filesystem/PathAlterationObserver.java |  346 -----
 .../PathAlterationObserverScheduler.java        |  189 ---
 .../util/filesystem/ThrottledFileSystem.java    |  206 ---
 .../gobblin/util/filters/AndPathFilter.java     |   44 -
 .../java/gobblin/util/filters/HiddenFilter.java |   42 -
 .../gobblin/util/filters/RegexPathFilter.java   |   52 -
 .../gobblin/util/filters/TarGpgPathFilter.java  |   32 -
 .../src/main/java/gobblin/util/guid/Guid.java   |  209 ---
 .../main/java/gobblin/util/guid/HasGuid.java    |   34 -
 .../java/gobblin/util/hadoop/TokenUtils.java    |  252 ----
 .../java/gobblin/util/http/HttpLimiterKey.java  |   51 -
 .../util/io/AdditionalCloseableInputStream.java |   49 -
 .../gobblin/util/io/BatchedMeterDecorator.java  |   70 -
 .../java/gobblin/util/io/CloseableHttpConn.java |   40 -
 .../gobblin/util/io/FilterStreamUnpacker.java   |   62 -
 .../gobblin/util/io/GsonInterfaceAdapter.java   |  195 ---
 .../gobblin/util/io/MeteredInputStream.java     |   93 --
 .../gobblin/util/io/MeteredOutputStream.java    |   89 --
 .../java/gobblin/util/io/MeteredStream.java     |   31 -
 .../gobblin/util/io/SeekableFSInputStream.java  |   77 --
 .../main/java/gobblin/util/io/StreamCopier.java |  149 ---
 .../util/io/StreamCopierSharedLimiterKey.java   |   89 --
 .../java/gobblin/util/io/StreamThrottler.java   |  110 --
 .../main/java/gobblin/util/io/StreamUtils.java  |  239 ----
 .../gobblin/util/io/ThrottledInputStream.java   |   96 --
 .../util/iterators/InterruptibleIterator.java   |   67 -
 .../gobblin/util/jdbc/DataSourceBuilder.java    |  145 ---
 .../gobblin/util/jdbc/DataSourceModule.java     |   45 -
 .../gobblin/util/jdbc/DataSourceProvider.java   |   76 --
 .../main/java/gobblin/util/json/JsonUtils.java  |   43 -
 .../gobblin/util/limiter/BaseLimiterType.java   |   79 --
 .../gobblin/util/limiter/CountBasedLimiter.java |   88 --
 .../util/limiter/DefaultLimiterFactory.java     |  141 --
 .../main/java/gobblin/util/limiter/Limiter.java |   60 -
 .../gobblin/util/limiter/LimiterFactory.java    |   28 -
 .../java/gobblin/util/limiter/MultiLimiter.java |  106 --
 .../util/limiter/NonRefillableLimiter.java      |   44 -
 .../java/gobblin/util/limiter/NoopLimiter.java  |   48 -
 .../util/limiter/NotEnoughPermitsException.java |   36 -
 .../gobblin/util/limiter/PoolBasedLimiter.java  |   99 --
 .../gobblin/util/limiter/RateBasedLimiter.java  |   92 --
 .../gobblin/util/limiter/TimeBasedLimiter.java  |  105 --
 .../limiter/broker/SharedLimiterFactory.java    |  149 ---
 .../util/limiter/broker/SharedLimiterKey.java   |   66 -
 .../util/logs/Log4jConfigurationHelper.java     |   70 -
 .../main/java/gobblin/util/logs/LogCopier.java  |  580 ---------
 .../CompactionRecordCountProvider.java          |   95 --
 .../IngestionRecordCountProvider.java           |   60 -
 .../LateFileRecordCountProvider.java            |   68 -
 .../reflection/GobblinConstructorUtils.java     |  119 --
 .../AllocatedRequestsIterator.java              |   31 -
 .../AllocatedRequestsIteratorBase.java          |   92 --
 .../request_allocation/BruteForceAllocator.java |   58 -
 .../ConcurrentBoundedPriorityIterable.java      |  232 ----
 .../request_allocation/GreedyAllocator.java     |   74 --
 .../HierarchicalAllocator.java                  |  183 ---
 .../HierarchicalPrioritizer.java                |   29 -
 .../request_allocation/PreOrderAllocator.java   |   84 --
 .../PriorityIterableBasedRequestAllocator.java  |   92 --
 .../PriorityMultiIterator.java                  |   80 --
 .../request_allocation/PushDownRequestor.java   |   34 -
 .../util/request_allocation/Request.java        |   25 -
 .../request_allocation/RequestAllocator.java    |   68 -
 .../RequestAllocatorConfig.java                 |   86 --
 .../RequestAllocatorUtils.java                  |   50 -
 .../util/request_allocation/Requestor.java      |   29 -
 .../request_allocation/ResourceEstimator.java   |   37 -
 .../util/request_allocation/ResourcePool.java   |  176 ---
 .../request_allocation/ResourceRequirement.java |  113 --
 .../SimpleHierarchicalPrioritizer.java          |   49 -
 .../util/request_allocation/VectorAlgebra.java  |   67 -
 .../util/service/StandardServiceConfig.java     |   56 -
 .../gobblin/util/test/FastSequentialSource.java |  106 --
 .../gobblin/util/test/HelloWorldSource.java     |  115 --
 .../java/gobblin/util/test/StdoutWriter.java    |   80 --
 .../java/gobblin/util/test/TestingSource.java   |   81 --
 .../broker/BrokerConfigurationKeyGenerator.java |   59 +
 .../gobblin/broker/DefaultBrokerCache.java      |  212 +++
 .../org/apache/gobblin/broker/EmptyKey.java     |   34 +
 .../broker/KeyedScopedConfigViewImpl.java       |   91 ++
 .../gobblin/broker/NonExtendableBrokerView.java |   41 +
 .../apache/gobblin/broker/ResourceInstance.java |   41 +
 .../org/apache/gobblin/broker/ScopeWrapper.java |   37 +
 .../broker/SharedResourcesBrokerFactory.java    |  155 +++
 .../broker/SharedResourcesBrokerImpl.java       |  328 +++++
 .../broker/SharedResourcesBrokerUtils.java      |   89 ++
 .../apache/gobblin/broker/TTLResourceEntry.java |   50 +
 .../org/apache/gobblin/broker/package-info.java |   46 +
 .../copy/hive/WhitelistBlacklist.java           |  149 +++
 .../gobblin/runtime/cli/CliApplication.java     |   30 +
 .../gobblin/runtime/cli/CliObjectFactory.java   |   61 +
 .../gobblin/runtime/cli/CliObjectOption.java    |   41 +
 .../gobblin/runtime/cli/CliObjectSupport.java   |   40 +
 ...tructorAndPublicMethodsCliObjectFactory.java |  134 ++
 .../apache/gobblin/runtime/cli/GobblinCli.java  |   61 +
 .../apache/gobblin/runtime/cli/NotOnCli.java    |   31 +
 .../cli/PublicMethodsCliObjectFactory.java      |  221 ++++
 .../gobblin/util/ApplicationLauncherUtils.java  |   35 +
 .../apache/gobblin/util/AutoCloseableLock.java  |   49 +
 .../gobblin/util/AutoReturnableObject.java      |   70 +
 .../org/apache/gobblin/util/AvroFlattener.java  |  489 +++++++
 .../java/org/apache/gobblin/util/AvroUtils.java |  655 ++++++++++
 .../org/apache/gobblin/util/AzkabanTags.java    |   71 +
 .../gobblin/util/CLIPasswordEncryptor.java      |  149 +++
 .../apache/gobblin/util/ClusterNameTags.java    |   62 +
 .../org/apache/gobblin/util/ClustersNames.java  |  180 +++
 .../org/apache/gobblin/util/ConfigUtils.java    |  507 ++++++++
 .../apache/gobblin/util/DatasetFilterUtils.java |  100 ++
 .../apache/gobblin/util/DatePartitionType.java  |  100 ++
 .../java/org/apache/gobblin/util/Either.java    |  109 ++
 .../org/apache/gobblin/util/EmailUtils.java     |  125 ++
 .../org/apache/gobblin/util/EmptyIterable.java  |   38 +
 .../org/apache/gobblin/util/ExecutorsUtils.java |  305 +++++
 .../apache/gobblin/util/ExponentialBackoff.java |  130 ++
 .../org/apache/gobblin/util/FileListUtils.java  |  175 +++
 .../apache/gobblin/util/ForkOperatorUtils.java  |  114 ++
 .../org/apache/gobblin/util/HadoopUtils.java    |  923 +++++++++++++
 .../gobblin/util/HeapDumpForTaskUtils.java      |   87 ++
 .../apache/gobblin/util/HiveJdbcConnector.java  |  288 +++++
 .../java/org/apache/gobblin/util/HostUtils.java |   19 +
 .../main/java/org/apache/gobblin/util/Id.java   |  187 +++
 .../gobblin/util/ImmutableProperties.java       |   71 +
 .../gobblin/util/JobConfigurationUtils.java     |  131 ++
 .../apache/gobblin/util/JobLauncherUtils.java   |  279 ++++
 .../java/org/apache/gobblin/util/JvmUtils.java  |   40 +
 .../util/LoggingUncaughtExceptionHandler.java   |   47 +
 .../org/apache/gobblin/util/NoopCloseable.java  |   34 +
 .../org/apache/gobblin/util/ParallelRunner.java |  380 ++++++
 .../java/org/apache/gobblin/util/PathUtils.java |  199 +++
 .../java/org/apache/gobblin/util/PortUtils.java |  165 +++
 .../apache/gobblin/util/PropertiesUtils.java    |   82 ++
 .../gobblin/util/ProxiedFileSystemCache.java    |  345 +++++
 .../gobblin/util/ProxiedFileSystemUtils.java    |  255 ++++
 .../gobblin/util/ProxiedFileSystemWrapper.java  |  166 +++
 .../org/apache/gobblin/util/PublisherUtils.java |  111 ++
 .../org/apache/gobblin/util/PullFileLoader.java |  271 ++++
 .../gobblin/util/RateControlledFileSystem.java  |  114 ++
 .../apache/gobblin/util/SerializationUtils.java |  193 +++
 .../apache/gobblin/util/StringParsingUtils.java |   83 ++
 .../org/apache/gobblin/util/TemplateUtils.java  |   44 +
 .../apache/gobblin/util/TimeRangeChecker.java   |  108 ++
 .../gobblin/util/WritableShimSerialization.java |  133 ++
 .../org/apache/gobblin/util/WriterUtils.java    |  335 +++++
 .../gobblin/util/binpacking/FieldWeighter.java  |   38 +
 .../WorstFitDecreasingBinPacking.java           |  171 +++
 .../apache/gobblin/util/callbacks/Callback.java |   35 +
 .../gobblin/util/callbacks/CallbackResult.java  |   76 ++
 .../util/callbacks/CallbacksDispatcher.java     |  204 +++
 .../gobblin/util/callbacks/package-info.java    |   63 +
 .../gobblin/util/concurrent/AutoResetEvent.java |   71 +
 .../util/concurrent/CancellableTask.java        |   44 +
 .../HashedWheelTimerTaskScheduler.java          |  170 +++
 .../ScheduledExecutorServiceTaskScheduler.java  |  153 +++
 .../gobblin/util/concurrent/ScheduledTask.java  |   37 +
 .../gobblin/util/concurrent/TaskScheduler.java  |  169 +++
 .../util/concurrent/TaskSchedulerFactory.java   |   66 +
 .../util/concurrent/TaskSchedulerType.java      |   63 +
 .../gobblin/util/dataset/DatasetUtils.java      |  134 ++
 .../util/deprecation/DeprecationUtils.java      |   56 +
 .../util/executors/ForceQueuePolicy.java        |   39 +
 .../util/executors/IteratorExecutor.java        |  172 +++
 .../util/executors/MDCPropagatingCallable.java  |   49 +
 .../MDCPropagatingExecutorService.java          |   58 +
 .../util/executors/MDCPropagatingRunnable.java  |   44 +
 .../MDCPropagatingScheduledExecutorService.java |  145 +++
 .../gobblin/util/executors/ScalingQueue.java    |   77 ++
 .../executors/ScalingThreadPoolExecutor.java    |   67 +
 ...CatchingPathAlterationListenerDecorator.java |  114 ++
 .../util/filesystem/FileStatusEntry.java        |  185 +++
 .../util/filesystem/FileSystemDecorator.java    |  533 ++++++++
 .../util/filesystem/FileSystemFactory.java      |  113 ++
 .../filesystem/FileSystemInstrumentation.java   |   62 +
 .../FileSystemInstrumentationFactory.java       |   43 +
 .../gobblin/util/filesystem/FileSystemKey.java  |   91 ++
 .../util/filesystem/FileSystemLimiterKey.java   |   66 +
 .../util/filesystem/InstrumentedFileSystem.java |   54 +
 .../filesystem/InstrumentedFileSystemUtils.java |  102 ++
 .../filesystem/InstrumentedHDFSFileSystem.java  |   36 +
 .../filesystem/InstrumentedLocalFileSystem.java |   37 +
 .../InstrumentedWebHDFSFileSystem.java          |   36 +
 .../util/filesystem/PathAlterationListener.java |   46 +
 .../PathAlterationListenerAdaptor.java          |   47 +
 .../util/filesystem/PathAlterationObserver.java |  346 +++++
 .../PathAlterationObserverScheduler.java        |  189 +++
 .../util/filesystem/ThrottledFileSystem.java    |  206 +++
 .../gobblin/util/filters/AndPathFilter.java     |   44 +
 .../gobblin/util/filters/HiddenFilter.java      |   42 +
 .../gobblin/util/filters/RegexPathFilter.java   |   52 +
 .../gobblin/util/filters/TarGpgPathFilter.java  |   32 +
 .../java/org/apache/gobblin/util/guid/Guid.java |  209 +++
 .../org/apache/gobblin/util/guid/HasGuid.java   |   34 +
 .../apache/gobblin/util/hadoop/TokenUtils.java  |  252 ++++
 .../gobblin/util/http/HttpLimiterKey.java       |   51 +
 .../util/io/AdditionalCloseableInputStream.java |   49 +
 .../gobblin/util/io/BatchedMeterDecorator.java  |   70 +
 .../gobblin/util/io/CloseableHttpConn.java      |   40 +
 .../gobblin/util/io/FilterStreamUnpacker.java   |   62 +
 .../gobblin/util/io/GsonInterfaceAdapter.java   |  195 +++
 .../gobblin/util/io/MeteredInputStream.java     |   93 ++
 .../gobblin/util/io/MeteredOutputStream.java    |   89 ++
 .../apache/gobblin/util/io/MeteredStream.java   |   31 +
 .../gobblin/util/io/SeekableFSInputStream.java  |   77 ++
 .../apache/gobblin/util/io/StreamCopier.java    |  149 +++
 .../util/io/StreamCopierSharedLimiterKey.java   |   89 ++
 .../apache/gobblin/util/io/StreamThrottler.java |  110 ++
 .../org/apache/gobblin/util/io/StreamUtils.java |  239 ++++
 .../gobblin/util/io/ThrottledInputStream.java   |   96 ++
 .../util/iterators/InterruptibleIterator.java   |   67 +
 .../gobblin/util/jdbc/DataSourceBuilder.java    |  145 +++
 .../gobblin/util/jdbc/DataSourceModule.java     |   45 +
 .../gobblin/util/jdbc/DataSourceProvider.java   |   76 ++
 .../org/apache/gobblin/util/json/JsonUtils.java |   43 +
 .../gobblin/util/limiter/BaseLimiterType.java   |   79 ++
 .../gobblin/util/limiter/CountBasedLimiter.java |   88 ++
 .../util/limiter/DefaultLimiterFactory.java     |  141 ++
 .../apache/gobblin/util/limiter/Limiter.java    |   60 +
 .../gobblin/util/limiter/LimiterFactory.java    |   28 +
 .../gobblin/util/limiter/MultiLimiter.java      |  106 ++
 .../util/limiter/NonRefillableLimiter.java      |   44 +
 .../gobblin/util/limiter/NoopLimiter.java       |   48 +
 .../util/limiter/NotEnoughPermitsException.java |   36 +
 .../gobblin/util/limiter/PoolBasedLimiter.java  |   99 ++
 .../gobblin/util/limiter/RateBasedLimiter.java  |   92 ++
 .../gobblin/util/limiter/TimeBasedLimiter.java  |  105 ++
 .../limiter/broker/SharedLimiterFactory.java    |  149 +++
 .../util/limiter/broker/SharedLimiterKey.java   |   66 +
 .../util/logs/Log4jConfigurationHelper.java     |   70 +
 .../org/apache/gobblin/util/logs/LogCopier.java |  580 +++++++++
 .../CompactionRecordCountProvider.java          |   95 ++
 .../IngestionRecordCountProvider.java           |   60 +
 .../LateFileRecordCountProvider.java            |   68 +
 .../reflection/GobblinConstructorUtils.java     |  119 ++
 .../AllocatedRequestsIterator.java              |   31 +
 .../AllocatedRequestsIteratorBase.java          |   92 ++
 .../request_allocation/BruteForceAllocator.java |   58 +
 .../ConcurrentBoundedPriorityIterable.java      |  232 ++++
 .../request_allocation/GreedyAllocator.java     |   74 ++
 .../HierarchicalAllocator.java                  |  183 +++
 .../HierarchicalPrioritizer.java                |   29 +
 .../request_allocation/PreOrderAllocator.java   |   84 ++
 .../PriorityIterableBasedRequestAllocator.java  |   92 ++
 .../PriorityMultiIterator.java                  |   80 ++
 .../request_allocation/PushDownRequestor.java   |   34 +
 .../util/request_allocation/Request.java        |   25 +
 .../request_allocation/RequestAllocator.java    |   68 +
 .../RequestAllocatorConfig.java                 |   86 ++
 .../RequestAllocatorUtils.java                  |   50 +
 .../util/request_allocation/Requestor.java      |   29 +
 .../request_allocation/ResourceEstimator.java   |   37 +
 .../util/request_allocation/ResourcePool.java   |  176 +++
 .../request_allocation/ResourceRequirement.java |  113 ++
 .../SimpleHierarchicalPrioritizer.java          |   49 +
 .../util/request_allocation/VectorAlgebra.java  |   67 +
 .../util/service/StandardServiceConfig.java     |   56 +
 .../gobblin/util/test/FastSequentialSource.java |  106 ++
 .../gobblin/util/test/HelloWorldSource.java     |  115 ++
 .../apache/gobblin/util/test/StdoutWriter.java  |   80 ++
 .../apache/gobblin/util/test/TestingSource.java |   81 ++
 ....filesystem.FileSystemInstrumentationFactory |   18 -
 ....filesystem.FileSystemInstrumentationFactory |   18 +
 .../services/org.apache.hadoop.fs.FileSystem    |    6 +-
 .../gobblin/broker/AutoscopedFactoryTest.java   |   86 --
 .../broker/DefaultGobblinBrokerTest.java        |  315 -----
 .../gobblin/broker/GobblinBrokerConfTest.java   |  114 --
 .../broker/GobblinBrokerCreationTest.java       |  110 --
 .../broker/KeyedScopedConfigViewImplTest.java   |   64 -
 .../SharedResourcesBrokerFactoryTest.java       |  105 --
 .../test/java/gobblin/broker/TestFactory.java   |   90 --
 .../gobblin/broker/TestFactoryWithRedirect.java |   32 -
 .../java/gobblin/broker/TestResourceKey.java    |   35 -
 ...torAndPublicMethodsCliObjectFactoryTest.java |   89 --
 .../cli/PublicMethodsCliObjectFactoryTest.java  |   89 --
 .../java/gobblin/util/AvroFlattenerTest.java    |  192 ---
 .../test/java/gobblin/util/AvroUtilsTest.java   |  259 ----
 .../java/gobblin/util/ClustersNamesTest.java    |   43 -
 .../test/java/gobblin/util/ConfigUtilsTest.java |  280 ----
 .../gobblin/util/DatePartitionTypeTest.java     |   29 -
 .../java/gobblin/util/ExecutorsUtilsTest.java   |  132 --
 .../java/gobblin/util/FileListUtilsTest.java    |  149 ---
 .../gobblin/util/ForkOperatorUtilsTest.java     |   85 --
 .../test/java/gobblin/util/HadoopUtilsTest.java |  284 ----
 .../gobblin/util/HeapDumpForTaskUtilsTest.java  |   79 --
 .../java/gobblin/util/JobLauncherUtilsTest.java |  208 ---
 .../LoggingUncaughtExceptionHandlerTest.java    |   48 -
 .../java/gobblin/util/ParallelRunnerTest.java   |  275 ----
 .../test/java/gobblin/util/PortUtilsTest.java   |  106 --
 .../java/gobblin/util/PropertiesUtilsTest.java  |   55 -
 .../java/gobblin/util/PullFileLoaderTest.java   |  227 ----
 .../util/RatedControlledFileSystemTest.java     |   88 --
 .../gobblin/util/SerializationUtilsTest.java    |   86 --
 .../gobblin/util/StringParsingUtilsTest.java    |   42 -
 .../gobblin/util/TestImmutableProperties.java   |   65 -
 .../java/gobblin/util/TimeRangeCheckerTest.java |   61 -
 .../test/java/gobblin/util/WriterUtilsTest.java |  140 --
 .../WorstFitDecreasingBinPackingTest.java       |  126 --
 .../util/callbacks/TestCallbacksDispatcher.java |  155 ---
 .../HashedWheelTimerTaskSchedulerTest.java      |   32 -
 ...heduledExecutorServiceTaskSchedulerTest.java |   32 -
 .../concurrent/TaskSchedulerFactoryTest.java    |   40 -
 .../util/concurrent/TaskSchedulerTest.java      |  121 --
 .../gobblin/util/dataset/DatasetUtilsTest.java  |   87 --
 .../util/executors/IteratorExecutorTest.java    |  203 ---
 .../util/filesystem/FileSystemFactoryTest.java  |  104 --
 .../InstrumentedLocalFileSystemTest.java        |   78 --
 .../filesystem/ThrottledFileSystemTest.java     |   90 --
 .../util/filters/RegexPathFilterTest.java       |   45 -
 .../test/java/gobblin/util/guid/GuidTest.java   |   81 --
 .../util/io/GsonInterfaceAdapterTest.java       |   46 -
 .../gobblin/util/io/MeteredInputStreamTest.java |   90 --
 .../util/io/MeteredOutputStreamTest.java        |   77 --
 .../java/gobblin/util/io/StreamCopierTest.java  |   74 --
 .../java/gobblin/util/io/StreamUtilsTest.java   |  183 ---
 .../util/io/ThrottledInputStreamTest.java       |   56 -
 .../util/limiter/CountBasedLimiterTest.java     |   47 -
 .../util/limiter/DefaultLimiterFactoryTest.java |   78 --
 .../gobblin/util/limiter/MultiLimiterTest.java  |   75 --
 .../util/limiter/RateBasedLimiterTest.java      |   70 -
 .../util/limiter/TimeBasedLimiterTest.java      |   62 -
 .../broker/SharedLimiterFactoryTest.java        |  129 --
 .../CompactionRecordCountProviderTest.java      |   40 -
 .../IngestionRecordCountProviderTest.java       |   38 -
 .../reflection/GobblinConstructorUtilsTest.java |  115 --
 .../BruteForceAllocatorTest.java                |   56 -
 .../ConcurrentBoundedPriorityIterableTest.java  |   98 --
 .../request_allocation/GreedyAllocatorTest.java |   61 -
 .../HierarchicalAllocatorTest.java              |   68 -
 .../PreOrderAllocatorTest.java                  |   64 -
 .../request_allocation/ResourcePoolTest.java    |   46 -
 .../util/request_allocation/StringRequest.java  |   56 -
 .../request_allocation/StringRequestor.java     |   67 -
 .../request_allocation/VectorAlgebraTest.java   |   49 -
 .../test/java/gobblin/util/test/BaseClass.java  |   37 -
 .../java/gobblin/util/test/ExtendedClass.java   |   33 -
 .../test/java/gobblin/util/test/TestClass.java  |   69 -
 .../gobblin/util/test/TestHelloWorldSource.java |   56 -
 .../gobblin/broker/AutoscopedFactoryTest.java   |   86 ++
 .../broker/DefaultGobblinBrokerTest.java        |  315 +++++
 .../gobblin/broker/GobblinBrokerConfTest.java   |  114 ++
 .../broker/GobblinBrokerCreationTest.java       |  110 ++
 .../broker/KeyedScopedConfigViewImplTest.java   |   64 +
 .../SharedResourcesBrokerFactoryTest.java       |  105 ++
 .../org/apache/gobblin/broker/TestFactory.java  |   90 ++
 .../gobblin/broker/TestFactoryWithRedirect.java |   32 +
 .../apache/gobblin/broker/TestResourceKey.java  |   35 +
 ...torAndPublicMethodsCliObjectFactoryTest.java |   89 ++
 .../cli/PublicMethodsCliObjectFactoryTest.java  |   89 ++
 .../apache/gobblin/util/AvroFlattenerTest.java  |  192 +++
 .../org/apache/gobblin/util/AvroUtilsTest.java  |  259 ++++
 .../apache/gobblin/util/ClustersNamesTest.java  |   43 +
 .../apache/gobblin/util/ConfigUtilsTest.java    |  280 ++++
 .../gobblin/util/DatePartitionTypeTest.java     |   29 +
 .../apache/gobblin/util/ExecutorsUtilsTest.java |  132 ++
 .../apache/gobblin/util/FileListUtilsTest.java  |  149 +++
 .../gobblin/util/ForkOperatorUtilsTest.java     |   85 ++
 .../apache/gobblin/util/HadoopUtilsTest.java    |  284 ++++
 .../gobblin/util/HeapDumpForTaskUtilsTest.java  |   79 ++
 .../gobblin/util/JobLauncherUtilsTest.java      |  208 +++
 .../LoggingUncaughtExceptionHandlerTest.java    |   48 +
 .../apache/gobblin/util/ParallelRunnerTest.java |  275 ++++
 .../org/apache/gobblin/util/PortUtilsTest.java  |  106 ++
 .../gobblin/util/PropertiesUtilsTest.java       |   55 +
 .../apache/gobblin/util/PullFileLoaderTest.java |  227 ++++
 .../util/RatedControlledFileSystemTest.java     |   88 ++
 .../gobblin/util/SerializationUtilsTest.java    |   86 ++
 .../gobblin/util/StringParsingUtilsTest.java    |   42 +
 .../gobblin/util/TestImmutableProperties.java   |   65 +
 .../gobblin/util/TimeRangeCheckerTest.java      |   61 +
 .../apache/gobblin/util/WriterUtilsTest.java    |  140 ++
 .../WorstFitDecreasingBinPackingTest.java       |  126 ++
 .../util/callbacks/TestCallbacksDispatcher.java |  155 +++
 .../HashedWheelTimerTaskSchedulerTest.java      |   32 +
 ...heduledExecutorServiceTaskSchedulerTest.java |   32 +
 .../concurrent/TaskSchedulerFactoryTest.java    |   40 +
 .../util/concurrent/TaskSchedulerTest.java      |  121 ++
 .../gobblin/util/dataset/DatasetUtilsTest.java  |   87 ++
 .../util/executors/IteratorExecutorTest.java    |  203 +++
 .../util/filesystem/FileSystemFactoryTest.java  |  104 ++
 .../InstrumentedLocalFileSystemTest.java        |   78 ++
 .../filesystem/ThrottledFileSystemTest.java     |   90 ++
 .../util/filters/RegexPathFilterTest.java       |   45 +
 .../org/apache/gobblin/util/guid/GuidTest.java  |   81 ++
 .../util/io/GsonInterfaceAdapterTest.java       |   46 +
 .../gobblin/util/io/MeteredInputStreamTest.java |   90 ++
 .../util/io/MeteredOutputStreamTest.java        |   77 ++
 .../gobblin/util/io/StreamCopierTest.java       |   74 ++
 .../apache/gobblin/util/io/StreamUtilsTest.java |  183 +++
 .../util/io/ThrottledInputStreamTest.java       |   56 +
 .../util/limiter/CountBasedLimiterTest.java     |   47 +
 .../util/limiter/DefaultLimiterFactoryTest.java |   78 ++
 .../gobblin/util/limiter/MultiLimiterTest.java  |   75 ++
 .../util/limiter/RateBasedLimiterTest.java      |   70 +
 .../util/limiter/TimeBasedLimiterTest.java      |   62 +
 .../broker/SharedLimiterFactoryTest.java        |  129 ++
 .../CompactionRecordCountProviderTest.java      |   40 +
 .../IngestionRecordCountProviderTest.java       |   38 +
 .../reflection/GobblinConstructorUtilsTest.java |  115 ++
 .../BruteForceAllocatorTest.java                |   56 +
 .../ConcurrentBoundedPriorityIterableTest.java  |   98 ++
 .../request_allocation/GreedyAllocatorTest.java |   61 +
 .../HierarchicalAllocatorTest.java              |   68 +
 .../PreOrderAllocatorTest.java                  |   64 +
 .../request_allocation/ResourcePoolTest.java    |   46 +
 .../util/request_allocation/StringRequest.java  |   56 +
 .../request_allocation/StringRequestor.java     |   67 +
 .../request_allocation/VectorAlgebraTest.java   |   49 +
 .../org/apache/gobblin/util/test/BaseClass.java |   37 +
 .../apache/gobblin/util/test/ExtendedClass.java |   33 +
 .../org/apache/gobblin/util/test/TestClass.java |   69 +
 .../gobblin/util/test/TestHelloWorldSource.java |   56 +
 .../gobblin/yarn/GobblinApplicationMaster.java  |  221 ----
 .../gobblin/yarn/GobblinYarnAppLauncher.java    |  806 ------------
 .../yarn/GobblinYarnConfigurationKeys.java      |   81 --
 .../gobblin/yarn/GobblinYarnEventConstants.java |   49 -
 .../java/gobblin/yarn/GobblinYarnLogSource.java |   95 --
 .../gobblin/yarn/GobblinYarnMetricTagNames.java |   29 -
 .../gobblin/yarn/GobblinYarnTaskRunner.java     |  187 ---
 .../java/gobblin/yarn/HelixMessageSubTypes.java |   36 -
 .../gobblin/yarn/YarnAppSecurityManager.java    |  294 -----
 .../yarn/YarnContainerSecurityManager.java      |  108 --
 .../main/java/gobblin/yarn/YarnHelixUtils.java  |  136 --
 .../src/main/java/gobblin/yarn/YarnService.java |  690 ----------
 .../event/ApplicationReportArrivalEvent.java    |   48 -
 .../yarn/event/ContainerShutdownRequest.java    |   46 -
 .../yarn/event/DelegationTokenUpdatedEvent.java |   28 -
 .../event/GetApplicationReportFailureEvent.java |   45 -
 .../gobblin/yarn/event/NewContainerRequest.java |   46 -
 .../gobblin/yarn/GobblinApplicationMaster.java  |  221 ++++
 .../gobblin/yarn/GobblinYarnAppLauncher.java    |  806 ++++++++++++
 .../yarn/GobblinYarnConfigurationKeys.java      |   81 ++
 .../gobblin/yarn/GobblinYarnEventConstants.java |   49 +
 .../gobblin/yarn/GobblinYarnLogSource.java      |   95 ++
 .../gobblin/yarn/GobblinYarnMetricTagNames.java |   29 +
 .../gobblin/yarn/GobblinYarnTaskRunner.java     |  187 +++
 .../gobblin/yarn/HelixMessageSubTypes.java      |   36 +
 .../gobblin/yarn/YarnAppSecurityManager.java    |  294 +++++
 .../yarn/YarnContainerSecurityManager.java      |  108 ++
 .../org/apache/gobblin/yarn/YarnHelixUtils.java |  136 ++
 .../org/apache/gobblin/yarn/YarnService.java    |  690 ++++++++++
 .../event/ApplicationReportArrivalEvent.java    |   48 +
 .../yarn/event/ContainerShutdownRequest.java    |   46 +
 .../yarn/event/DelegationTokenUpdatedEvent.java |   28 +
 .../event/GetApplicationReportFailureEvent.java |   45 +
 .../gobblin/yarn/event/NewContainerRequest.java |   46 +
 .../yarn/GobblinYarnAppLauncherTest.java        |  276 ----
 .../gobblin/yarn/YarnSecurityManagerTest.java   |  211 ---
 .../yarn/GobblinYarnAppLauncherTest.java        |  276 ++++
 .../gobblin/yarn/YarnSecurityManagerTest.java   |  211 +++
 gobblin-yarn/src/test/resources/reference.conf  |    2 +-
 gradle/scripts/defaultBuildProperties.gradle    |    4 +-
 ligradle/findbugs/findbugsExclude.xml           |    8 +-
 4951 files changed, 285766 insertions(+), 287186 deletions(-)
----------------------------------------------------------------------



Mime
View raw message