gobblin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hut...@apache.org
Subject incubator-gobblin git commit: [GOBBLIN-56] Changing statestore checker to work with any state store Making non-required the state store url Adding required text to help for required properties Fixing issue where it read second paramater as value due to u
Date Mon, 28 Aug 2017 16:02:13 GMT
Repository: incubator-gobblin
Updated Branches:
  refs/heads/master 841193a37 -> 4e9453fba


[GOBBLIN-56] Changing statestore checker to work with any state store
Making non-required the state store url
Adding required text to help for required properties
Fixing issue where it read second paramater as value due to using hasArgs instead of hasArg

Closes #2077 from
treff7es/mysql_state_store_checker


Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/4e9453fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/4e9453fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/4e9453fb

Branch: refs/heads/master
Commit: 4e9453fba6e1ceb51077ec1fe5871f4d65109ff1
Parents: 841193a
Author: treff7es <tamas.nemeth@prezi.com>
Authored: Mon Aug 28 09:01:13 2017 -0700
Committer: Hung Tran <hutran@linkedin.com>
Committed: Mon Aug 28 09:01:13 2017 -0700

----------------------------------------------------------------------
 .../runtime/util/JobStateToJsonConverter.java   | 34 +++++++++++-------
 .../util/JobStateToJsonConverterTest.java       | 38 +++++++++++++++++---
 2 files changed, 55 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/4e9453fb/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java
----------------------------------------------------------------------
diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java
b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java
index 5860daa..71a66db 100644
--- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java
+++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java
@@ -33,9 +33,8 @@ import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -43,9 +42,10 @@ import com.google.common.io.Closer;
 import com.google.gson.stream.JsonWriter;
 
 import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.metastore.DatasetStateStore;
 import org.apache.gobblin.metastore.StateStore;
-import org.apache.gobblin.runtime.FsDatasetStateStore;
 import org.apache.gobblin.runtime.JobState;
+import org.apache.gobblin.util.ConfigUtils;
 import org.apache.gobblin.util.JobConfigurationUtils;
 
 
@@ -66,11 +66,13 @@ public class JobStateToJsonConverter {
   public JobStateToJsonConverter(Properties props, String storeUrl, boolean keepConfig) throws
IOException {
     Configuration conf = new Configuration();
     JobConfigurationUtils.putPropertiesIntoConfiguration(props, conf);
-    Path storePath = new Path(storeUrl);
-    FileSystem fs = storePath.getFileSystem(conf);
-    String storeRootDir = storePath.toUri().getPath();
-    this.jobStateStore = new FsDatasetStateStore(fs, storeRootDir);
+
+    if (StringUtils.isNotBlank(storeUrl)) {
+      props.setProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, storeUrl);
+    }
+
     this.keepConfig = keepConfig;
+    this.jobStateStore = (StateStore) DatasetStateStore.buildDatasetStateStore(ConfigUtils.propertiesToConfig(props));
   }
 
   /**
@@ -154,19 +156,19 @@ public class JobStateToJsonConverter {
   @SuppressWarnings("all")
   public static void main(String[] args) throws Exception {
     Option sysConfigOption = Option.builder("sc").argName("system configuration file")
-        .desc("Gobblin system configuration file").longOpt("sysconfig").hasArgs().build();
+        .desc("Gobblin system configuration file (required if no state store URL specified)").longOpt("sysconfig").hasArg().build();
     Option storeUrlOption = Option.builder("u").argName("gobblin state store URL")
-        .desc("Gobblin state store root path URL").longOpt("storeurl").hasArgs().required().build();
-    Option jobNameOption = Option.builder("n").argName("gobblin job name").desc("Gobblin
job name").longOpt("name")
-        .hasArgs().required().build();
+        .desc("Gobblin state store root path URL (required if no sysconfig specified)").longOpt("storeurl").hasArg().build();
+    Option jobNameOption = Option.builder("n").argName("gobblin job name").desc("Gobblin
job name (required)").longOpt("name")
+        .hasArg().required().build();
     Option jobIdOption =
-        Option.builder("i").argName("gobblin job id").desc("Gobblin job id").longOpt("id").hasArgs().build();
+        Option.builder("i").argName("gobblin job id").desc("Gobblin job id").longOpt("id").hasArg().build();
     Option convertAllOption =
         Option.builder("a").desc("Whether to convert all past job states of the given job").longOpt("all").build();
     Option keepConfigOption =
         Option.builder("kc").desc("Whether to keep all configuration properties").longOpt("keepConfig").build();
     Option outputToFile =
-        Option.builder("t").argName("output file name").desc("Output file name").longOpt("toFile").hasArgs().build();
+        Option.builder("t").argName("output file name").desc("Output file name").longOpt("toFile").hasArg().build();
 
     Options options = new Options();
     options.addOption(sysConfigOption);
@@ -187,6 +189,12 @@ public class JobStateToJsonConverter {
       System.exit(1);
     }
 
+    if (!cmd.hasOption(sysConfigOption.getLongOpt()) && !cmd.hasOption(storeUrlOption.getLongOpt())
){
+      HelpFormatter formatter = new HelpFormatter();
+      formatter.printHelp("JobStateToJsonConverter", options);
+      System.exit(1);
+    }
+
     Properties sysConfig = new Properties();
     if (cmd.hasOption(sysConfigOption.getLongOpt())) {
       sysConfig = JobConfigurationUtils.fileToProperties(cmd.getOptionValue(sysConfigOption.getLongOpt()));

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/4e9453fb/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java
----------------------------------------------------------------------
diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java
b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java
index e4bf4f9..66ca54f 100644
--- a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java
+++ b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java
@@ -16,17 +16,22 @@
  */
 package org.apache.gobblin.runtime.util;
 
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import com.google.gson.stream.JsonReader;
 import java.io.IOException;
 import java.io.StringReader;
 import java.io.StringWriter;
 import java.util.Properties;
+
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.stream.JsonReader;
+
+import org.apache.gobblin.configuration.ConfigurationKeys;
+
+
 @Test(groups = { "gobblin.runtime" })
 public class JobStateToJsonConverterTest {
 
@@ -40,7 +45,9 @@ public class JobStateToJsonConverterTest {
   public void testJsonKeepConfig()
       throws IOException {
     String stateStorePath = getClass().getClassLoader().getResource(TEST_STORE).getPath();
+
     boolean keepConfig = true;
+
     JobStateToJsonConverter converter = new JobStateToJsonConverter(new Properties(), stateStorePath,
keepConfig);
 
     StringWriter stringWriter = new StringWriter();
@@ -53,4 +60,27 @@ public class JobStateToJsonConverterTest {
       Assert.assertNotNull(taskState.getAsJsonObject().get(PROPERTIES));
     }
   }
+
+  @Test
+  public void testJsonKeepConfigWithoutStoreUrl()
+      throws IOException {
+    String stateStorePath = getClass().getClassLoader().getResource(TEST_STORE).getPath();
+    Properties properties = new Properties();
+
+    properties.setProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, stateStorePath);
+
+    boolean keepConfig = true;
+
+    JobStateToJsonConverter converter = new JobStateToJsonConverter(properties, null, keepConfig);
+
+    StringWriter stringWriter = new StringWriter();
+    converter.convert(TEST_JOB, stringWriter);
+
+    JsonObject json = new JsonParser().parse(new JsonReader(new StringReader(stringWriter.toString()))).getAsJsonObject();
+
+    Assert.assertNotNull(json.get(PROPERTIES));
+    for (JsonElement taskState: json.get(TASK_STATES).getAsJsonArray()) {
+      Assert.assertNotNull(taskState.getAsJsonObject().get(PROPERTIES));
+    }
+  }
 }


Mime
View raw message