tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bi...@apache.org
Subject [32/50] [abbrv] git commit: TEZ-1550. TestEnvironmentUpdateUtils.testMultipleUpdateEnvironment fails on Windows (Prakash Ramachandran via bikas)
Date Mon, 29 Sep 2014 00:35:38 GMT
TEZ-1550. TestEnvironmentUpdateUtils.testMultipleUpdateEnvironment fails on Windows (Prakash
Ramachandran via bikas)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/97528e91
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/97528e91
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/97528e91

Branch: refs/heads/branch-0.5
Commit: 97528e9188a05b323699222e21ed7f22fd7ff8e3
Parents: 41fe736
Author: Bikas Saha <bikas@apache.org>
Authored: Sun Sep 21 13:46:58 2014 -0700
Committer: Bikas Saha <bikas@apache.org>
Committed: Sun Sep 21 13:46:58 2014 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |   2 +
 .../tez/common/EnvironmentUpdateUtils.java      | 127 +++++++++++++++++++
 .../tez/dag/utils/EnvironmentUpdateUtils.java   |  92 --------------
 .../tez/common/TestEnvironmentUpdateUtils.java  |  34 +++++
 .../dag/utils/TestEnvironmentUpdateUtils.java   |  34 -----
 .../java/org/apache/tez/client/LocalClient.java |   2 +-
 .../app/launcher/LocalContainerLauncher.java    |   6 +-
 .../library/output/TestOnFileSortedOutput.java  |   2 +-
 .../library/shuffle/common/TestFetcher.java     |   2 +-
 9 files changed, 171 insertions(+), 130 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ab8b533..f069959 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -49,6 +49,8 @@ ALL CHANGES
   TestTezCommonUtils for Windows
   TEZ-1598. DAGClientTimelineImpl uses ReflectiveOperationException (which has JDK 1.7 dependency)
   TEZ-1599. TezClient.preWarm() is not enabled
+  TEZ-1550. TestEnvironmentUpdateUtils.testMultipleUpdateEnvironment fails on
+  Windows
 
 Release 0.5.0: 2014-09-03
 

http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-common/src/main/java/org/apache/tez/common/EnvironmentUpdateUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/EnvironmentUpdateUtils.java b/tez-common/src/main/java/org/apache/tez/common/EnvironmentUpdateUtils.java
new file mode 100644
index 0000000..cd519d9
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/common/EnvironmentUpdateUtils.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.util.Shell;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A utility class which allows one to dynamically update/change Environment variables
+ */
+@InterfaceAudience.Private
+public class EnvironmentUpdateUtils {
+
+  /**
+   * Allows dynamic update to the environment variables. After calling put,
+   * System.getenv(key) will then return value.
+   *
+   * @param key System environment variable
+   * @param value Value to assign to system environment variable
+   */
+  public static void put(String key, String value){
+    Map<String, String> environment = new HashMap<String, String>(System.getenv());
+    environment.put(key, value);
+    if (!Shell.WINDOWS) {
+      updateEnvironment(environment);
+    } else {
+      updateEnvironmentOnWindows(environment);
+    }
+  }
+
+  /**
+   * Allows dynamic update to a collection of environment variables. After
+   * calling putAll, System.getenv(key) will then return value for each entry
+   * in the map
+   *
+   * @param additionalEnvironment Collection where the key is the System
+   * environment variable and the value is the value to assign the system
+   * environment variable
+   */
+  public static void putAll(Map<String, String> additionalEnvironment) {
+    Map<String, String> environment = new HashMap<String, String>(System.getenv());
+    environment.putAll(additionalEnvironment);
+    if (!Shell.WINDOWS) {
+      updateEnvironment(environment);
+    } else {
+      updateEnvironmentOnWindows(environment);
+    }
+  }
+
+  /**
+   * Finds and modifies internal storage for system environment variables using
+   * reflection
+   *
+   * @param environment Collection where the key is the System
+   * environment variable and the value is the value to assign the system
+   * environment variable
+   */
+  @SuppressWarnings("unchecked")
+  private static void updateEnvironment(Map<String, String> environment) {
+    final Map<String, String> currentEnv = System.getenv();
+    copyMapValuesToPrivateField(currentEnv.getClass(), currentEnv, "m", environment);
+  }
+
+  /**
+   * Finds and modifies internal storage for system environment variables using reflection.
This
+   * method works only on windows. Note that the actual env is not modified, rather the copy
of env
+   * which the JVM creates at the beginning of execution is.
+   *
+   * @param environment Collection where the key is the System
+   * environment variable and the value is the value to assign the system
+   * environment variable
+   */
+  @SuppressWarnings("unchecked")
+  private static void updateEnvironmentOnWindows(Map<String, String> environment) {
+    try {
+      Class<?> processEnvironmentClass = Class.forName("java.lang.ProcessEnvironment");
+      copyMapValuesToPrivateField(processEnvironmentClass, null, "theEnvironment", environment);
+      copyMapValuesToPrivateField(processEnvironmentClass, null, "theCaseInsensitiveEnvironment",
+          environment);
+    } catch (ClassNotFoundException e) {
+      throw new IllegalStateException("Failed to update Environment variables", e);
+    }
+  }
+
+  /**
+   * Copies the given map values to the field specified by {@code fieldName}
+   * @param klass The {@code Class} of the object
+   * @param object The object to modify or null if the field is static
+   * @param fieldName The name of the field to set
+   * @param newMapValues The values to replace the current map.
+   */
+  @SuppressWarnings("unchecked")
+  private static void copyMapValuesToPrivateField(Class<?> klass, Object object, String
fieldName,
+                                                  Map<String, String> newMapValues)
{
+    try {
+      Field field = klass.getDeclaredField(fieldName);
+      field.setAccessible(true);
+      Map<String, String> currentMap = (Map<String, String>) field.get(object);
+      currentMap.clear();
+      currentMap.putAll(newMapValues);
+    } catch (NoSuchFieldException e) {
+      throw new IllegalStateException("Failed to update Environment variables", e);
+    } catch (IllegalAccessException e) {
+      throw new IllegalStateException("Failed to update Environment variables", e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-common/src/main/java/org/apache/tez/dag/utils/EnvironmentUpdateUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/utils/EnvironmentUpdateUtils.java
b/tez-common/src/main/java/org/apache/tez/dag/utils/EnvironmentUpdateUtils.java
deleted file mode 100644
index a8b4ced..0000000
--- a/tez-common/src/main/java/org/apache/tez/dag/utils/EnvironmentUpdateUtils.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.utils;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import java.lang.reflect.Field;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * A utility class which allows one to dynamically update/change Environment variables
- */
-@InterfaceAudience.Private
-public class EnvironmentUpdateUtils {
-
-  /**
-   * Allows dynamic update to the environment variables. After calling put,
-   * System.getenv(key) will then return value.
-   *
-   * @param key System environment variable
-   * @param value Value to assign to system environment variable
-   */
-  public static void put(String key, String value){
-    Map<String, String> environment = new HashMap<String, String>(System.getenv());
-    environment.put(key, value);
-    updateEnvironment(environment);
-  }
-
-  /**
-   * Allows dynamic update to a collection of environment variables. After
-   * calling putAll, System.getenv(key) will then return value for each entry
-   * in the map
-   *
-   * @param additionalEnvironment Collection where the key is the System
-   * environment variable and the value is the value to assign the system
-   * environment variable
-   */
-  public static void putAll(Map<String, String> additionalEnvironment) {
-    Map<String, String> environment = new HashMap<String, String>(System.getenv());
-    environment.putAll(additionalEnvironment);
-    updateEnvironment(environment);
-  }
-
-  /**
-   * Finds and modifies internal storage for system environment variables using
-   * reflection
-   *
-   * @param environment Collection where the key is the System
-   * environment variable and the value is the value to assign the system
-   * environment variable
-   */
-  @SuppressWarnings("unchecked")
-  private static void updateEnvironment(Map<String, String> environment) {
-    try {
-      Class<?>[] classes = Collections.class.getDeclaredClasses();
-      for (Class<?> clazz : classes) {
-        if ("java.util.Collections$UnmodifiableMap".equals(clazz.getName())) {
-          Field field = clazz.getDeclaredField("m");
-          field.setAccessible(true);
-          Object obj = field.get(System.getenv());
-          Map<String, String> map = (Map<String, String>)obj;
-          map.clear();
-          map.putAll(environment);
-        }
-      }
-    }
-    catch (NoSuchFieldException e) {
-      throw new IllegalStateException("Failed to update Environment variables", e);
-    }
-    catch (IllegalAccessException e) {
-      throw new IllegalStateException("Failed to update Environment variables", e);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-common/src/test/java/org/apache/tez/common/TestEnvironmentUpdateUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/test/java/org/apache/tez/common/TestEnvironmentUpdateUtils.java
b/tez-common/src/test/java/org/apache/tez/common/TestEnvironmentUpdateUtils.java
new file mode 100644
index 0000000..d8c6980
--- /dev/null
+++ b/tez-common/src/test/java/org/apache/tez/common/TestEnvironmentUpdateUtils.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+public class TestEnvironmentUpdateUtils {
+
+  @Test
+  public void testMultipleUpdateEnvironment() {
+    EnvironmentUpdateUtils.put("test.environment1", "test.value1");
+    EnvironmentUpdateUtils.put("test.environment2", "test.value2");
+    assertEquals("Environment was not set propertly", "test.value1", System.getenv("test.environment1"));
+    assertEquals("Environment was not set propertly", "test.value2", System.getenv("test.environment2"));
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-common/src/test/java/org/apache/tez/dag/utils/TestEnvironmentUpdateUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/test/java/org/apache/tez/dag/utils/TestEnvironmentUpdateUtils.java
b/tez-common/src/test/java/org/apache/tez/dag/utils/TestEnvironmentUpdateUtils.java
deleted file mode 100644
index d4a42c4..0000000
--- a/tez-common/src/test/java/org/apache/tez/dag/utils/TestEnvironmentUpdateUtils.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.utils;
-
-import static org.junit.Assert.assertEquals;
-
-import org.junit.Test;
-
-public class TestEnvironmentUpdateUtils {
-
-  @Test
-  public void testMultipleUpdateEnvironment() {
-    EnvironmentUpdateUtils.put("test.environment1", "test.value1");
-    EnvironmentUpdateUtils.put("test.environment2", "test.value2");
-    assertEquals("Environment was not set propertly", "test.value1", System.getenv("test.environment1"));
-    assertEquals("Environment was not set propertly", "test.value2", System.getenv("test.environment2"));
-  }
-}

http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
index 0a95cf0..bd7089f 100644
--- a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
+++ b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.util.SystemClock;
 import org.apache.log4j.Logger;
+import org.apache.tez.common.EnvironmentUpdateUtils;
 import org.apache.tez.common.TezCommonUtils;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;
@@ -53,7 +54,6 @@ import org.apache.tez.dag.app.AppContext;
 import org.apache.tez.dag.app.DAGAppMaster;
 import org.apache.tez.dag.app.DAGAppMasterState;
 import org.apache.tez.dag.app.dag.DAG;
-import org.apache.tez.dag.utils.EnvironmentUpdateUtils;
 
 import com.google.common.annotations.VisibleForTesting;
 

http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
index 7407544..baeb9a3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
@@ -46,8 +46,10 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.common.EnvironmentUpdateUtils;
 import org.apache.tez.common.TezCommonUtils;
 import org.apache.tez.common.TezTaskUmbilicalProtocol;
 import org.apache.tez.dag.api.TezConfiguration;
@@ -65,7 +67,6 @@ import org.apache.tez.dag.app.rm.container.AMContainerEventLaunched;
 import org.apache.tez.dag.app.rm.container.AMContainerEventType;
 import org.apache.tez.dag.history.DAGHistoryEvent;
 import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
-import org.apache.tez.dag.utils.EnvironmentUpdateUtils;
 import org.apache.tez.runtime.task.TezChild;
 
 
@@ -111,6 +112,9 @@ public class LocalContainerLauncher extends AbstractService implements
         Base64.encodeBase64String(ByteBuffer.allocate(4).putInt(0).array()));
     EnvironmentUpdateUtils.put(Environment.NM_HOST.toString(),
         InetAddress.getLocalHost().getHostName());
+    if (Shell.WINDOWS) {
+      EnvironmentUpdateUtils.put(Environment.USER.name(), System.getenv("USERNAME"));
+    }
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
index 8a7f2a6..f9a3dd5 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
@@ -25,11 +25,11 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.tez.common.EnvironmentUpdateUtils;
 import org.apache.tez.common.TezUtils;
 import org.apache.tez.common.TezRuntimeFrameworkConfigs;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.utils.EnvironmentUpdateUtils;
 import org.apache.tez.runtime.api.Event;
 import org.apache.tez.runtime.api.MemoryUpdateCallback;
 import org.apache.tez.runtime.api.OutputContext;

http://git-wip-us.apache.org/repos/asf/tez/blob/97528e91/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java
index 9502555..2ef6ce6 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/TestFetcher.java
@@ -40,8 +40,8 @@ import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.common.EnvironmentUpdateUtils;
 import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.utils.EnvironmentUpdateUtils;
 import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
 import org.apache.tez.runtime.library.common.InputAttemptIdentifier;
 import org.apache.tez.runtime.library.common.sort.impl.TezIndexRecord;


Mime
View raw message