incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tof...@apache.org
Subject svn commit: r1332773 - in /incubator/hcatalog/trunk/src/java/org/apache/hcatalog: data/HCatRecordSerDe.java data/JsonSerDe.java mapreduce/FileOutputCommitterContainer.java
Date Tue, 01 May 2012 18:44:28 GMT
Author: toffer
Date: Tue May  1 18:44:28 2012
New Revision: 1332773

URL: http://svn.apache.org/viewvc?rev=1332773&view=rev
Log:
HCATALOG-68 addendum to patch missed some changes in comments (avandana via toffer)

Modified:
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java?rev=1332773&r1=1332772&r2=1332773&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java Tue May
 1 18:44:28 2012
@@ -87,8 +87,8 @@ public class HCatRecordSerDe implements 
     }
 
 
-    LOG.debug("columns: {}",columnNameProperty,columnNames);
-    LOG.debug("types: {}", columnTypeProperty, columnTypes);
+    LOG.debug("columns: {} {}",columnNameProperty,columnNames);
+    LOG.debug("types: {} {}", columnTypeProperty, columnTypes);
     assert (columnNames.size() == columnTypes.size());
 
     rowTypeInfo = (StructTypeInfo) TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java?rev=1332773&r1=1332772&r2=1332773&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java Tue May  1 18:44:28
2012
@@ -150,10 +150,12 @@ public class JsonSerDe implements SerDe 
         populateRecord(r,token,p,schema);
       }
     } catch (JsonParseException e) {
-        LOG.warn("Error [{}] parsing json location [{}].", e.getMessage(), e.getLocation());
+        LOG.warn("Error [{}] parsing json text [{}].", e, t);
+        LOG.debug(null,e);
       throw new SerDeException(e);
     } catch (IOException e) {
-        LOG.warn("Error [{}] parsing json text [{}].", e.getMessage(),t);
+        LOG.warn("Error [{}] parsing json text [{}].", e, t);
+        LOG.debug(null,e);
       throw new SerDeException(e);
     }
 

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1332773&r1=1332772&r2=1332773&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
Tue May  1 18:44:28 2012
@@ -48,6 +48,8 @@ import org.apache.hcatalog.data.schema.H
 import org.apache.hcatalog.data.schema.HCatSchemaUtils;
 import org.apache.hcatalog.har.HarOutputCommitterPostProcessor;
 import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.net.URI;
@@ -64,6 +66,7 @@ import java.util.Map.Entry;
  */
 class FileOutputCommitterContainer extends OutputCommitterContainer {
 
+    private static final Logger LOG = LoggerFactory.getLogger(FileOutputCommitterContainer.class);
     private final boolean dynamicPartitioningUsed;
     private boolean partitionsDiscovered;
 
@@ -436,20 +439,18 @@ class FileOutputCommitterContainer exten
 
         partition.setParameters(params);
 
-        // Sets permissions and group name on partition dirs.
+        // Sets permissions and group name on partition dirs and files.
 
         Path partPath = new Path(partLocnRoot);
-        for(FieldSchema partKey : table.getPartitionKeys()){
-            partPath = constructPartialPartPath(partPath, partKey.getName().toLowerCase(),
partKVs);
-//        LOG.info("Setting perms for "+partPath.toString());
-            fs.setPermission(partPath, perms);
-            try{
-                fs.setOwner(partPath, null, grpName);
-            } catch(AccessControlException ace){
-                // log the messages before ignoring. Currently, logging is not built in Hcatalog.
-//          LOG.warn(ace);
+        int i = 0;
+        for (FieldSchema partKey : table.getPartitionKeys()) {
+            if (i++ != 0) {
+                applyGroupAndPerms(fs, partPath, perms, grpName, false);
             }
+            partPath = constructPartialPartPath(partPath, partKey.getName().toLowerCase(),
partKVs);
         }
+        // Apply the group and permissions to the leaf partition and files.
+        applyGroupAndPerms(fs, partPath, perms, grpName, true);
         if (dynamicPartitioningUsed){
             String dynamicPartitionDestination = getFinalDynamicPartitionDestination(table,partKVs);
             if (harProcessor.isEnabled()){
@@ -466,7 +467,30 @@ class FileOutputCommitterContainer exten
         return partition;
     }
 
-
+    private void applyGroupAndPerms(FileSystem fs, Path dir, FsPermission permission,
+            String group, boolean recursive)
+            throws IOException {
+        fs.setPermission(dir, permission);
+        try {
+            fs.setOwner(dir, null, group);
+        } catch (AccessControlException ace) {
+            LOG.warn("Error changing group of " + dir, ace);
+        }
+        if (recursive) {
+            for (FileStatus fileStatus : fs.listStatus(dir)) {
+                if (fileStatus.isDir()) {
+                    applyGroupAndPerms(fs, fileStatus.getPath(), permission, group, recursive);
+                } else {
+                    fs.setPermission(fileStatus.getPath(), permission);
+                    try {
+                        fs.setOwner(dir, null, group);
+                    } catch (AccessControlException ace) {
+                        LOG.warn("Error changing group of " + dir, ace);
+                    }
+                }
+            }
+        }
+    }
 
     private String getFinalDynamicPartitionDestination(Table table, Map<String,String>
partKVs) {
         // file:///tmp/hcat_junit_warehouse/employee/_DYN0.7770480401313761/emp_country=IN/emp_state=KA
 ->



Mime
View raw message