tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jihoon...@apache.org
Subject tajo git commit: TAJO-1384: Duplicated output file path problem. (jihoon)
Date Wed, 11 Mar 2015 01:15:24 GMT
Repository: tajo
Updated Branches:
  refs/heads/master 3b9a2e5f2 -> 513a8c308


TAJO-1384: Duplicated output file path problem. (jihoon)

Closes #406


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/513a8c30
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/513a8c30
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/513a8c30

Branch: refs/heads/master
Commit: 513a8c3086b889d9b701d5c97eda3fb8c5453834
Parents: 3b9a2e5
Author: Jihoon Son <jihoonson@apache.org>
Authored: Wed Mar 11 10:07:01 2015 +0900
Committer: Jihoon Son <jihoonson@apache.org>
Committed: Wed Mar 11 10:07:40 2015 +0900

----------------------------------------------------------------------
 CHANGES                                         |  2 +
 .../org/apache/tajo/storage/StorageUtil.java    | 30 ++++------
 .../apache/tajo/storage/TestStorageUtil.java    | 58 ++++++++++++++++++++
 3 files changed, 72 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/513a8c30/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index dc1a5f7..68aabf5 100644
--- a/CHANGES
+++ b/CHANGES
@@ -16,6 +16,8 @@ Release 0.11.0 - unreleased
 
   BUG FIXES
 
+    TAJO-1384: Duplicated output file path problem. (jihoon)
+
     TAJO-1386: CURRENT_DATE generates parsing errors sometimes.
     (Contributed by navis, Committed by hyunsik)
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/513a8c30/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
index 68e96d8..926b5d3 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
@@ -18,15 +18,10 @@
 
 package org.apache.tajo.storage;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.tajo.catalog.Column;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.util.FileUtil;
 import sun.nio.ch.DirectBuffer;
 
 import java.io.DataInput;
@@ -34,8 +29,6 @@ import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
 
 public class StorageUtil extends StorageConstants {
 
@@ -119,32 +112,33 @@ public class StorageUtil extends StorageConstants {
     }
 
     int maxValue = -1;
-    List<Path> fileNamePatternMatchedList = new ArrayList<Path>();
 
     for (FileStatus eachFile: files) {
       // In the case of partition table, return largest value within all partition dirs.
+      int value;
       if (eachFile.isDirectory() && recursive) {
-        int value = getMaxFileSequence(fs, eachFile.getPath(), recursive);
+        value = getMaxFileSequence(fs, eachFile.getPath(), recursive);
         if (value > maxValue) {
           maxValue = value;
         }
       } else {
         if (eachFile.getPath().getName().matches(fileNamePatternV08) ||
             eachFile.getPath().getName().matches(fileNamePatternV09)) {
-          fileNamePatternMatchedList.add(eachFile.getPath());
+          value = getSequence(eachFile.getPath().getName());
+          if (value > maxValue) {
+            maxValue = value;
+          }
         }
       }
     }
 
-    if (fileNamePatternMatchedList.isEmpty()) {
-      return maxValue;
-    }
-    Path lastFile = fileNamePatternMatchedList.get(fileNamePatternMatchedList.size() - 1);
-    String pathName = lastFile.getName();
+    return maxValue;
+  }
 
-    // 0.8: pathName = part-<ExecutionBlockId.seq>-<TaskId.seq>
-    // 0.9: pathName = part-<ExecutionBlockId.seq>-<TaskId.seq>-<Sequence>
-    String[] pathTokens = pathName.split("-");
+  // 0.8: pathName = part-<ExecutionBlockId.seq>-<TaskId.seq>
+  // 0.9: pathName = part-<ExecutionBlockId.seq>-<TaskId.seq>-<Sequence>
+  private static int getSequence(String name) {
+    String[] pathTokens = name.split("-");
     if (pathTokens.length == 3) {
       return -1;
     } else if(pathTokens.length == 4) {

http://git-wip-us.apache.org/repos/asf/tajo/blob/513a8c30/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorageUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorageUtil.java
b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorageUtil.java
new file mode 100644
index 0000000..4b7c929
--- /dev/null
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorageUtil.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.util.CommonTestingUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestStorageUtil {
+  private TajoConf conf;
+  private static String TEST_PATH = "target/test-data/TestStorageUtil";
+  private Path testDir;
+  private FileSystem fs;
+
+  @Before
+  public void setUp() throws Exception {
+    conf = new TajoConf();
+    testDir = CommonTestingUtil.getTestDir(TEST_PATH);
+    fs = testDir.getFileSystem(conf);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+  }
+
+  @Test
+  public final void testGetMaxFileSequence() throws IOException {
+    for (int i = 0; i < 7; i++) {
+      fs.create(new Path(testDir, "part-00-00000-00"+i), false);
+    }
+
+    assertEquals(6, StorageUtil.getMaxFileSequence(fs, testDir, true));
+  }
+}


Mime
View raw message