drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amansi...@apache.org
Subject [1/3] drill git commit: DRILL-5127: Revert the fix for DRILL-4831
Date Sun, 08 Jan 2017 21:39:53 GMT
Repository: drill
Updated Branches:
  refs/heads/master d375eeb45 -> 4d4e0c2b2


DRILL-5127: Revert the fix for DRILL-4831

close apache/drill#718


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/167f0ce3
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/167f0ce3
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/167f0ce3

Branch: refs/heads/master
Commit: 167f0ce339426ffaca5591412ea87a0bef48ad86
Parents: d375eeb
Author: Padma Penumarthy <ppenumar97@yahoo.com>
Authored: Tue Jan 3 14:01:00 2017 -0800
Committer: Aman Sinha <asinha@maprtech.com>
Committed: Sun Jan 8 12:27:56 2017 -0800

----------------------------------------------------------------------
 .../drill/exec/store/parquet/Metadata.java      | 67 +++-----------------
 1 file changed, 8 insertions(+), 59 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/167f0ce3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
index 48b4497..d85d6f1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
@@ -23,7 +23,6 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Iterator;
-import java.util.UUID;
 
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
@@ -41,8 +40,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.Options;
 
 import org.apache.parquet.column.statistics.Statistics;
 import org.apache.parquet.hadoop.ParquetFileReader;
@@ -213,17 +210,11 @@ public class Metadata {
     for (String oldname : OLD_METADATA_FILENAMES) {
       fs.delete(new Path(p, oldname), false);
     }
-    // writeFile creates and writes to a tmp file first and then renames it
-    // to final metadata cache file name. We want the UUID appended to tmp file
-    // to be same for METADATA_FILENAME and METADATA_DIRECTORIES_FILENAME
-    // so we can track/debug things better.
-    // Generate UUID used for tmp file creation here
-    UUID tmpUUID =  UUID.randomUUID();
-    writeFile(parquetTableMetadata, path, tmpUUID);
+    writeFile(parquetTableMetadata, new Path(p, METADATA_FILENAME));
 
     if (directoryList.size() > 0 && childFiles.size() == 0) {
       ParquetTableMetadataDirs parquetTableMetadataDirs = new ParquetTableMetadataDirs(directoryList);
-      writeFile(parquetTableMetadataDirs, path, tmpUUID);
+      writeFile(parquetTableMetadataDirs, new Path(p, METADATA_DIRECTORIES_FILENAME));
       logger.info("Creating metadata files recursively took {} ms", timer.elapsed(TimeUnit.MILLISECONDS));
       timer.stop();
       return Pair.of(parquetTableMetadata, parquetTableMetadataDirs);
@@ -501,35 +492,13 @@ public class Metadata {
   }
 
   /**
-   * Renames Path srcPath to Path dstPath.
-   *
-   * @param srcPath
-   * @param dstPath
-   * @throws IOException
-   */
-  private void renameFile(Path srcPath, Path dstPath) throws IOException {
-    try {
-      // Use fileContext API as FileSystem rename is deprecated.
-      FileContext fileContext = FileContext.getFileContext(srcPath.toUri());
-      fileContext.rename(srcPath, dstPath, Options.Rename.OVERWRITE);
-    } catch (Exception e) {
-      logger.info("Metadata cache file rename from {} to {} failed", srcPath.toString(),
dstPath.toString(), e);
-      throw new IOException("metadata cache file rename failed", e);
-    } finally {
-      if (fs.exists(srcPath)) {
-        fs.delete(srcPath, false);
-      }
-    }
-  }
-
-  /**
    * Serialize parquet metadata to json and write to a file
    *
    * @param parquetTableMetadata
    * @param p
    * @throws IOException
    */
-  private void writeFile(ParquetTableMetadata_v3 parquetTableMetadata, String path, UUID
tmpUUID) throws IOException {
+  private void writeFile(ParquetTableMetadata_v3 parquetTableMetadata, Path p) throws IOException
{
     JsonFactory jsonFactory = new JsonFactory();
     jsonFactory.configure(Feature.AUTO_CLOSE_TARGET, false);
     jsonFactory.configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, false);
@@ -537,39 +506,23 @@ public class Metadata {
     SimpleModule module = new SimpleModule();
     module.addSerializer(ColumnMetadata_v3.class, new ColumnMetadata_v3.Serializer());
     mapper.registerModule(module);
-
-    // If multiple clients are updating metadata cache file concurrently, the cache file
-    // can get corrupted. To prevent this, write to a unique temporary file and then do
-    // atomic rename.
-    Path tmpPath = new Path(path, METADATA_FILENAME + "." + tmpUUID);
-    FSDataOutputStream os = fs.create(tmpPath);
+    FSDataOutputStream os = fs.create(p);
     mapper.writerWithDefaultPrettyPrinter().writeValue(os, parquetTableMetadata);
     os.flush();
     os.close();
-
-    Path finalPath = new Path(path, METADATA_FILENAME);
-    renameFile(tmpPath, finalPath);
   }
 
-  private void writeFile(ParquetTableMetadataDirs parquetTableMetadataDirs, String path,
UUID tmpUUID) throws IOException {
+  private void writeFile(ParquetTableMetadataDirs parquetTableMetadataDirs, Path p) throws
IOException {
     JsonFactory jsonFactory = new JsonFactory();
     jsonFactory.configure(Feature.AUTO_CLOSE_TARGET, false);
     jsonFactory.configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, false);
     ObjectMapper mapper = new ObjectMapper(jsonFactory);
     SimpleModule module = new SimpleModule();
     mapper.registerModule(module);
-
-    // If multiple clients are updating metadata cache file concurrently, the cache file
-    // can get corrupted. To prevent this, write to a unique temporary file and then do
-    // atomic rename.
-    Path tmpPath = new Path(path, METADATA_DIRECTORIES_FILENAME + "." + tmpUUID);
-    FSDataOutputStream os = fs.create(tmpPath);
+    FSDataOutputStream os = fs.create(p);
     mapper.writerWithDefaultPrettyPrinter().writeValue(os, parquetTableMetadataDirs);
     os.flush();
     os.close();
-
-    Path finalPath = new Path(path,  METADATA_DIRECTORIES_FILENAME);
-    renameFile(tmpPath, finalPath);
   }
 
   /**
@@ -612,10 +565,8 @@ public class Metadata {
       logger.info("Took {} ms to read directories from directory cache file", timer.elapsed(TimeUnit.MILLISECONDS));
       timer.stop();
       if (!alreadyCheckedModification && tableModified(parquetTableMetadataDirs.getDirectories(),
p, parentDir, metaContext)) {
-        // Do not remove scheme and authority from the path passed to createMetaFilesRecursively
-        // as we need full path to obtain proper fileContext in writeFile
         parquetTableMetadataDirs =
-            (createMetaFilesRecursively(p.getParent().toString())).getRight();
+            (createMetaFilesRecursively(Path.getPathWithoutSchemeAndAuthority(p.getParent()).toString())).getRight();
         newMetadata = true;
       }
     } else {
@@ -623,10 +574,8 @@ public class Metadata {
       logger.info("Took {} ms to read metadata from cache file", timer.elapsed(TimeUnit.MILLISECONDS));
       timer.stop();
       if (!alreadyCheckedModification && tableModified(parquetTableMetadata.getDirectories(),
p, parentDir, metaContext)) {
-        // Do not remove scheme and authority from the path passed to createMetaFilesRecursively
-        // as we need full path to obtain proper fileContext in writeFile
         parquetTableMetadata =
-            (createMetaFilesRecursively(p.getParent().toString())).getLeft();
+            (createMetaFilesRecursively(Path.getPathWithoutSchemeAndAuthority(p.getParent()).toString())).getLeft();
         newMetadata = true;
       }
 


Mime
View raw message