carbondata-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zzcclp <...@git.apache.org>
Subject [GitHub] carbondata pull request #1516: [CARBONDATA-1729]Fix the compatibility issue ...
Date Fri, 17 Nov 2017 09:09:35 GMT
Github user zzcclp commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1516#discussion_r151633344
  
    --- Diff: core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
---
    @@ -154,52 +155,68 @@ public boolean delete() {
        * This method will delete the data in file data from a given offset
        */
       @Override public boolean truncate(String fileName, long validDataEndOffset) {
    -    DataOutputStream dataOutputStream = null;
    -    DataInputStream dataInputStream = null;
         boolean fileTruncatedSuccessfully = false;
    -    // if bytes to read less than 1024 then buffer size should be equal to the given
offset
    -    int bufferSize = validDataEndOffset > CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR
?
    -        CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR :
    -        (int) validDataEndOffset;
    -    // temporary file name
    -    String tempWriteFilePath = fileName + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
    -    FileFactory.FileType fileType = FileFactory.getFileType(fileName);
         try {
    -      CarbonFile tempFile;
    -      // delete temporary file if it already exists at a given path
    -      if (FileFactory.isFileExist(tempWriteFilePath, fileType)) {
    -        tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
    -        tempFile.delete();
    -      }
    -      // create new temporary file
    -      FileFactory.createNewFile(tempWriteFilePath, fileType);
    -      tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
    -      byte[] buff = new byte[bufferSize];
    -      dataInputStream = FileFactory.getDataInputStream(fileName, fileType);
    -      // read the data
    -      int read = dataInputStream.read(buff, 0, buff.length);
    -      dataOutputStream = FileFactory.getDataOutputStream(tempWriteFilePath, fileType);
    -      dataOutputStream.write(buff, 0, read);
    -      long remaining = validDataEndOffset - read;
    -      // anytime we should not cross the offset to be read
    -      while (remaining > 0) {
    -        if (remaining > bufferSize) {
    -          buff = new byte[bufferSize];
    -        } else {
    -          buff = new byte[(int) remaining];
    +      // if hadoop version >= 2.7, it can call method 'truncate' to truncate file,
    +      // this method was new in hadoop 2.7
    +      FileSystem fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
    +      Method truncateMethod = fs.getClass().getDeclaredMethod("truncate",
    +          new Class[]{Path.class, long.class});
    +      fileTruncatedSuccessfully = (boolean)truncateMethod.invoke(fs,
    +          new Object[]{fileStatus.getPath(), validDataEndOffset});
    +    } catch (NoSuchMethodException e) {
    +      LOGGER.error("there is no 'truncate' method in FileSystem, the version of hadoop
is"
    +          + " below 2.7, It needs to implement truncate file by other way.");
    +      DataOutputStream dataOutputStream = null;
    +      DataInputStream dataInputStream = null;
    +      // if bytes to read less than 1024 then buffer size should be equal to the given
offset
    +      int bufferSize = validDataEndOffset > CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR
?
    +          CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR :
    +          (int) validDataEndOffset;
    +      // temporary file name
    +      String tempWriteFilePath = fileName + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
    +      FileFactory.FileType fileType = FileFactory.getFileType(fileName);
    +      try {
    +        CarbonFile tempFile;
    +        // delete temporary file if it already exists at a given path
    +        if (FileFactory.isFileExist(tempWriteFilePath, fileType)) {
    +          tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
    +          tempFile.delete();
             }
    -        read = dataInputStream.read(buff, 0, buff.length);
    +        // create new temporary file
    +        FileFactory.createNewFile(tempWriteFilePath, fileType);
    +        tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
    +        byte[] buff = new byte[bufferSize];
    +        dataInputStream = FileFactory.getDataInputStream(fileName, fileType);
    +        // read the data
    +        int read = dataInputStream.read(buff, 0, buff.length);
    +        dataOutputStream = FileFactory.getDataOutputStream(tempWriteFilePath, fileType);
             dataOutputStream.write(buff, 0, read);
    -        remaining = remaining - read;
    +        long remaining = validDataEndOffset - read;
    +        // anytime we should not cross the offset to be read
    +        while (remaining > 0) {
    +          if (remaining > bufferSize) {
    +            buff = new byte[bufferSize];
    +          } else {
    +            buff = new byte[(int) remaining];
    +          }
    +          read = dataInputStream.read(buff, 0, buff.length);
    +          dataOutputStream.write(buff, 0, read);
    +          remaining = remaining - read;
    +        }
    +        CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
    +        // rename the temp file to original file
    +        tempFile.renameForce(fileName);
    +        fileTruncatedSuccessfully = true;
    +      } catch (IOException ioe) {
    +        LOGGER.error("IOException occurred while truncating the file " + ioe.getMessage());
    +      } finally {
    +        CarbonUtil.closeStreams(dataOutputStream, dataInputStream);
           }
    -      CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
    -      // rename the temp file to original file
    -      tempFile.renameForce(fileName);
    -      fileTruncatedSuccessfully = true;
         } catch (IOException e) {
    -      LOGGER.error("Exception occurred while truncating the file " + e.getMessage());
    -    } finally {
    -      CarbonUtil.closeStreams(dataOutputStream, dataInputStream);
    +      LOGGER.error("IOException occurred while truncating the file " + e.getMessage());
    +    } catch (Exception e) {
    --- End diff --
    
    Method.invoke may throw other exceptions when running, such as: IllegalAccessException,

     IllegalArgumentException, InvocationTargetException;
    So need to add this catch for above exceptions.
    
    The finally block is for the try block which is in catch NoSuchMethodException branch.



---

Mime
View raw message