hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bch...@apache.org
Subject svn commit: r1616105 - in /hadoop/common/branches/MR-2841/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/ hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/ hadoop-mapreduce-client/hadoop-mapreduce-...
Date Wed, 06 Aug 2014 06:01:12 GMT
Author: bchang
Date: Wed Aug  6 06:01:12 2014
New Revision: 1616105

URL: http://svn.apache.org/r1616105
Log:
MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common. Contributed by Binglin Chang

Removed:
    hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/lz4/lz4.c
    hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/lz4/lz4.h
Modified:
    hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
    hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
    hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
    hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING
    hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
    hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
    hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc

Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt?rev=1616105&r1=1616104&r2=1616105&view=diff
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt (original)
+++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt Wed
Aug  6 06:01:12 2014
@@ -9,3 +9,4 @@ MAPREDUCE-6000. native-task: Simplify By
 MAPREDUCE-5991. native-task should not run unit tests if native profile is not enabled. (Binglin
Chang)
 MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
 MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
+MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common (Binglin Chang)

Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml?rev=1616105&r1=1616104&r2=1616105&view=diff
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
(original)
+++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
Wed Aug  6 06:01:12 2014
@@ -163,6 +163,12 @@
                 </goals>
                 <configuration>
                   <target>
+                    <copy file="${basedir}/../../../hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.h"
+                      todir="${project.build.directory}/native/" />
+                    <copy file="${basedir}/../../../hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4_encoder.h"
+                      todir="${project.build.directory}/native/" />
+                    <copy file="${basedir}/../../../hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c"
+                      todir="${project.build.directory}/native/" />
                     <copy todir="${project.build.directory}/native/test/testData"
                       overwrite="true">
                       <fileset dir="${basedir}/src/main/native/testData" />

Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt?rev=1616105&r1=1616104&r2=1616105&view=diff
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
(original)
+++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
Wed Aug  6 06:01:12 2014
@@ -155,7 +155,7 @@ include_directories(
     ${D}/test
     ${CMAKE_CURRENT_SOURCE_DIR}
     #${CMAKE_CURRENT_SOURCE_DIR}/src
-    #${CMAKE_BINARY_DIR}
+    ${CMAKE_BINARY_DIR}
     ${JNI_INCLUDE_DIRS}
     ${SNAPPY_INCLUDE_DIR}
 )
@@ -174,7 +174,7 @@ else (${CMAKE_SYSTEM_NAME} MATCHES "Darw
 endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
 
 add_dual_library(nativetask
-    ${D}/lz4/lz4.c
+    ${CMAKE_BINARY_DIR}/lz4.c
     ${D}/cityhash/city.cc
     ${D}/src/codec/BlockCodec.cc
     ${D}/src/codec/GzipCodec.cc

Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING?rev=1616105&r1=1616104&r2=1616105&view=diff
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING
(original)
+++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING
Wed Aug  6 06:01:12 2014
@@ -56,20 +56,20 @@ LZ4
 ---------------------------------------------------------------------
    LZ4 - Fast LZ compression algorithm
    Header File
-   Copyright (C) 2011, Yann Collet.
-   BSD License
+   Copyright (C) 2011-2014, Yann Collet.
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
    met:
-  
+
        * Redistributions of source code must retain the above copyright
    notice, this list of conditions and the following disclaimer.
        * Redistributions in binary form must reproduce the above
    copyright notice, this list of conditions and the following disclaimer
    in the documentation and/or other materials provided with the
    distribution.
-  
+
    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
@@ -81,3 +81,7 @@ LZ4
    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - LZ4 source repository : http://code.google.com/p/lz4/
+   - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
\ No newline at end of file

Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc?rev=1616105&r1=1616104&r2=1616105&view=diff
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
(original)
+++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc
Wed Aug  6 06:01:12 2014
@@ -17,33 +17,15 @@
  */
 
 #include "commons.h"
+#include "lz4.h"
 #include "NativeTask.h"
 #include "Lz4Codec.h"
 
-extern "C" {
-extern int LZ4_compress(char* source, char* dest, int isize);
-extern int LZ4_uncompress(char* source, char* dest, int osize);
-
-/*
- LZ4_compress() :
- return : the number of bytes in compressed buffer dest
- note : destination buffer must be already allocated.
- To avoid any problem, size it to handle worst cases situations (input data not compressible)
- Worst case size is : "inputsize + 0.4%", with "0.4%" being at least 8 bytes.
-
- LZ4_uncompress() :
- osize  : is the output size, therefore the original size
- return : the number of bytes read in the source buffer
- If the source stream is malformed, the function will stop decoding and return a negative
result, indicating the byte position of the faulty instruction
- This version never writes beyond dest + osize, and is therefore protected against malicious
data packets
- note 2 : destination buffer must be already allocated
- */
-}
 
 namespace NativeTask {
 
 static int32_t LZ4_MaxCompressedSize(int32_t orig) {
-  return std::max((int32_t)(orig * 1.005), orig + 8);
+  return LZ4_compressBound(orig);
 }
 
 Lz4CompressStream::Lz4CompressStream(OutputStream * stream, uint32_t bufferSizeHint)

Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc?rev=1616105&r1=1616104&r2=1616105&view=diff
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
(original)
+++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
Wed Aug  6 06:01:12 2014
@@ -16,6 +16,7 @@
  * limitations under the License.
  */
 
+#include "lz4.h"
 #include "snappy.h"
 #include "commons.h"
 #include "Path.h"
@@ -172,11 +173,6 @@ public:
   }
 };
 
-extern "C" {
-extern int LZ4_compress(char* source, char* dest, int isize);
-extern int LZ4_uncompress(char* source, char* dest, int osize);
-}
-;
 
 void MeasureSingleFileLz4(const string & path, CompressResult & total, size_t blockSize,
     int times) {

Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc?rev=1616105&r1=1616104&r2=1616105&view=diff
==============================================================================
--- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
(original)
+++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
Wed Aug  6 06:01:12 2014
@@ -162,7 +162,7 @@ TEST(Perf, IFile) {
 // The Glibc has a bug in the file tell api, it will overwrite the file data 
 // unexpected. 
 // Please check https://rhn.redhat.com/errata/RHBA-2013-0279.html
-// This case is to check wether the bug exists.
+// This case is to check whether the bug exists.
 // If it exists, it means you need to upgrade the glibc.
 TEST(IFile, TestGlibCBug) {
   std::string path("./testData/testGlibCBugSpill.out");



Mime
View raw message