Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B851A1858B for ; Sun, 6 Dec 2015 07:13:04 +0000 (UTC) Received: (qmail 4152 invoked by uid 500); 6 Dec 2015 07:13:04 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 3973 invoked by uid 500); 6 Dec 2015 07:13:04 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 3842 invoked by uid 99); 6 Dec 2015 07:13:04 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 06 Dec 2015 07:13:04 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E3AD1DFF4F; Sun, 6 Dec 2015 07:13:03 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: asuresh@apache.org To: common-commits@hadoop.apache.org Date: Sun, 06 Dec 2015 07:13:03 -0000 Message-Id: <5403537e05c0447f827148a3c20cca60@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [01/38] hadoop git commit: HADOOP-11505. Various native parts use bswap incorrectly and unportably (Alan Burlison via aw) Repository: hadoop Updated Branches: refs/heads/yarn-2877 9f256d1d7 -> 742632e34 HADOOP-11505. Various native parts use bswap incorrectly and unportably (Alan Burlison via aw) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6725e7f1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6725e7f1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6725e7f1 Branch: refs/heads/yarn-2877 Commit: 6725e7f1beb96177b0b59a6082a05869aab2e37b Parents: 9f256d1 Author: Allen Wittenauer Authored: Fri Nov 27 18:25:45 2015 -0800 Committer: Allen Wittenauer Committed: Fri Nov 27 18:25:45 2015 -0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + .../hadoop-common/src/CMakeLists.txt | 125 +++++++++++++++++++ .../src/org/apache/hadoop/util/bulk_crc32.c | 17 +-- .../src/CMakeLists.txt | 1 + .../mapred/nativetask/INativeComparable.java | 4 +- .../src/main/native/src/NativeTask.h | 9 -- .../src/main/native/src/codec/BlockCodec.cc | 4 +- .../src/main/native/src/codec/Lz4Codec.cc | 4 +- .../src/main/native/src/codec/SnappyCodec.cc | 4 +- .../main/native/src/handler/CombineHandler.cc | 15 +-- .../src/handler/MCollectorOutputHandler.cc | 10 +- .../src/handler/MCollectorOutputHandler.h | 2 - .../src/main/native/src/lib/Buffers.h | 13 +- .../src/main/native/src/lib/IFile.cc | 4 +- .../src/main/native/src/lib/IFile.h | 4 +- .../main/native/src/lib/NativeObjectFactory.cc | 16 +-- .../src/main/native/src/lib/SpillInfo.cc | 4 +- .../src/main/native/src/lib/commons.h | 1 + .../src/main/native/src/lib/primitives.h | 50 ++------ .../src/main/native/src/util/WritableUtils.cc | 26 ++-- .../src/main/native/test/TestIFile.cc | 2 +- .../src/main/native/test/TestSort.cc | 8 +- .../src/main/native/test/lib/TestKVBuffer.cc | 4 +- .../native/test/lib/TestMemBlockIterator.cc | 2 +- .../src/main/native/test/lib/TestMemoryBlock.cc | 6 +- .../main/native/test/lib/TestPartitionBucket.cc | 26 ++-- 26 files changed, 225 insertions(+), 139 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 7cdf21b..4f35432 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -537,6 +537,9 @@ Trunk (Unreleased) HADOOP-12553. [JDK8] Fix javadoc error caused by illegal tag. (aajisaka) + HADOOP-11505. Various native parts use bswap incorrectly and unportably + (Alan Burlison via aw) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-common-project/hadoop-common/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt index 63bb773..a8762d5 100644 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -41,6 +41,131 @@ endif() # Configure JNI. include(HadoopJNI) +# +# Endian configuration, as per http://austingroupbugs.net/view.php?id=162#c665 +# + +# Work out the endianness, set header macro values. +include(TestBigEndian) +include(CheckIncludeFile) +include(CheckSymbolExists) +test_big_endian(_bigendian) +if(_bigendian) + set(HADOOP_BYTE_ORDER "HADOOP_BIG_ENDIAN") +else() + set(HADOOP_BYTE_ORDER "HADOOP_LITTLE_ENDIAN") +endif() + +# Linux, NetBSD, FreeBSD and OpenBSD all provide htoXXX definitions in endian.h or sys/endian.h. +check_include_file("endian.h" _endian_h) +if (_endian_h) + set(HADOOP_ENDIAN_H "endian.h") +else() + check_include_file("sys/endian.h" _sys_endian_h) + if (_sys_endian_h) + set(HADOOP_ENDIAN_H "sys/endian.h") + endif() +endif() +if(DEFINED HADOOP_ENDIAN_H) +check_symbol_exists("be64toh" ${HADOOP_ENDIAN_H} _be64toh) + if( _be64toh) + set(HADOOP_HTOBE16 "htobe16") + set(HADOOP_HTOLE16 "htole16") + set(HADOOP_BE16TOH "be16toh") + set(HADOOP_LE16TOH "le16toh") + set(HADOOP_HTOBE32 "htobe32") + set(HADOOP_HTOLE32 "htole32") + set(HADOOP_BE32TOH "be32toh") + set(HADOOP_LE32TOH "le32toh") + set(HADOOP_HTOBE64 "htobe64") + set(HADOOP_HTOLE64 "htole64") + set(HADOOP_BE64TOH "be64toh") + set(HADOOP_LE64TOH "le64toh") + set(_have_endian TRUE) + unset(_be64toh) + else() + message(FATAL_ERROR "endian.h located but doesn't contain be64toh") + endif() +endif() + +# Solaris doesn't provide htoXXX, we have to provide alternatives. +if(NOT _have_endian) + check_include_file("sys/byteorder.h" _sys_byteorder_h) + if(_sys_byteorder_h) + set(HADOOP_ENDIAN_H "sys/byteorder.h") + check_symbol_exists("BSWAP_64" ${HADOOP_ENDIAN_H} _bswap_64) + endif() + if(_sys_byteorder_h AND _bswap_64) + if(_bigendian) + set(HADOOP_HTOBE16 "") + set(HADOOP_HTOLE16 "BSWAP_16") + set(HADOOP_BE16TOH "") + set(HADOOP_LE16TOH "BSWAP_16") + set(HADOOP_HTOBE32 "") + set(HADOOP_HTOLE32 "BSWAP_32") + set(HADOOP_BE32TOH "") + set(HADOOP_LE32TOH "BSWAP_32") + set(HADOOP_HTOBE64 "") + set(HADOOP_HTOLE64 "BSWAP_64") + set(HADOOP_BE64TOH "") + set(HADOOP_LE64TOH "BSWAP_64") + else() + set(HADOOP_HTOBE16 "BSWAP_16") + set(HADOOP_HTOLE16 "") + set(HADOOP_BE16TOH "BSWAP_16") + set(HADOOP_LE16TOH "") + set(HADOOP_HTOBE32 "BSWAP_32") + set(HADOOP_HTOLE32 "") + set(HADOOP_BE32TOH "BSWAP_32") + set(HADOOP_LE32TOH "") + set(HADOOP_HTOBE64 "BSWAP_64") + set(HADOOP_HTOLE64 "") + set(HADOOP_BE64TOH "BSWAP_64") + set(HADOOP_LE64TOH "") + endif() + set(_have_endian TRUE) + unset(_sys_byteorder_h) + unset(_bswap_64) + endif() +endif() + +# OSX uses libkern/OSByteOrder.h and OSSwapXtoY. +if(NOT _have_endian) + check_include_file("libkern/OSByteOrder.h" _libkern_osbyteorder_h) + if(_libkern_osbyteorder_h) + set(HADOOP_ENDIAN_H "libkern/OSByteOrder.h") + check_symbol_exists("OSSwapHostToLittleInt64" ${HADOOP_ENDIAN_H} _osswaphosttolittleint64) + endif() + if(_libkern_osbyteorder_h AND _osswaphosttolittleint64) + set(HADOOP_HTOBE16 "OSSwapHostToBigInt16") + set(HADOOP_HTOLE16 "OSSwapHostToLittleInt16") + set(HADOOP_BE16TOH "OSSwapBigToHostInt16") + set(HADOOP_LE16TOH "OSSwapLittleToHostInt16") + set(HADOOP_HTOBE32 "OSSwapHostToBigInt32") + set(HADOOP_HTOLE32 "OSSwapHostToLittleInt32") + set(HADOOP_BE32TOH "OSSwapBigToHostInt32") + set(HADOOP_LE32TOH "OSSwapLittleToHostInt32") + set(HADOOP_HTOBE64 "OSSwapHostToBigInt64") + set(HADOOP_HTOLE64 "OSSwapHostToLittleInt64") + set(HADOOP_BE64TOH "OSSwapBigToHostInt64") + set(HADOOP_LE64TOH "OSSwapLittleToHostInt64") + set(_have_endian TRUE) + unset(_libkern_osbyteorder_h) + unset(_osswaphosttolittleint64) + endif() +endif() + +# Bail if we don't know the endian definitions for this platform. +if(NOT _have_endian) + message(FATAL_ERROR "Can't provide endianness definitions for this platform") +endif() + +# Configure the hadoop_endian.h header file. +configure_file(${CMAKE_SOURCE_DIR}/hadoop_endian.h.cmake ${CMAKE_BINARY_DIR}/hadoop_endian.h) +unset(_bigendian) +unset(_have_endian) +unset(HADOOP_ENDIAN_H) + # Require zlib. set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) hadoop_set_find_shared_library_version("1") http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c index b3bb699..988ccf2 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c @@ -37,6 +37,7 @@ #include "crc32c_tables.h" #include "bulk_crc32.h" #include "gcc_optimizations.h" +#include "hadoop_endian.h" #define CRC_INITIAL_VAL 0xffffffff @@ -163,7 +164,7 @@ static uint32_t crc32c_sb8(uint32_t crc, const uint8_t *buf, size_t length) { for (li=0; li < running_length/8; li++) { uint32_t term1; uint32_t term2; - crc ^= *(uint32_t *)buf; + crc ^= hadoop_htole32(*(uint32_t *)buf); buf += 4; term1 = CRC32C_T8_7[crc & 0x000000FF] ^ CRC32C_T8_6[(crc >> 8) & 0x000000FF]; @@ -171,10 +172,10 @@ static uint32_t crc32c_sb8(uint32_t crc, const uint8_t *buf, size_t length) { crc = term1 ^ CRC32C_T8_5[term2 & 0x000000FF] ^ CRC32C_T8_4[(term2 >> 8) & 0x000000FF]; - term1 = CRC32C_T8_3[(*(uint32_t *)buf) & 0x000000FF] ^ - CRC32C_T8_2[((*(uint32_t *)buf) >> 8) & 0x000000FF]; + term1 = CRC32C_T8_3[hadoop_htole32(*(uint32_t *)buf) & 0x000000FF] ^ + CRC32C_T8_2[(hadoop_htole32(*(uint32_t *)buf) >> 8) & 0x000000FF]; - term2 = (*(uint32_t *)buf) >> 16; + term2 = hadoop_htole32((*(uint32_t *)buf)) >> 16; crc = crc ^ term1 ^ CRC32C_T8_1[term2 & 0x000000FF] ^ @@ -209,7 +210,7 @@ static uint32_t crc32_zlib_sb8( for (li=0; li < running_length/8; li++) { uint32_t term1; uint32_t term2; - crc ^= *(uint32_t *)buf; + crc ^= hadoop_htole32(*(uint32_t *)buf); buf += 4; term1 = CRC32_T8_7[crc & 0x000000FF] ^ CRC32_T8_6[(crc >> 8) & 0x000000FF]; @@ -217,10 +218,10 @@ static uint32_t crc32_zlib_sb8( crc = term1 ^ CRC32_T8_5[term2 & 0x000000FF] ^ CRC32_T8_4[(term2 >> 8) & 0x000000FF]; - term1 = CRC32_T8_3[(*(uint32_t *)buf) & 0x000000FF] ^ - CRC32_T8_2[((*(uint32_t *)buf) >> 8) & 0x000000FF]; + term1 = CRC32_T8_3[hadoop_htole32(*(uint32_t *)buf) & 0x000000FF] ^ + CRC32_T8_2[(hadoop_htole32(*(uint32_t *)buf) >> 8) & 0x000000FF]; - term2 = (*(uint32_t *)buf) >> 16; + term2 = hadoop_htole32(*(uint32_t *)buf) >> 16; crc = crc ^ term1 ^ CRC32_T8_1[term2 & 0x000000FF] ^ http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt index f878a94..99428b0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt @@ -90,6 +90,7 @@ include_directories( ${SRC}/src/util ${SRC}/src/lib ${SRC}/test + ../../../../hadoop-common-project/hadoop-common/target/native ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_BINARY_DIR} ${JNI_INCLUDE_DIRS} http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java index 1ec05db..df6570a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeComparable.java @@ -42,8 +42,8 @@ import org.apache.hadoop.classification.InterfaceStability; * * int HivePlatform::HiveKeyComparator(const char * src, uint32_t srcLength, * const char * dest, uint32_t destLength) { - * uint32_t sl = bswap(*(uint32_t*)src); - * uint32_t dl = bswap(*(uint32_t*)dest); + * uint32_t sl = hadoop_be32toh(*(uint32_t*)src); + * uint32_t dl = hadoop_be32toh(*(uint32_t*)dest); * return NativeObjectFactory::BytesComparator(src + 4, sl, dest + 4, dl); * } * http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h index ba026f5..f1336ef 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h @@ -40,15 +40,6 @@ enum NativeObjectType { BatchHandlerType = 1, }; -/** - * Enduim setting - * - */ -enum Endium { - LITTLE_ENDIUM = 0, - LARGE_ENDIUM = 1 -}; - #define NATIVE_COMBINER "native.combiner.class" #define NATIVE_PARTITIONER "native.partitioner.class" #define NATIVE_MAPPER "native.mapper.class" http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc index ce36239..7ce26f1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc @@ -104,8 +104,8 @@ int32_t BlockDecompressStream::read(void * buff, uint32_t length) { THROW_EXCEPTION(IOException, "readFully get incomplete data"); } _compressedBytesRead += rd; - sizes[0] = bswap(sizes[0]); - sizes[1] = bswap(sizes[1]); + sizes[0] = hadoop_be32toh(sizes[0]); + sizes[1] = hadoop_be32toh(sizes[1]); if (sizes[0] <= length) { uint32_t len = decompressOneBlock(sizes[1], buff, sizes[0]); if (len != sizes[0]) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc index 48c96b5..23c6c46 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc @@ -38,8 +38,8 @@ void Lz4CompressStream::compressOneBlock(const void * buff, uint32_t length) { int ret = LZ4_compress((char*)buff, _tempBuffer + 8, length); if (ret > 0) { compressedLength = ret; - ((uint32_t*)_tempBuffer)[0] = bswap(length); - ((uint32_t*)_tempBuffer)[1] = bswap((uint32_t)compressedLength); + ((uint32_t*)_tempBuffer)[0] = hadoop_be32toh(length); + ((uint32_t*)_tempBuffer)[1] = hadoop_be32toh((uint32_t)compressedLength); _stream->write(_tempBuffer, compressedLength + 8); _compressedBytesWritten += (compressedLength + 8); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc index a0417e0..04380ac 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc @@ -37,8 +37,8 @@ void SnappyCompressStream::compressOneBlock(const void * buff, uint32_t length) snappy_status ret = snappy_compress((const char*)buff, length, _tempBuffer + 8, &compressedLength); if (ret == SNAPPY_OK) { - ((uint32_t*)_tempBuffer)[0] = bswap(length); - ((uint32_t*)_tempBuffer)[1] = bswap((uint32_t)compressedLength); + ((uint32_t*)_tempBuffer)[0] = hadoop_be32toh(length); + ((uint32_t*)_tempBuffer)[1] = hadoop_be32toh((uint32_t)compressedLength); _stream->write(_tempBuffer, compressedLength + 8); _compressedBytesWritten += (compressedLength + 8); } else if (ret == SNAPPY_INVALID_INPUT) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc index 5f3863e..b18d057 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + #include "CombineHandler.h" namespace NativeTask { @@ -48,8 +49,8 @@ uint32_t CombineHandler::feedDataToJavaInWritableSerialization() { if (_kvCached) { uint32_t kvLength = _key.outerLength + _value.outerLength + KVBuffer::headerLength(); - outputInt(bswap(_key.outerLength)); - outputInt(bswap(_value.outerLength)); + outputInt(hadoop_be32toh(_key.outerLength)); + outputInt(hadoop_be32toh(_value.outerLength)); outputKeyOrValue(_key, _kType); outputKeyOrValue(_value, _vType); @@ -73,8 +74,8 @@ uint32_t CombineHandler::feedDataToJavaInWritableSerialization() { } else { firstKV = false; //write final key length and final value length - outputInt(bswap(_key.outerLength)); - outputInt(bswap(_value.outerLength)); + outputInt(hadoop_be32toh(_key.outerLength)); + outputInt(hadoop_be32toh(_value.outerLength)); outputKeyOrValue(_key, _kType); outputKeyOrValue(_value, _vType); @@ -101,7 +102,7 @@ void CombineHandler::outputKeyOrValue(SerializeInfo & KV, KeyValueType type) { output(KV.buffer.data(), KV.buffer.length()); break; case BytesType: - outputInt(bswap(KV.buffer.length())); + outputInt(hadoop_be32toh(KV.buffer.length())); output(KV.buffer.data(), KV.buffer.length()); break; default: @@ -202,8 +203,8 @@ void CombineHandler::write(char * buf, uint32_t length) { uint32_t outputRecordCount = 0; while (remain > 0) { kv = (KVBuffer *)pos; - kv->keyLength = bswap(kv->keyLength); - kv->valueLength = bswap(kv->valueLength); + kv->keyLength = hadoop_be32toh(kv->keyLength); + kv->valueLength = hadoop_be32toh(kv->valueLength); _writer->write(kv->getKey(), kv->keyLength, kv->getValue(), kv->valueLength); outputRecordCount++; remain -= kv->length(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc index 7e4ae44..4921b33 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc @@ -30,7 +30,7 @@ using std::vector; namespace NativeTask { MCollectorOutputHandler::MCollectorOutputHandler() - : _collector(NULL), _dest(NULL), _endium(LARGE_ENDIUM) { + : _collector(NULL), _dest(NULL) { } MCollectorOutputHandler::~MCollectorOutputHandler() { @@ -73,11 +73,9 @@ void MCollectorOutputHandler::handleInput(ByteBuffer & in) { THROW_EXCEPTION(IOException, "k/v meta information incomplete"); } - if (_endium == LARGE_ENDIUM) { - kvBuffer->partitionId = bswap(kvBuffer->partitionId); - kvBuffer->buffer.keyLength = bswap(kvBuffer->buffer.keyLength); - kvBuffer->buffer.valueLength = bswap(kvBuffer->buffer.valueLength); - } + kvBuffer->partitionId = hadoop_be32toh(kvBuffer->partitionId); + kvBuffer->buffer.keyLength = hadoop_be32toh(kvBuffer->buffer.keyLength); + kvBuffer->buffer.valueLength = hadoop_be32toh(kvBuffer->buffer.valueLength); uint32_t kvLength = kvBuffer->buffer.length(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.h index fe4635f..2e21806 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.h @@ -35,8 +35,6 @@ private: // state info for large KV pairs char * _dest; - Endium _endium; - public: MCollectorOutputHandler(); virtual ~MCollectorOutputHandler(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h index 4929426..09606d8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h @@ -115,7 +115,7 @@ public: * read uint32_t big endian */ inline uint32_t read_uint32_be() { - return bswap(read_uint32_le()); + return hadoop_be32toh(read_uint32_le()); } }; @@ -198,7 +198,7 @@ public: } inline void write_uint32_be(uint32_t v) { - write_uint32_le(bswap(v)); + write_uint32_le(hadoop_be32toh(v)); } inline void write_uint64_le(uint64_t v) { @@ -211,7 +211,7 @@ public: } inline void write_uint64_be(uint64_t v) { - write_uint64_le(bswap64(v)); + write_uint64_le(hadoop_be64toh(v)); } inline void write_vlong(int64_t v) { @@ -278,12 +278,11 @@ struct KVBuffer { } uint32_t length() { - return keyLength + valueLength + SIZE_OF_KEY_LENGTH + SIZE_OF_VALUE_LENGTH; + return keyLength + valueLength + SIZE_OF_KV_LENGTH; } uint32_t lengthConvertEndium() { - long value = bswap64(*((long *)this)); - return (value >> 32) + value + SIZE_OF_KEY_LENGTH + SIZE_OF_VALUE_LENGTH; + return hadoop_be32toh(keyLength) + hadoop_be32toh(valueLength) + SIZE_OF_KV_LENGTH; } void fill(const void * key, uint32_t keylen, const void * value, uint32_t vallen) { @@ -299,7 +298,7 @@ struct KVBuffer { } static uint32_t headerLength() { - return SIZE_OF_KEY_LENGTH + SIZE_OF_VALUE_LENGTH; + return SIZE_OF_KV_LENGTH; } }; http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc index 2d3e0b5..cbe1b28 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc @@ -60,7 +60,7 @@ bool IFileReader::nextPartition() { if (4 != _stream->readFully(&chsum, 4)) { THROW_EXCEPTION(IOException, "read ifile checksum failed"); } - uint32_t actual = bswap(chsum); + uint32_t actual = hadoop_be32toh(chsum); uint32_t expect = _source->getChecksum(); if (actual != expect) { THROW_EXCEPTION_EX(IOException, "read ifile checksum not match, actual %x expect %x", actual, @@ -130,7 +130,7 @@ void IFileWriter::endPartition() { } uint32_t chsum = _dest->getChecksum(); - chsum = bswap(chsum); + chsum = hadoop_be32toh(chsum); _stream->write(&chsum, sizeof(chsum)); _stream->flush(); IFileSegment * info = &(_spillFileSegments[_spillFileSegments.size() - 1]); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h index e397f90..414dc27 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h @@ -74,7 +74,7 @@ public: keyLen = WritableUtils::ReadVInt(kvbuff, len); break; case BytesType: - keyLen = bswap(*(uint32_t*)kvbuff); + keyLen = hadoop_be32toh(*(uint32_t*)kvbuff); len = 4; break; default: @@ -89,7 +89,7 @@ public: _valuePos = vbuff + len; break; case BytesType: - _valueLen = bswap(*(uint32_t*)vbuff); + _valueLen = hadoop_be32toh(*(uint32_t*)vbuff); _valuePos = vbuff + 4; break; default: http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc index 2185798..5633fcf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc @@ -317,8 +317,8 @@ int NativeObjectFactory::IntComparator(const char * src, uint32_t srcLength, con uint32_t destLength) { int result = (*src) - (*dest); if (result == 0) { - uint32_t from = bswap(*(uint32_t*)src); - uint32_t to = bswap(*(uint32_t*)dest); + uint32_t from = hadoop_be32toh(*(uint32_t*)src); + uint32_t to = hadoop_be32toh(*(uint32_t*)dest); if (from > to) { return 1; } else if (from == to) { @@ -335,8 +335,8 @@ int NativeObjectFactory::LongComparator(const char * src, uint32_t srcLength, co int result = (int)(*src) - (int)(*dest); if (result == 0) { - uint64_t from = bswap64(*(uint64_t*)src); - uint64_t to = bswap64(*(uint64_t*)dest); + uint64_t from = hadoop_be64toh(*(uint64_t*)src); + uint64_t to = hadoop_be64toh(*(uint64_t*)dest); if (from > to) { return 1; } else if (from == to) { @@ -380,8 +380,8 @@ int NativeObjectFactory::FloatComparator(const char * src, uint32_t srcLength, c THROW_EXCEPTION_EX(IOException, "float comparator, while src/dest lengt is not 4"); } - uint32_t from = bswap(*(uint32_t*)src); - uint32_t to = bswap(*(uint32_t*)dest); + uint32_t from = hadoop_be32toh(*(uint32_t*)src); + uint32_t to = hadoop_be32toh(*(uint32_t*)dest); float * srcValue = (float *)(&from); float * destValue = (float *)(&to); @@ -401,8 +401,8 @@ int NativeObjectFactory::DoubleComparator(const char * src, uint32_t srcLength, THROW_EXCEPTION_EX(IOException, "double comparator, while src/dest lengt is not 4"); } - uint64_t from = bswap64(*(uint64_t*)src); - uint64_t to = bswap64(*(uint64_t*)dest); + uint64_t from = hadoop_be64toh(*(uint64_t*)src); + uint64_t to = hadoop_be64toh(*(uint64_t*)dest); double * srcValue = (double *)(&from); double * destValue = (double *)(&to); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc index 9cff529..c1a36ce 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc @@ -58,10 +58,10 @@ void SingleSpillInfo::writeSpillInfo(const std::string & filepath) { appendBuffer.flush(); uint32_t chsum = dest.getChecksum(); #ifdef SPILLRECORD_CHECKSUM_UINT - chsum = bswap(chsum); + chsum = hadoop_be32toh(chsum); fout->write(&chsum, sizeof(uint32_t)); #else - uint64_t wtchsum = bswap64((uint64_t)chsum); + uint64_t wtchsum = hadoop_be64toh((uint64_t)chsum); fout->write(&wtchsum, sizeof(uint64_t)); #endif } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h index 57500b7..9c69f42 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h @@ -41,6 +41,7 @@ #include #include +#include "hadoop_endian.h" #include "lib/primitives.h" #include "lib/Log.h" #include "NativeTask.h" http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/primitives.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/primitives.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/primitives.h index 3bf5f76..8a74a63 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/primitives.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/primitives.h @@ -28,6 +28,7 @@ #include #include #include +#include "hadoop_endian.h" #ifdef __GNUC__ #define likely(x) __builtin_expect((x),1) @@ -94,39 +95,6 @@ inline void simple_memcpy(void * dest, const void * src, size_t len) { #endif /** - * little-endian to big-endian or vice versa - */ -inline uint32_t bswap(uint32_t val) { -#ifdef __aarch64__ - __asm__("rev %w[dst], %w[src]" : [dst]"=r"(val) : [src]"r"(val)); -#else - __asm__("bswap %0" : "=r" (val) : "0" (val)); -#endif - return val; -} - -inline uint64_t bswap64(uint64_t val) { -#ifdef __aarch64__ - __asm__("rev %[dst], %[src]" : [dst]"=r"(val) : [src]"r"(val)); -#else -#ifdef __X64 - __asm__("bswapq %0" : "=r" (val) : "0" (val)); -#else - - uint64_t lower = val & 0xffffffffU; - uint32_t higher = (val >> 32) & 0xffffffffU; - - lower = bswap(lower); - higher = bswap(higher); - - return (lower << 32) + higher; - -#endif -#endif - return val; -} - -/** * Fast memcmp */ inline int64_t fmemcmp(const char * src, const char * dest, uint32_t len) { @@ -158,16 +126,16 @@ inline int64_t fmemcmp(const char * src, const char * dest, uint32_t len) { return ((int64_t)src8[2] - (int64_t)dest8[2]); } case 4: { - return (int64_t)bswap(*(uint32_t*)src) - (int64_t)bswap(*(uint32_t*)dest); + return (int64_t)hadoop_be32toh(*(uint32_t*)src) - (int64_t)hadoop_be32toh(*(uint32_t*)dest); } } if (len < 8) { - int64_t ret = ((int64_t)bswap(*(uint32_t*)src) - (int64_t)bswap(*(uint32_t*)dest)); + int64_t ret = ((int64_t)hadoop_be32toh(*(uint32_t*)src) - (int64_t)hadoop_be32toh(*(uint32_t*)dest)); if (ret) { return ret; } - return ((int64_t)bswap(*(uint32_t*)(src + len - 4)) - - (int64_t)bswap(*(uint32_t*)(dest + len - 4))); + return ((int64_t)hadoop_be32toh(*(uint32_t*)(src + len - 4)) + - (int64_t)hadoop_be32toh(*(uint32_t*)(dest + len - 4))); } uint32_t cur = 0; uint32_t end = len & (0xffffffffU << 3); @@ -175,8 +143,8 @@ inline int64_t fmemcmp(const char * src, const char * dest, uint32_t len) { uint64_t l = *(uint64_t*)(src8 + cur); uint64_t r = *(uint64_t*)(dest8 + cur); if (l != r) { - l = bswap64(l); - r = bswap64(r); + l = hadoop_be64toh(l); + r = hadoop_be64toh(r); return l > r ? 1 : -1; } cur += 8; @@ -184,8 +152,8 @@ inline int64_t fmemcmp(const char * src, const char * dest, uint32_t len) { uint64_t l = *(uint64_t*)(src8 + len - 8); uint64_t r = *(uint64_t*)(dest8 + len - 8); if (l != r) { - l = bswap64(l); - r = bswap64(r); + l = hadoop_be64toh(l); + r = hadoop_be64toh(r); return l > r ? 1 : -1; } return 0; http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc index 8ed8dd2..b9f434e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc @@ -120,29 +120,29 @@ void WritableUtils::WriteVLongInner(int64_t v, char * pos, uint32_t & len) { len = 4; } else if (value < (1ULL << 32)) { *(pos++) = base - 3; - *(uint32_t*)(pos) = bswap((uint32_t)value); + *(uint32_t*)(pos) = hadoop_be32toh((uint32_t)value); len = 5; } else if (value < (1ULL << 40)) { *(pos++) = base - 4; - *(uint32_t*)(pos) = bswap((uint32_t)(value >> 8)); + *(uint32_t*)(pos) = hadoop_be32toh((uint32_t)(value >> 8)); *(uint8_t*)(pos + 4) = value; len = 6; } else if (value < (1ULL << 48)) { *(pos++) = base - 5; - *(uint32_t*)(pos) = bswap((uint32_t)(value >> 16)); + *(uint32_t*)(pos) = hadoop_be32toh((uint32_t)(value >> 16)); *(uint8_t*)(pos + 4) = value >> 8; *(uint8_t*)(pos + 5) = value; len = 7; } else if (value < (1ULL << 56)) { *(pos++) = base - 6; - *(uint32_t*)(pos) = bswap((uint32_t)(value >> 24)); + *(uint32_t*)(pos) = hadoop_be32toh((uint32_t)(value >> 24)); *(uint8_t*)(pos + 4) = value >> 16; *(uint8_t*)(pos + 5) = value >> 8; *(uint8_t*)(pos + 6) = value; len = 8; } else { *(pos++) = base - 7; - *(uint64_t*)pos = bswap64(value); + *(uint64_t*)pos = hadoop_be64toh(value); len = 9; } } @@ -168,7 +168,7 @@ int64_t WritableUtils::ReadLong(InputStream * stream) { if (stream->readFully(&ret, 8) != 8) { THROW_EXCEPTION(IOException, "ReadLong reach EOF"); } - return (int64_t)bswap64(ret); + return (int64_t)hadoop_be64toh(ret); } int32_t WritableUtils::ReadInt(InputStream * stream) { @@ -176,7 +176,7 @@ int32_t WritableUtils::ReadInt(InputStream * stream) { if (stream->readFully(&ret, 4) != 4) { THROW_EXCEPTION(IOException, "ReadInt reach EOF"); } - return (int32_t)bswap(ret); + return (int32_t)hadoop_be32toh(ret); } int16_t WritableUtils::ReadShort(InputStream * stream) { @@ -192,7 +192,7 @@ float WritableUtils::ReadFloat(InputStream * stream) { if (stream->readFully(&ret, 4) != 4) { THROW_EXCEPTION(IOException, "ReadFloat reach EOF"); } - ret = bswap(ret); + ret = hadoop_be32toh(ret); return *(float*)&ret; } @@ -232,12 +232,12 @@ void WritableUtils::WriteVLong(OutputStream * stream, int64_t v) { } void WritableUtils::WriteLong(OutputStream * stream, int64_t v) { - uint64_t be = bswap64((uint64_t)v); + uint64_t be = hadoop_be64toh((uint64_t)v); stream->write(&be, 8); } void WritableUtils::WriteInt(OutputStream * stream, int32_t v) { - uint32_t be = bswap((uint32_t)v); + uint32_t be = hadoop_be32toh((uint32_t)v); stream->write(&be, 4); } @@ -249,7 +249,7 @@ void WritableUtils::WriteShort(OutputStream * stream, int16_t v) { void WritableUtils::WriteFloat(OutputStream * stream, float v) { uint32_t intv = *(uint32_t*)&v; - intv = bswap(intv); + intv = hadoop_be32toh(intv); stream->write(&intv, 4); } @@ -286,10 +286,10 @@ void WritableUtils::toString(string & dest, KeyValueType type, const void * data dest.append(*(uint8_t*)data ? "true" : "false"); break; case IntType: - dest.append(StringUtil::ToString((int32_t)bswap(*(uint32_t*)data))); + dest.append(StringUtil::ToString((int32_t)hadoop_be32toh(*(uint32_t*)data))); break; case LongType: - dest.append(StringUtil::ToString((int64_t)bswap64(*(uint64_t*)data))); + dest.append(StringUtil::ToString((int64_t)hadoop_be64toh(*(uint64_t*)data))); break; case FloatType: dest.append(StringUtil::ToString(*(float*)data)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc index e1e32d4..93417b4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc @@ -190,7 +190,7 @@ TEST(IFile, TestGlibCBug) { reader->nextPartition(); uint32_t index = 0; while (NULL != (key = reader->nextKey(length))) { - int32_t realKey = (int32_t)bswap(*(uint32_t *)(key)); + int32_t realKey = (int32_t)hadoop_be32toh(*(uint32_t *)(key)); ASSERT_LT(index, 5); ASSERT_EQ(expect[index], realKey); index++; http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc index 1c391a6..6d40dc2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc @@ -38,8 +38,8 @@ inline int fmemcmporig(const char * src, const char * dest, uint32_t len) { uint64_t l = *src8; uint64_t r = *dest8; if (l != r) { - l = bswap64(l); - r = bswap64(r); + l = hadoop_be64toh(l); + r = hadoop_be64toh(r); return l > r ? 1 : -1; } ++src8; @@ -59,8 +59,8 @@ inline int fmemcmporig(const char * src, const char * dest, uint32_t len) { if (l == r) { return 0; } - l = bswap64(l); - r = bswap64(r); + l = hadoop_be64toh(l); + r = hadoop_be64toh(r); return l > r ? 1 : -1; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc index e47e169..dac79ba 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc @@ -43,8 +43,8 @@ TEST(KVBuffer, test) { ASSERT_EQ(8, kv1->getKey() - buff); ASSERT_EQ(strlen(KEY) + 8, kv1->getValue() - buff); - kv1->keyLength = bswap(kv1->keyLength); - kv1->valueLength = bswap(kv1->valueLength); + kv1->keyLength = hadoop_be32toh(kv1->keyLength); + kv1->valueLength = hadoop_be32toh(kv1->valueLength); ASSERT_EQ(8, kv1->headerLength()); ASSERT_EQ(strlen(KEY) + strlen(VALUE) + 8, kv1->lengthConvertEndium()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc index 8d784fb..4025e3c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc @@ -59,7 +59,7 @@ class MemoryBlockFactory { kv->keyLength = 4; kv->valueLength = 4; uint32_t * key = (uint32_t *)kv->getKey(); - *key = bswap(index); + *key = hadoop_be32toh(index); } return block1; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc index 6af73c5..fd9c29b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc @@ -85,17 +85,17 @@ TEST(MemoryBlock, sort) { medium->keyLength = 4; medium->valueLength = 4; uint32_t * mediumKey = (uint32_t *)medium->getKey(); - *mediumKey = bswap(MEDIUM); + *mediumKey = hadoop_be32toh(MEDIUM); small->keyLength = 4; small->valueLength = 4; uint32_t * smallKey = (uint32_t *)small->getKey(); - *smallKey = bswap(SMALL); + *smallKey = hadoop_be32toh(SMALL); big->keyLength = 4; big->valueLength = 4; uint32_t * bigKey = (uint32_t *)big->getKey(); - *bigKey = bswap(BIG); + *bigKey = hadoop_be32toh(BIG); ComparatorPtr bytesComparator = NativeTask::get_comparator(BytesType, NULL); block.sort(CPPSORT, bytesComparator); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6725e7f1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc index 79e1b5e..d13987a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "lib/commons.h" +#include "hadoop_endian.h" #include "test_commons.h" #include "lib/PartitionBucket.h" #include "lib/PartitionBucketIterator.h" @@ -129,15 +129,15 @@ TEST(PartitionBucket, sort) { const uint32_t BIG = 1000; kv1->keyLength = 4; - *((uint32_t *)kv1->getKey()) = bswap(BIG); + *((uint32_t *)kv1->getKey()) = hadoop_be32toh(BIG); kv1->valueLength = KV_SIZE - kv1->headerLength() - kv1->keyLength; kv2->keyLength = 4; - *((uint32_t *)kv2->getKey()) = bswap(SMALL); + *((uint32_t *)kv2->getKey()) = hadoop_be32toh(SMALL); kv2->valueLength = KV_SIZE - kv2->headerLength() - kv2->keyLength; kv3->keyLength = 4; - *((uint32_t *)kv3->getKey()) = bswap(MEDIUM); + *((uint32_t *)kv3->getKey()) = hadoop_be32toh(MEDIUM); kv3->valueLength = KV_SIZE - kv3->headerLength() - kv3->keyLength; bucket->sort(DUALPIVOTSORT); @@ -148,13 +148,13 @@ TEST(PartitionBucket, sort) { Buffer value; iter->next(key, value); - ASSERT_EQ(SMALL, bswap(*(uint32_t * )key.data())); + ASSERT_EQ(SMALL, hadoop_be32toh(*(uint32_t * )key.data())); iter->next(key, value); - ASSERT_EQ(MEDIUM, bswap(*(uint32_t * )key.data())); + ASSERT_EQ(MEDIUM, hadoop_be32toh(*(uint32_t * )key.data())); iter->next(key, value); - ASSERT_EQ(BIG, bswap(*(uint32_t * )key.data())); + ASSERT_EQ(BIG, hadoop_be32toh(*(uint32_t * )key.data())); delete iter; delete bucket; @@ -181,15 +181,15 @@ TEST(PartitionBucket, spill) { const uint32_t BIG = 1000; kv1->keyLength = 4; - *((uint32_t *)kv1->getKey()) = bswap(BIG); + *((uint32_t *)kv1->getKey()) = hadoop_be32toh(BIG); kv1->valueLength = KV_SIZE - KVBuffer::headerLength() - kv1->keyLength; kv2->keyLength = 4; - *((uint32_t *)kv2->getKey()) = bswap(SMALL); + *((uint32_t *)kv2->getKey()) = hadoop_be32toh(SMALL); kv2->valueLength = KV_SIZE - KVBuffer::headerLength() - kv2->keyLength; kv3->keyLength = 4; - *((uint32_t *)kv3->getKey()) = bswap(MEDIUM); + *((uint32_t *)kv3->getKey()) = hadoop_be32toh(MEDIUM); kv3->valueLength = KV_SIZE - KVBuffer::headerLength() - kv3->keyLength; bucket->sort(DUALPIVOTSORT); @@ -203,17 +203,17 @@ TEST(PartitionBucket, spill) { KVBuffer * first = (KVBuffer *)writer.buff(); ASSERT_EQ(4, first->keyLength); ASSERT_EQ(KV_SIZE - KVBuffer::headerLength() - 4, first->valueLength); - ASSERT_EQ(bswap(SMALL), (*(uint32_t * )(first->getKey()))); + ASSERT_EQ(hadoop_be32toh(SMALL), (*(uint32_t * )(first->getKey()))); KVBuffer * second = first->next(); ASSERT_EQ(4, second->keyLength); ASSERT_EQ(KV_SIZE - KVBuffer::headerLength() - 4, second->valueLength); - ASSERT_EQ(bswap(MEDIUM), (*(uint32_t * )(second->getKey()))); + ASSERT_EQ(hadoop_be32toh(MEDIUM), (*(uint32_t * )(second->getKey()))); KVBuffer * third = second->next(); ASSERT_EQ(4, third->keyLength); ASSERT_EQ(KV_SIZE - KVBuffer::headerLength() - 4, third->valueLength); - ASSERT_EQ(bswap(BIG), (*(uint32_t * )(third->getKey()))); + ASSERT_EQ(hadoop_be32toh(BIG), (*(uint32_t * )(third->getKey()))); delete [] buff; delete bucket;