Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D0174106A3 for ; Fri, 1 Nov 2013 00:55:50 +0000 (UTC) Received: (qmail 67455 invoked by uid 500); 1 Nov 2013 00:55:43 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 67249 invoked by uid 500); 1 Nov 2013 00:55:43 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 66249 invoked by uid 99); 1 Nov 2013 00:55:41 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 01 Nov 2013 00:55:41 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id A975B6F06; Fri, 1 Nov 2013 00:55:41 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ctubbsii@apache.org To: commits@accumulo.apache.org Date: Fri, 01 Nov 2013 00:56:00 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [21/54] [partial] ACCUMULO-658, ACCUMULO-656 Split server into separate modules http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/pom.xml ---------------------------------------------------------------------- diff --git a/server/pom.xml b/server/pom.xml deleted file mode 100644 index f7da3bb..0000000 --- a/server/pom.xml +++ /dev/null @@ -1,183 +0,0 @@ - - - - 4.0.0 - - org.apache.accumulo - accumulo-project - 1.6.0-SNAPSHOT - - accumulo-server - Server - - - com.beust - jcommander - - - com.google.code.gson - gson - - - jline - jline - - - org.apache.accumulo - accumulo-core - - - org.apache.accumulo - accumulo-fate - - - org.apache.accumulo - accumulo-start - - - org.apache.accumulo - accumulo-trace - - - org.apache.thrift - libthrift - - - commons-codec - commons-codec - provided - - - commons-collections - commons-collections - provided - - - commons-configuration - commons-configuration - provided - - - commons-io - commons-io - provided - - - commons-lang - commons-lang - provided - - - javax.servlet - servlet-api - provided - - - log4j - log4j - provided - - - org.apache.hadoop - hadoop-client - provided - - - org.apache.zookeeper - zookeeper - provided - - - org.mortbay.jetty - jetty - provided - - - junit - junit - test - - - org.slf4j - slf4j-api - test - - - org.slf4j - slf4j-log4j12 - test - - - - - - true - src/test/resources - - **/*.walog - - - - false - src/test/resources - - **/*.walog - - - - - - - org.apache.rat - apache-rat-plugin - - - src/main/resources/web/flot/**/*.js - - - - - - - - - native - - - - org.codehaus.mojo - exec-maven-plugin - - - generate-native-in-memory-map - - exec - - compile - - make - ${basedir}/src/main/c++ - - - - - - - - - http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/Makefile ---------------------------------------------------------------------- diff --git a/server/src/main/c++/Makefile b/server/src/main/c++/Makefile deleted file mode 100644 index 2ad4514..0000000 --- a/server/src/main/c++/Makefile +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# ya... I know, this is a bad recursive makefile... -all: nm - -nm: - cd nativeMap ; $(MAKE) - -clean: - cd nativeMap ; $(MAKE) $@ http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/BlockAllocator.h ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/BlockAllocator.h b/server/src/main/c++/nativeMap/BlockAllocator.h deleted file mode 100644 index d62ffd5..0000000 --- a/server/src/main/c++/nativeMap/BlockAllocator.h +++ /dev/null @@ -1,268 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - - -#ifndef _BLOCK_ALLOCATOR_H_ -#define _BLOCK_ALLOCATOR_H_ 1 - -#include -#include -#include -#include -#include -#include -#include - -struct Block { - unsigned char *data; - unsigned char *currentPos; - unsigned char *end; - unsigned char *prevPos; - - Block(uint32_t size){ - data = new unsigned char[size]; - end = data + size; - currentPos = data; - prevPos = NULL; - } - - ~Block(){ - } - - void *allocate(size_t amount){ - unsigned char *nextPos = currentPos + amount; - - if(nextPos > end){ - return NULL; - } - - prevPos = currentPos; - currentPos = nextPos; - return prevPos; - } - - size_t rollback(void *p){ - if(p == prevPos){ - size_t diff = currentPos - prevPos; - currentPos = prevPos; - return diff; - }else{ - std::cerr << "Tried to delete something that was not previous allocation " << p << " " << prevPos << std::endl; - exit(-1); - } - - return 0; - } - - size_t getMemoryFree() { - return end - currentPos; - } -}; - -struct BigBlock { - unsigned char *ptr; - size_t length; - - BigBlock(unsigned char *p, size_t len):ptr(p),length(len){} -}; - -struct LinkedBlockAllocator { - - std::vector blocks; - std::vector bigBlocks; - int blockSize; - int bigBlockSize; - int64_t memused; - void *lastAlloc; - - LinkedBlockAllocator(int blockSize, int bigBlockSize){ - this->blockSize = blockSize; - this->bigBlockSize = bigBlockSize; - lastAlloc = NULL; - memused = 0; - } - - void *allocate(size_t amount){ - - if(amount > (size_t)bigBlockSize){ - unsigned char *p = new unsigned char[amount]; - bigBlocks.push_back(BigBlock(p, amount)); - memused += sizeof(BigBlock) + amount; - return p; - }else{ - if(blocks.size() == 0){ - //do lazy allocation of memory, do not allocate a block until it is used - blocks.push_back(Block(blockSize)); - memused += sizeof(Block) + blockSize; - } - - lastAlloc = blocks.back().allocate(amount); - if(lastAlloc == NULL){ - blocks.push_back(Block(blockSize)); - lastAlloc = blocks.back().allocate(amount); - memused += sizeof(Block) + blockSize; - } - - return lastAlloc; - } - } - - void deleteLast(void *p){ - if(p != NULL){ - if(p == lastAlloc){ - blocks.back().rollback(p); - lastAlloc = NULL; - return; - }else if(bigBlocks.back().ptr == p){ - memused -= (sizeof(BigBlock) + bigBlocks.back().length); - bigBlocks.pop_back(); - delete((unsigned char *)p); - return; - } - } - - std::cerr << "Tried to delete something that was not last allocation " << p << " " << lastAlloc << std::endl; - exit(-1); - } - - size_t getMemoryUsed(){ - if(blocks.size() == 0) - return memused; - else - return memused - blocks.back().getMemoryFree(); - } - - ~LinkedBlockAllocator(){ - //std::cout << "Deleting " << blocks.size() << " blocks, memused : " << memused << std::endl; - std::vector::iterator iter = blocks.begin(); - while(iter != blocks.end()){ - delete [] (iter->data); - iter++; - } - - std::vector::iterator iter2 = bigBlocks.begin(); - while(iter2 != bigBlocks.end()){ - delete [] (iter2->ptr); - iter2++; - } - } - -}; - - - /** - * @brief An allocator that uses global new, as per [20.4]. - * - * This is precisely the allocator defined in the C++ Standard. - * - all allocation calls operator new - * - all deallocation calls operator delete - */ - template - class BlockAllocator - { - public: - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef _Tp* pointer; - typedef const _Tp* const_pointer; - typedef _Tp& reference; - typedef const _Tp& const_reference; - typedef _Tp value_type; - - LinkedBlockAllocator *lba; - - template - struct rebind - { typedef BlockAllocator<_Tp1> other; }; - - BlockAllocator() throw() { - lba = NULL; - } - - BlockAllocator(LinkedBlockAllocator *lba) throw() { - this->lba = lba; - } - - BlockAllocator(const BlockAllocator& ba) throw() { - lba = ba.lba; - } - - template - BlockAllocator(const BlockAllocator<_Tp1>& ba) throw() { - lba = ba.lba; - } - - ~BlockAllocator() throw() { } - - pointer - address(reference __x) const { return &__x; } - - const_pointer - address(const_reference __x) const { return &__x; } - - // NB: __n is permitted to be 0. The C++ standard says nothing - // about what the return value is when __n == 0. - pointer - allocate(size_type __n, const void* = 0) - { - if (__builtin_expect(__n > this->max_size(), false)) - std::__throw_bad_alloc(); - - - //void *p = ::operator new(__n * sizeof(_Tp)); - void *p = lba->allocate(__n * sizeof(_Tp)); - - //std::cout << "Allocating "<< name <<" " << __n * sizeof(_Tp) << " " << ((unsigned long long)p) % 4 << " " << ((unsigned long long)p) % 8 << std::endl; - - return static_cast<_Tp*>(p); - } - - // __p is not permitted to be a null pointer. - void - deallocate(pointer __p, size_type) - { - //::operator delete(__p); - } - - size_type - max_size() const throw() - { return size_t(-1) / sizeof(_Tp); } - - // _GLIBCXX_RESOLVE_LIB_DEFECTS - // 402. wrong new expression in [some_] allocator::construct - void - construct(pointer __p, const _Tp& __val) - { ::new(__p) _Tp(__val); } - - void - destroy(pointer __p) { __p->~_Tp(); } - - - }; - - - template - inline bool - operator==(const BlockAllocator<_Tp>& ba1, const BlockAllocator<_Tp>& ba2) - { return true; } - - template - inline bool - operator!=(const BlockAllocator<_Tp>& ba1, const BlockAllocator<_Tp>& ba2) - { return false; } - -#endif http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/Field.h ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/Field.h b/server/src/main/c++/nativeMap/Field.h deleted file mode 100644 index 7c3884e..0000000 --- a/server/src/main/c++/nativeMap/Field.h +++ /dev/null @@ -1,143 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -#include -#include -#include -#include -#include -#include -#include -#include "BlockAllocator.h" - -using namespace std; - -#ifndef __FIELD__ -#define __FIELD__ - -struct Field { - uint8_t *field; - int32_t len; - - int compare(const uint8_t *d1, int len1, const uint8_t *d2, int len2) const{ - int result = memcmp(d1, d2, len1 < len2 ? len1 : len2); - - if(result != 0) - return result; - if(len1 == len2) - return 0; - if(len1 < len2) - return -1; - - return 1; - } - - Field(){} - - Field(LinkedBlockAllocator *lba, JNIEnv *env, jbyteArray f, int l){ - len = l; - field=(uint8_t *)lba->allocate(len); - env->GetByteArrayRegion(f, 0, len, (jbyte *)field); - } - - Field(LinkedBlockAllocator *lba, JNIEnv *env, jbyteArray f){ - len = env->GetArrayLength(f); - field=(uint8_t *)lba->allocate(len); - env->GetByteArrayRegion(f, 0, len, (jbyte *)field); - } - - Field(uint8_t *f, int32_t l):field(f),len(l){ - } - - Field(const char *cstr){ - //constructor for testing C++ - len = strlen(cstr); - field=new uint8_t[len]; - memcpy(field, cstr, len); - } - - Field(LinkedBlockAllocator *lba, const char *cstr){ - //constructor for testing C++ - len = strlen(cstr); - field=(uint8_t *)lba->allocate(len); - memcpy(field, cstr, len); - } - - void set(const char *d, int l){ - if(l < 0 || l > len){ - cerr << "Tried to set field with value that is too long " << l << " " << len << endl; - } - memcpy(field, d, l); - len = l; - } - - void set(JNIEnv *env, jbyteArray f, int l){ - if(l < 0 || l > len){ - cerr << "Tried to set field with value that is too long " << l << " " << len << endl; - } - len = l; - env->GetByteArrayRegion(f, 0, len, (jbyte *)field); - } - - int compare(const Field &of) const{ - return compare(field, len, of.field, of.len); - } - - bool operator<(const Field &of) const{ - return compare(of) < 0; - } - - int32_t length() const { - return len; - } - - void fillIn(JNIEnv *env, jbyteArray d) const { - //TODO ensure lengths match up - env->SetByteArrayRegion(d, 0, len, (jbyte *)field); - } - - jbyteArray createJByteArray(JNIEnv *env) const{ - jbyteArray valData = env->NewByteArray(len); - env->SetByteArrayRegion(valData, 0, len, (jbyte *)field); - return valData; - } - - string toString() const{ - return string((char *)field, len); - } - - void clear(){ - //delete(field); - } - - void clear(LinkedBlockAllocator *lba){ - lba->deleteLast(field); - } -}; - -struct LocalField : public Field { - LocalField(JNIEnv *env, jbyteArray f){ - len = env->GetArrayLength(f); - field= new uint8_t[len]; - env->GetByteArrayRegion(f, 0, len, (jbyte *)field); - } - - ~LocalField(){ - delete(field); - } -}; -#endif - http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/Key.h ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/Key.h b/server/src/main/c++/nativeMap/Key.h deleted file mode 100644 index f902426..0000000 --- a/server/src/main/c++/nativeMap/Key.h +++ /dev/null @@ -1,143 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -#include -#include -#include -#include -#include - -using namespace std; - -class Key { - -public: - int32_t colFamilyOffset; - int32_t colQualifierOffset; - int32_t colVisibilityOffset; - int32_t totalLen; - - uint8_t *keyData; - - int64_t timestamp; - bool deleted; - - int compare(const uint8_t *d1, int len1, const uint8_t *d2, int len2) const{ - int result = memcmp(d1, d2, len1 < len2 ? len1 : len2); - - if(result != 0) - return result; - if(len1 == len2) - return 0; - if(len1 < len2) - return -1; - - return 1; - } - - /** - * Constructor for testing purposes - */ - - Key(){} - - Key(const string &r, const string &cf, const string &cq, const string &cv, long ts, bool del){ - - colFamilyOffset = r.length(); - colQualifierOffset = colFamilyOffset + cf.length(); - colVisibilityOffset = colQualifierOffset + cq.length(); - totalLen = colVisibilityOffset + cv.length(); - - keyData = new uint8_t[totalLen]; - - copy(r.begin(), r.end(), keyData); - copy(cf.begin(), cf.end(), keyData+colFamilyOffset); - copy(cq.begin(), cq.end(), keyData+colQualifierOffset); - copy(cv.begin(), cv.end(), keyData+colVisibilityOffset); - - timestamp = ts; - deleted = del; - } - - /** - * Constructor used for taking data from Java Key - */ - - Key(JNIEnv *env, jbyteArray kd, jint cfo, jint cqo, jint cvo, jint tl, jlong ts, jboolean del){ - - colFamilyOffset = cfo; - colQualifierOffset = cqo; - colVisibilityOffset = cvo; - totalLen = tl; - timestamp = ts; - deleted = del == JNI_TRUE ? true : false; - - keyData = new uint8_t[totalLen]; - env->GetByteArrayRegion(kd, 0, totalLen, (jbyte *)keyData); - } - - bool operator<(const Key &key) const{ - int result = compare(keyData, colFamilyOffset, key.keyData, key.colFamilyOffset); - if(result != 0) return result < 0; - - result = compare(keyData + colFamilyOffset, colQualifierOffset - colFamilyOffset, key.keyData + key.colFamilyOffset, key.colQualifierOffset - key.colFamilyOffset); - if(result != 0) return result < 0; - - result = compare(keyData + colQualifierOffset, colVisibilityOffset - colQualifierOffset, key.keyData + key.colQualifierOffset, key.colVisibilityOffset - key.colQualifierOffset); - if(result != 0) return result < 0; - - result = compare(keyData + colVisibilityOffset, totalLen - colVisibilityOffset, key.keyData + key.colVisibilityOffset, key.totalLen - key.colVisibilityOffset); - if(result != 0) return result < 0; - - if(timestamp < key.timestamp){ - return false; - }else if(timestamp > key.timestamp){ - return true; - } - - return deleted && !key.deleted; - } - -}; - - -class LocalKey : public Key { - -public: - - JNIEnv *envPtr; - jbyteArray kd; - - LocalKey(JNIEnv *env, jbyteArray kd, jint cfo, jint cqo, jint cvo, jint tl, jlong ts, jboolean del){ - envPtr = env; - - colFamilyOffset = cfo; - colQualifierOffset = cqo; - colVisibilityOffset = cvo; - totalLen = tl; - timestamp = ts; - deleted = del == JNI_TRUE ? true : false; - - this->kd = kd; - keyData = (uint8_t *)env->GetByteArrayElements((jbyteArray)kd, NULL); - } - - ~LocalKey(){ - envPtr->ReleaseByteArrayElements(kd, (jbyte *)keyData, JNI_ABORT); - } - -}; - http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/Makefile ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/Makefile b/server/src/main/c++/nativeMap/Makefile deleted file mode 100644 index 4e3aa55..0000000 --- a/server/src/main/c++/nativeMap/Makefile +++ /dev/null @@ -1,82 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SRCS=$(filter-out test.cc,$(wildcard *.cc)) -HDRS=$(wildcard *.h) org_apache_accumulo_server_tabletserver_NativeMap.h -CXX=g++ - -ifeq ($(shell uname),Linux) - LIBS_32 := libNativeMap-Linux-i386-32.so - LIBS_64 := libNativeMap-Linux-amd64-64.so - - ifneq ($(DARCH),) - ifeq ($(DARCH),64) - LIBS := $(LIBS_64) - endif - ifeq ($(DARCH),32) - LIBS := $(LIBS_32) - endif - ifeq ($(DARCH),both) - LIBS := $(LIBS_64) $(LIBS_32) - endif - ifeq ($(LIBS),) - LIBS := $(LIBS_64) $(LIBS_32) - endif - else - DARCH := $(shell getconf LONG_BIT) - LIBS := $(LIBS_$(DARCH)) - endif - - CXXFLAGS=-g -fPIC -shared -O2 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -I$(JAVA_HOME)/include/linux -I$(JAVA_HOME)/include -endif - -ifeq ($(shell uname),Darwin) - LIBS:= libNativeMap-Mac_OS_X-x86_64-64.jnilib - # Update flags for OSX-10.9 and Xcode 5.0.1 - # I think that we should be able to remove the `-undefined dynamic_lookup` option, - # but I don't know exactly how to go about this. - CXXFLAGS=-m64 -dynamiclib -undefined dynamic_lookup -O3 -I/System/Library/Frameworks/JavaVM.framework/Headers -I${JAVA_HOME}/include -I${JAVA_HOME}/include/darwin \ - -I/usr/include/c++/4.2.1 -endif - -INSTALL_DIR=../../../../../lib/native/map -INSTALLED_LIBS=$(patsubst %,$(INSTALL_DIR)/%,$(LIBS)) - -all : $(INSTALLED_LIBS) - -clean: - rm -f $(INSTALLED_LIBS) $(LIBS) - -org_apache_accumulo_server_tabletserver_NativeMap.h : - javah -classpath ../../../../../lib/accumulo-server.jar org.apache.accumulo.server.tabletserver.NativeMap - -test: $(SRCS) $(HDRS) - g++ -g -o test -Wall -I$(JAVA_HOME)/include/linux -I$(JAVA_HOME)/include $(SRCS) test.cc - -$(INSTALLED_LIBS) : $(INSTALL_DIR) $(LIBS) - cp $(LIBS) $(INSTALL_DIR) - -$(INSTALL_DIR): - mkdir -p $@ - -libNativeMap-Linux-amd64-64.so : $(SRCS) $(HDRS) - $(CXX) $(CXXFLAGS) -m64 -o $@ $(SRCS) - -libNativeMap-Linux-i386-32.so : $(SRCS) $(HDRS) - $(CXX) $(CXXFLAGS) -m32 -o $@ $(SRCS) - -libNativeMap-Mac_OS_X-x86_64-64.jnilib : $(SRCS) $(HDRS) - $(CXX) $(CXXFLAGS) -m64 -o $@ $(SRCS) - http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/NativeMap.h ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/NativeMap.h b/server/src/main/c++/nativeMap/NativeMap.h deleted file mode 100644 index bc44a98..0000000 --- a/server/src/main/c++/nativeMap/NativeMap.h +++ /dev/null @@ -1,208 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -#ifndef __NATIVE_MAP_A12__ -#define __NATIVE_MAP_A12__ - -#include "SubKey.h" -#include "Field.h" -#include "BlockAllocator.h" -#include -#include -#include - -using namespace std; - -typedef map, BlockAllocator > > ColumnMap; -typedef map, BlockAllocator > > RowMap; - - -struct NativeMapData { - LinkedBlockAllocator *lba; - RowMap rowmap; - int count; - - NativeMapData(int blockSize, int bigBlockSize):lba(new LinkedBlockAllocator(blockSize, bigBlockSize)), - rowmap(RowMap(std::less(), BlockAllocator >(lba))){ - } - - ~NativeMapData(){ - rowmap.clear(); //if row map is not cleared here, it will be deconstructed after lba is deleted - delete(lba); - } -}; - - -struct Iterator { - NativeMapData &nativeMap; - RowMap::iterator rowIter; - ColumnMap::iterator colIter; - - Iterator(NativeMapData &nm, int32_t *ia):nativeMap(nm){ - rowIter = nativeMap.rowmap.begin(); - if(rowIter == nativeMap.rowmap.end()){ - return; - } - - colIter = rowIter->second.begin(); - - skipAndFillIn(ia, true); - } - - - Iterator(NativeMapData &nm, Field &row, SubKey &sk, int32_t *ia):nativeMap(nm){ - rowIter = nativeMap.rowmap.lower_bound(row); - if(rowIter == nativeMap.rowmap.end()){ - return; - } - - //TODO use equals instead of compare - if(rowIter->first.compare(row) == 0){ - colIter = rowIter->second.lower_bound(sk); - }else{ - colIter = rowIter->second.begin(); - } - - skipAndFillIn(ia, true); - } - - bool skipAndFillIn(int32_t *ia, bool firstCall) - { - bool rowChanged = false; - - while(colIter == rowIter->second.end()){ - rowIter++; - rowChanged = true; - if(rowIter == nativeMap.rowmap.end()){ - return false; - } - colIter = rowIter->second.begin(); - } - - ia[0] = (firstCall || rowChanged) ? rowIter->first.length() : -1; - ia[1] = colIter->first.getCFLen(); - ia[2] = colIter->first.getCQLen(); - ia[3] = colIter->first.getCVLen(); - ia[4] = colIter->first.isDeleted() ? 1 : 0; - ia[5] = colIter->second.length(); - ia[6] = colIter->first.getMC(); - - return true; - } - - bool atEnd(){ - return rowIter == nativeMap.rowmap.end(); - } - - void advance(int32_t *ia){ - colIter++; - skipAndFillIn(ia, false); - } -}; - -struct NativeMap : public NativeMapData { - - NativeMap(int blockSize, int bigBlockSize):NativeMapData(blockSize, bigBlockSize){ - count = 0; - } - - ~NativeMap(){ - - } - - - ColumnMap *startUpdate(JNIEnv * env, jbyteArray r){ - Field row(lba, env, r); - return startUpdate(row); - } - - ColumnMap *startUpdate(const char *r){ - - Field row(lba, r); - return startUpdate(row); - } - - ColumnMap *startUpdate(Field &row){ - //cout << "Starting update "< insertResult = rowmap.insert(pair(row, ColumnMap(std::less(), BlockAllocator >(lba)))); - - if(!insertResult.second) - row.clear(lba); - - return &(insertResult.first->second); - - } - - void update(ColumnMap *cm, JNIEnv *env, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del, jbyteArray val, jint mutationCount){ - - SubKey sk(lba, env, cf, cq, cv, ts, del, mutationCount); - //cout << "Updating " << sk.toString() << " " << sk.getTimestamp() << " " << sk.isDeleted() << endl; - //do not bother allocating value if not needed - Field value(NULL, 0); - - pair insertResult = cm->insert(pair(sk, value)); - if(insertResult.second){ - insertResult.first->second = Field(lba, env, val); - count++; - }else{ - sk.clear(lba); - int valLen = env->GetArrayLength(val); - if(valLen <= insertResult.first->second.length()){ - insertResult.first->second.set(env, val, valLen); - }else{ - insertResult.first->second.clear(); - insertResult.first->second = Field(lba, env, val, valLen); - } - } - } - - void update(ColumnMap *cm, const char *cf, const char *cq, const char *cv, long ts, bool del, const char *val, int valLen, int mutationCount){ - - SubKey sk(lba, cf, cq, cv, ts, del, mutationCount); - //do not bother allocating value if not needed - Field value(NULL, 0); - - pair insertResult = cm->insert(pair(sk, value)); - if(insertResult.second){ - insertResult.first->second = Field(lba, val); - count++; - }else{ - sk.clear(lba); - if(insertResult.first->second.length() <= valLen){ - insertResult.first->second.set(val, valLen); - }else{ - insertResult.first->second.clear(); - insertResult.first->second = Field(lba, val); - } - } - } - - Iterator *iterator(int32_t *ia){ - return new Iterator(*this, ia); - } - - Iterator *iterator(Field &row, SubKey &sk, int32_t *ia){ - return new Iterator(*this, row, sk, ia); - } - - int64_t getMemoryUsed(){ - return lba->getMemoryUsed(); - } -}; - -#endif - http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/SubKey.h ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/SubKey.h b/server/src/main/c++/nativeMap/SubKey.h deleted file mode 100644 index 59e7a6d..0000000 --- a/server/src/main/c++/nativeMap/SubKey.h +++ /dev/null @@ -1,193 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -#include -#include -#include "Field.h" -#include "BlockAllocator.h" - -using namespace std; - -#ifndef __SUB_KEY__ -#define __SUB_KEY__ - -class SubKey { - -public: - - int32_t colQualifierOffset; - int32_t colVisibilityOffset; - int32_t totalLen; - - uint8_t *keyData; - - int64_t timestamp; - - int32_t mutationCount; - - bool deleted; - - int compare(const uint8_t *d1, int len1, const uint8_t *d2, int len2) const{ - int result = memcmp(d1, d2, len1 < len2 ? len1 : len2); - - if(result != 0) - return result; - if(len1 == len2) - return 0; - if(len1 < len2) - return -1; - - return 1; - } - - /** - * Constructor for testing purposes - */ - - SubKey(LinkedBlockAllocator *lba, const string &cf, const string &cq, const string &cv, int64_t ts, bool del, int32_t mc){ - - colQualifierOffset = cf.length(); - colVisibilityOffset = colQualifierOffset + cq.length(); - totalLen = colVisibilityOffset + cv.length(); - - keyData = (uint8_t *)lba->allocate(totalLen); - - copy(cf.begin(), cf.end(), keyData); - copy(cq.begin(), cq.end(), keyData+colQualifierOffset); - copy(cv.begin(), cv.end(), keyData+colVisibilityOffset); - - timestamp = ts; - deleted = del; - - mutationCount = mc; - } - - SubKey(LinkedBlockAllocator *lba, JNIEnv *env, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del, int32_t mc){ - - int cfLen = env->GetArrayLength(cf); - int cqLen = env->GetArrayLength(cq); - int cvLen = env->GetArrayLength(cv); - - colQualifierOffset = cfLen; - colVisibilityOffset = colQualifierOffset + cqLen; - totalLen = colVisibilityOffset + cvLen; - - if(lba == NULL) - keyData = new uint8_t[totalLen]; - else - keyData = (uint8_t *)lba->allocate(totalLen); - - - env->GetByteArrayRegion(cf, 0, cfLen, (jbyte *)keyData); - env->GetByteArrayRegion(cq, 0, cqLen, (jbyte *)(keyData+colQualifierOffset)); - env->GetByteArrayRegion(cv, 0, cvLen, (jbyte *)(keyData+colVisibilityOffset)); - - timestamp = ts; - deleted = del; - - mutationCount = mc; - } - - - bool operator<(const SubKey &key) const{ - - int result = compare(keyData, colQualifierOffset, key.keyData, key.colQualifierOffset); - if(result != 0) return result < 0; - - result = compare(keyData + colQualifierOffset, colVisibilityOffset - colQualifierOffset, key.keyData + key.colQualifierOffset, key.colVisibilityOffset - key.colQualifierOffset); - if(result != 0) return result < 0; - - result = compare(keyData + colVisibilityOffset, totalLen - colVisibilityOffset, key.keyData + key.colVisibilityOffset, key.totalLen - key.colVisibilityOffset); - if(result != 0) return result < 0; - - if(timestamp < key.timestamp){ - return false; - }else if(timestamp > key.timestamp){ - return true; - } - - if(deleted != key.deleted) - return deleted && !key.deleted; - - return mutationCount > key.mutationCount; - } - - void clear(){ - //delete(keyData); - } - - void clear(LinkedBlockAllocator *lba){ - lba->deleteLast(keyData); - } - - - int64_t bytesUsed() const{ - return totalLen + 9; - } - - bool isDeleted() const{ - return deleted; - } - - int32_t getCFLen() const{ - return colQualifierOffset; - } - - int32_t getCQLen() const { - return colVisibilityOffset - colQualifierOffset; - } - - int32_t getCVLen() const { - return totalLen - colVisibilityOffset; - } - - const Field getCF() const{ - return Field(keyData, getCFLen()); - } - - const Field getCQ() const{ - return Field(keyData + colQualifierOffset, getCQLen()); - } - - const Field getCV() const{ - return Field(keyData + colVisibilityOffset, getCVLen()); - } - - string toString() const{ - return getCF().toString()+":"+getCQ().toString()+":"+getCV().toString(); - } - - int64_t getTimestamp() const{ - return timestamp; - } - - int32_t getMC() const{ - return mutationCount; - } - -}; - -struct LocalSubKey : public SubKey { - - LocalSubKey(JNIEnv *env, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del):SubKey(NULL, env, cf, cq, cv, ts, del, INT_MAX){} - - ~LocalSubKey(){ - delete(keyData); - } -}; - -#endif - http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap.cc ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap.cc b/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap.cc deleted file mode 100644 index 5049bc8..0000000 --- a/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap.cc +++ /dev/null @@ -1,149 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -#include "org_apache_accumulo_server_tabletserver_NativeMap.h" -#include "SubKey.h" -#include "Field.h" -#include "NativeMap.h" -#include -#include -#include -#include - -#ifdef _POSIX_MEMLOCK -#include -#endif - -using namespace std; - -JNIEXPORT jlong JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_createNM(JNIEnv *env, jclass cls){ - return (jlong)(new NativeMap(1<<17, 1<<11)); -} - -JNIEXPORT jint JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_sizeNM(JNIEnv *env, jclass cls, jlong nm) -{ - return ((NativeMap *)nm)->count; -} - -JNIEXPORT jlong JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_memoryUsedNM(JNIEnv *env, jclass cls, jlong nm) -{ - return ((NativeMap *)nm)->getMemoryUsed(); -} - -JNIEXPORT void JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_singleUpdate(JNIEnv *env, jclass cls, jlong nm, jbyteArray r, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del, jbyteArray val, jint mutationCount) -{ - jlong uid = Java_org_apache_accumulo_server_tabletserver_NativeMap_startUpdate(env, cls, nm, r); - Java_org_apache_accumulo_server_tabletserver_NativeMap_update(env, cls, nm, uid, cf, cq, cv, ts, del, val, mutationCount); -} - -JNIEXPORT jlong JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_startUpdate(JNIEnv *env, jclass cls, jlong nm, jbyteArray r) -{ - NativeMap *nativeMap = (NativeMap *)nm; - - ColumnMap *cm = nativeMap->startUpdate(env, r); - - return (jlong)cm; -} - -JNIEXPORT void JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_update(JNIEnv *env, jclass cls, jlong nm, jlong uid, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del, jbyteArray val, jint mutationCount) -{ - NativeMap *nativeMap = (NativeMap *)nm; - nativeMap->update((ColumnMap *)uid, env, cf, cq, cv, ts, del, val, mutationCount); -} - -JNIEXPORT jlong JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_deleteNM(JNIEnv *env, jclass cls, jlong nm) -{ - NativeMap *nativeMap = (NativeMap *)nm; - delete(nativeMap); - return 0; -} - - -JNIEXPORT jlong JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_createNMI__J_3I(JNIEnv *env, jclass cls, jlong nm, jintArray lens) -{ - NativeMap *nativeMap = (NativeMap *)nm; - - int32_t ia[7]; - Iterator *iter = nativeMap->iterator(ia); - if(iter->atEnd()){ - delete(iter); - return 0; - } - - env->SetIntArrayRegion(lens, 0, 7, ia); - - return (jlong)iter; - -} - -JNIEXPORT jlong JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_createNMI__J_3B_3B_3B_3BJZ_3I(JNIEnv *env, jclass cls, jlong nm, jbyteArray r, jbyteArray cf, jbyteArray cq, jbyteArray cv, jlong ts, jboolean del, jintArray lens) -{ - NativeMap *nativeMap = (NativeMap *)nm; - - LocalField row(env, r); - LocalSubKey sk(env, cf, cq, cv, ts, del); - - int32_t ia[7]; - Iterator *iter = nativeMap->iterator(row, sk, ia); - - if(iter->atEnd()){ - delete(iter); - return 0; - } - - env->SetIntArrayRegion(lens, 0, 7, ia); - - return (jlong)iter; - -} - -JNIEXPORT jboolean JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_nmiNext(JNIEnv *env, jclass cls, jlong ip, jintArray lens){ - Iterator &iter = *((Iterator *)ip); - - int32_t ia[7]; - iter.advance(ia); - if(iter.atEnd()){ - return false; - } - - env->SetIntArrayRegion(lens, 0, 7, ia); - return true; -} - -JNIEXPORT void JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_nmiGetData(JNIEnv *env, jclass cls, jlong ip, jbyteArray r, jbyteArray cf, jbyteArray cq, jbyteArray cv, jbyteArray val){ - Iterator &iter = *((Iterator *)ip); - - if(r != NULL){ - iter.rowIter->first.fillIn(env, r); - } - - iter.colIter->first.getCF().fillIn(env, cf); - iter.colIter->first.getCQ().fillIn(env, cq); - iter.colIter->first.getCV().fillIn(env, cv); - iter.colIter->second.fillIn(env, val); -} - -JNIEXPORT jlong JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_nmiGetTS(JNIEnv *env, jclass cls, jlong ip){ - Iterator &iter = *((Iterator *)ip); - return iter.colIter->first.getTimestamp(); -} - - - -JNIEXPORT void JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_deleteNMI(JNIEnv *env, jclass cls, jlong ip){ - delete((Iterator *)ip); -} - http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap.h ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap.h b/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap.h deleted file mode 100644 index 1082f33..0000000 --- a/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap.h +++ /dev/null @@ -1,133 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -/* Header for class org_apache_accumulo_server_tabletserver_NativeMap */ - -#ifndef _Included_org_apache_accumulo_server_tabletserver_NativeMap -#define _Included_org_apache_accumulo_server_tabletserver_NativeMap -#ifdef __cplusplus -extern "C" { -#endif -/* - * Class: org_apache_accumulo_server_tabletserver_NativeMap - * Method: createNM - * Signature: ()J - */ -JNIEXPORT jlong JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_createNM - (JNIEnv *, jclass); - -/* - * Class: org_apache_accumulo_server_tabletserver_NativeMap - * Method: singleUpdate - * Signature: (J[B[B[B[BJZ[BI)V - */ -JNIEXPORT void JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_singleUpdate - (JNIEnv *, jclass, jlong, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jlong, jboolean, jbyteArray, jint); - -/* - * Class: org_apache_accumulo_server_tabletserver_NativeMap - * Method: startUpdate - * Signature: (J[B)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_startUpdate - (JNIEnv *, jclass, jlong, jbyteArray); - -/* - * Class: org_apache_accumulo_server_tabletserver_NativeMap - * Method: update - * Signature: (JJ[B[B[BJZ[BI)V - */ -JNIEXPORT void JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_update - (JNIEnv *, jclass, jlong, jlong, jbyteArray, jbyteArray, jbyteArray, jlong, jboolean, jbyteArray, jint); - -/* - * Class: org_apache_accumulo_server_tabletserver_NativeMap - * Method: sizeNM - * Signature: (J)I - */ -JNIEXPORT jint JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_sizeNM - (JNIEnv *, jclass, jlong); - -/* - * Class: org_apache_accumulo_server_tabletserver_NativeMap - * Method: memoryUsedNM - * Signature: (J)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_memoryUsedNM - (JNIEnv *, jclass, jlong); - -/* - * Class: org_apache_accumulo_server_tabletserver_NativeMap - * Method: deleteNM - * Signature: (J)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_deleteNM - (JNIEnv *, jclass, jlong); - -/* - * Class: org_apache_accumulo_server_tabletserver_NativeMap - * Method: createNMI - * Signature: (J[I)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_createNMI__J_3I - (JNIEnv *, jclass, jlong, jintArray); - -/* - * Class: org_apache_accumulo_server_tabletserver_NativeMap - * Method: createNMI - * Signature: (J[B[B[B[BJZ[I)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_createNMI__J_3B_3B_3B_3BJZ_3I - (JNIEnv *, jclass, jlong, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jlong, jboolean, jintArray); - -/* - * Class: org_apache_accumulo_server_tabletserver_NativeMap - * Method: nmiNext - * Signature: (J[I)Z - */ -JNIEXPORT jboolean JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_nmiNext - (JNIEnv *, jclass, jlong, jintArray); - -/* - * Class: org_apache_accumulo_server_tabletserver_NativeMap - * Method: nmiGetData - * Signature: (J[B[B[B[B[B)V - */ -JNIEXPORT void JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_nmiGetData - (JNIEnv *, jclass, jlong, jbyteArray, jbyteArray, jbyteArray, jbyteArray, jbyteArray); - -/* - * Class: org_apache_accumulo_server_tabletserver_NativeMap - * Method: nmiGetTS - * Signature: (J)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_nmiGetTS - (JNIEnv *, jclass, jlong); - -/* - * Class: org_apache_accumulo_server_tabletserver_NativeMap - * Method: deleteNMI - * Signature: (J)V - */ -JNIEXPORT void JNICALL Java_org_apache_accumulo_server_tabletserver_NativeMap_deleteNMI - (JNIEnv *, jclass, jlong); - -#ifdef __cplusplus -} -#endif -#endif http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator.h ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator.h b/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator.h deleted file mode 100644 index fedfdf1..0000000 --- a/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator.h +++ /dev/null @@ -1,33 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -/* Header for class org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator */ - -#ifndef _Included_org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator -#define _Included_org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator -#ifdef __cplusplus -extern "C" { -#endif -#undef org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator_MAX_READ_AHEAD_ENTRIES -#define org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator_MAX_READ_AHEAD_ENTRIES 16L -#undef org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator_READ_AHEAD_BYTES -#define org_apache_accumulo_server_tabletserver_NativeMap_ConcurrentIterator_READ_AHEAD_BYTES 4096L -#ifdef __cplusplus -} -#endif -#endif http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMEntry.h ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMEntry.h b/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMEntry.h deleted file mode 100644 index 48533e8..0000000 --- a/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMEntry.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -/* Header for class org_apache_accumulo_server_tabletserver_NativeMap_NMEntry */ - -#ifndef _Included_org_apache_accumulo_server_tabletserver_NativeMap_NMEntry -#define _Included_org_apache_accumulo_server_tabletserver_NativeMap_NMEntry -#ifdef __cplusplus -extern "C" { -#endif -#ifdef __cplusplus -} -#endif -#endif http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMIterator.h ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMIterator.h b/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMIterator.h deleted file mode 100644 index 2b0b26e..0000000 --- a/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMIterator.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -/* Header for class org_apache_accumulo_server_tabletserver_NativeMap_NMIterator */ - -#ifndef _Included_org_apache_accumulo_server_tabletserver_NativeMap_NMIterator -#define _Included_org_apache_accumulo_server_tabletserver_NativeMap_NMIterator -#ifdef __cplusplus -extern "C" { -#endif -#ifdef __cplusplus -} -#endif -#endif http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMSKVIter.h ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMSKVIter.h b/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMSKVIter.h deleted file mode 100644 index e2b92a8..0000000 --- a/server/src/main/c++/nativeMap/org_apache_accumulo_server_tabletserver_NativeMap_NMSKVIter.h +++ /dev/null @@ -1,29 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -/* Header for class org_apache_accumulo_server_tabletserver_NativeMap_NMSKVIter */ - -#ifndef _Included_org_apache_accumulo_server_tabletserver_NativeMap_NMSKVIter -#define _Included_org_apache_accumulo_server_tabletserver_NativeMap_NMSKVIter -#ifdef __cplusplus -extern "C" { -#endif -#ifdef __cplusplus -} -#endif -#endif http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/test.cc ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/test.cc b/server/src/main/c++/nativeMap/test.cc deleted file mode 100644 index eda4531..0000000 --- a/server/src/main/c++/nativeMap/test.cc +++ /dev/null @@ -1,152 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -#include "NativeMap.h" -#include "util.h" -#include "Key.h" -#include - -void runTest(int numRows, int numCf, int numCq, int rowLen, int cfLen, int cqLen, int cvLen, int valLen, bool testOld){ - - map tm; - tm.insert(pair("a","b")); - tm.insert(pair("a","c")).first->second="c"; - - cout << tm["a"] << std::endl; - - size_t initialMemUsage = getMemUsage(); - - NativeMap nm(1<<17, 1<<11); - - cout << " size pair : " << sizeof(std::pair) << endl; - cout << " size pair : " << sizeof(std::pair) << endl; - cout << " size pair : " << sizeof(std::pair) << endl; - -// std::less comp; -// map, block_allocator > > oldMap(comp, block_allocator >(&lba)); - map oldMap; - - int entries = 0; - - - char rowFmt[16]; - char cfFmt[16]; - char cqFmt[16]; - char cvFmt[16]; - char valFmt[16]; - - sprintf(rowFmt, "%s0%dd", "%", rowLen); - sprintf(cfFmt, "%s0%dd", "%", cfLen); - sprintf(cqFmt, "%s0%dd", "%", cqLen); - sprintf(cvFmt, "%s0%dd", "%", cvLen); - sprintf(valFmt, "%s0%dd", "%", valLen); - - for(int r = 0; r < numRows; r++){ - char row[rowLen+1]; - snprintf(row, rowLen+1, rowFmt, r); - - ColumnMap *cmt = NULL; - if(!testOld){ - cmt = nm.startUpdate(row); - } - - for(int cf = 0; cf < numCf; cf++){ - char colf[cfLen+1]; - snprintf(colf, cfLen+1, cfFmt, cf); - - for(int cq = 0; cq < numCq; cq++){ - char colq[cqLen+1]; - snprintf(colq, cqLen+1, cqFmt, cq); - - char colv[cvLen+1]; - snprintf(colv, cvLen+1, cvFmt, 1); - - char val[valLen+1]; - snprintf(val, valLen+1, valFmt, entries); - - if(!testOld){ - nm.update(cmt, colf, colq, colv, 5, false, val, valLen, cf * numCq + numCq); - }else{ - oldMap.insert(pair(Key(row, colf, colq, colv, 5, false), Field(val))); - } - - - entries++; - - } - } - } - - int expectedRowBytes = rowLen * numRows; - int expectedEntryBytes = entries * (cfLen + cqLen + cvLen + 9 + valLen); - - cout << "row bytes : " << expectedRowBytes << endl; - cout << "entry bytes : " << expectedEntryBytes << endl; - cout << "count : " << nm.count << " " << entries << endl; - size_t memUsage = getMemUsage(); - cout << " mem delta " << memUsage - initialMemUsage << endl; - cout << " mem usage " << memUsage << endl; - cout << " simple overhead " << ( (getMemUsage() - initialMemUsage) - (entries * (rowLen + cfLen + cqLen + cvLen + 9 + valLen) ) ) / ((double)entries) << endl; - - if(testOld){ - map::iterator iter = oldMap.begin(); - while(iter != oldMap.end()){ - delete(iter->first.keyData); - delete(iter->second.field); - iter++; - } - } - -/* - - int32_t ia[6]; - Iterator *iter = nm.iterator(ia); - - while(!iter->atEnd()){ - const Field &row = iter->rowIter->first; - const SubKey &sk = iter->colIter->first; - const Field &value = iter->colIter->second; - cout << row.toString() << " " << sk.toString() << " " << value.toString() << endl; - - iter->advance(ia); - } - - delete(iter); -*/ -} - -int main(int argc, char **argv){ - int numRows,numCf, numCq, rowLen, cfLen, cqLen, cvLen, valLen, testOld; - - if(argc != 10){ - cout << "Usage : " << argv[0] << " " << endl; - return -1; - } - - sscanf(argv[1], "%d", &numRows); - sscanf(argv[2], "%d", &numCf); - sscanf(argv[3], "%d", &numCq); - sscanf(argv[4], "%d", &rowLen); - sscanf(argv[5], "%d", &cfLen); - sscanf(argv[6], "%d", &cqLen); - sscanf(argv[7], "%d", &cvLen); - sscanf(argv[8], "%d", &valLen); - sscanf(argv[9], "%d", &testOld); - - for(int i = 0; i < 3; i++){ - runTest(numRows,numCf, numCq, rowLen, cfLen, cqLen, cvLen, valLen, testOld); - } -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/util.cc ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/util.cc b/server/src/main/c++/nativeMap/util.cc deleted file mode 100644 index d4907e3..0000000 --- a/server/src/main/c++/nativeMap/util.cc +++ /dev/null @@ -1,40 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -#include -#include -#include - -#include "util.h" - -size_t getMemUsage(){ - pid_t pid = getpid(); - - char cmd[1000]; - - sprintf(cmd, "cat /proc/%d/status | grep VmData | awk '{print $2}'", pid); - - - FILE *f = popen(cmd, "r"); - - int dataSize; - - fscanf(f, "%d\n", &dataSize); - - pclose(f); - - return (size_t)dataSize * (size_t)1024; -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/c++/nativeMap/util.h ---------------------------------------------------------------------- diff --git a/server/src/main/c++/nativeMap/util.h b/server/src/main/c++/nativeMap/util.h deleted file mode 100644 index 65d5721..0000000 --- a/server/src/main/c++/nativeMap/util.h +++ /dev/null @@ -1,19 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -size_t getMemUsage(); - http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/java/org/apache/accumulo/server/Accumulo.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/accumulo/server/Accumulo.java b/server/src/main/java/org/apache/accumulo/server/Accumulo.java deleted file mode 100644 index 57c4e2a..0000000 --- a/server/src/main/java/org/apache/accumulo/server/Accumulo.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.server; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Map.Entry; -import java.util.TreeMap; - -import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.trace.DistributedTrace; -import org.apache.accumulo.core.util.UtilWaitThread; -import org.apache.accumulo.core.util.Version; -import org.apache.accumulo.core.zookeeper.ZooUtil; -import org.apache.accumulo.server.client.HdfsZooInstance; -import org.apache.accumulo.server.conf.ServerConfiguration; -import org.apache.accumulo.server.fs.VolumeManager; -import org.apache.accumulo.server.util.time.SimpleTimer; -import org.apache.accumulo.server.zookeeper.ZooReaderWriter; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; -import org.apache.log4j.helpers.FileWatchdog; -import org.apache.log4j.helpers.LogLog; -import org.apache.log4j.xml.DOMConfigurator; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; - -public class Accumulo { - - private static final Logger log = Logger.getLogger(Accumulo.class); - - public static synchronized void updateAccumuloVersion(VolumeManager fs) { - try { - if (getAccumuloPersistentVersion(fs) == ServerConstants.PREV_DATA_VERSION) { - fs.create(new Path(ServerConstants.getDataVersionLocation() + "/" + ServerConstants.DATA_VERSION)); - fs.delete(new Path(ServerConstants.getDataVersionLocation() + "/" + ServerConstants.PREV_DATA_VERSION)); - } - } catch (IOException e) { - throw new RuntimeException("Unable to set accumulo version: an error occurred.", e); - } - } - - public static synchronized int getAccumuloPersistentVersion(VolumeManager fs) { - int dataVersion; - try { - FileStatus[] files = fs.getDefaultVolume().listStatus(ServerConstants.getDataVersionLocation()); - if (files == null || files.length == 0) { - dataVersion = -1; // assume it is 0.5 or earlier - } else { - dataVersion = Integer.parseInt(files[0].getPath().getName()); - } - return dataVersion; - } catch (IOException e) { - throw new RuntimeException("Unable to read accumulo version: an error occurred.", e); - } - } - - public static void enableTracing(String address, String application) { - try { - DistributedTrace.enable(HdfsZooInstance.getInstance(), ZooReaderWriter.getInstance(), application, address); - } catch (Exception ex) { - log.error("creating remote sink for trace spans", ex); - } - } - - private static class LogMonitor extends FileWatchdog implements Watcher { - String path; - - protected LogMonitor(String instance, String filename, int delay) { - super(filename); - setDelay(delay); - this.path = ZooUtil.getRoot(instance) + Constants.ZMONITOR_LOG4J_PORT; - } - - private void setMonitorPort() { - try { - String port = new String(ZooReaderWriter.getInstance().getData(path, null)); - System.setProperty("org.apache.accumulo.core.host.log.port", port); - log.info("Changing monitor log4j port to "+port); - doOnChange(); - } catch (Exception e) { - log.error("Error reading zookeeper data for monitor log4j port", e); - } - } - - @Override - public void run() { - try { - if (ZooReaderWriter.getInstance().getZooKeeper().exists(path, this) != null) - setMonitorPort(); - log.info("Set watch for monitor log4j port"); - } catch (Exception e) { - log.error("Unable to set watch for monitor log4j port " + path); - } - super.run(); - } - - @Override - protected void doOnChange() { - LogManager.resetConfiguration(); - new DOMConfigurator().doConfigure(filename, LogManager.getLoggerRepository()); - } - - @Override - public void process(WatchedEvent event) { - setMonitorPort(); - if (event.getPath() != null) { - try { - ZooReaderWriter.getInstance().exists(event.getPath(), this); - } catch (Exception ex) { - log.error("Unable to reset watch for monitor log4j port", ex); - } - } - } - } - - public static void init(VolumeManager fs, ServerConfiguration config, String application) throws UnknownHostException { - - System.setProperty("org.apache.accumulo.core.application", application); - - if (System.getenv("ACCUMULO_LOG_DIR") != null) - System.setProperty("org.apache.accumulo.core.dir.log", System.getenv("ACCUMULO_LOG_DIR")); - else - System.setProperty("org.apache.accumulo.core.dir.log", System.getenv("ACCUMULO_HOME") + "/logs/"); - - String localhost = InetAddress.getLocalHost().getHostName(); - System.setProperty("org.apache.accumulo.core.ip.localhost.hostname", localhost); - - if (System.getenv("ACCUMULO_LOG_HOST") != null) - System.setProperty("org.apache.accumulo.core.host.log", System.getenv("ACCUMULO_LOG_HOST")); - else - System.setProperty("org.apache.accumulo.core.host.log", localhost); - - int logPort = config.getConfiguration().getPort(Property.MONITOR_LOG4J_PORT); - System.setProperty("org.apache.accumulo.core.host.log.port", Integer.toString(logPort)); - - // Use a specific log config, if it exists - String logConfig = String.format("%s/%s_logger.xml", System.getenv("ACCUMULO_CONF_DIR"), application); - if (!new File(logConfig).exists()) { - // otherwise, use the generic config - logConfig = String.format("%s/generic_logger.xml", System.getenv("ACCUMULO_CONF_DIR")); - } - // Turn off messages about not being able to reach the remote logger... we protect against that. - LogLog.setQuietMode(true); - - // Configure logging - if (logPort==0) - new LogMonitor(config.getInstance().getInstanceID(), logConfig, 5000).start(); - else - DOMConfigurator.configureAndWatch(logConfig, 5000); - - // Read the auditing config - String auditConfig = String.format("%s/conf/auditLog.xml", System.getenv("ACCUMULO_HOME"), application); - - DOMConfigurator.configureAndWatch(auditConfig, 5000); - - log.info(application + " starting"); - log.info("Instance " + config.getInstance().getInstanceID()); - int dataVersion = Accumulo.getAccumuloPersistentVersion(fs); - log.info("Data Version " + dataVersion); - Accumulo.waitForZookeeperAndHdfs(fs); - - Version codeVersion = new Version(Constants.VERSION); - if (dataVersion != ServerConstants.DATA_VERSION && dataVersion != ServerConstants.PREV_DATA_VERSION) { - throw new RuntimeException("This version of accumulo (" + codeVersion + ") is not compatible with files stored using data version " + dataVersion); - } - - TreeMap sortedProps = new TreeMap(); - for (Entry entry : config.getConfiguration()) - sortedProps.put(entry.getKey(), entry.getValue()); - - for (Entry entry : sortedProps.entrySet()) { - String key = entry.getKey(); - log.info(key + " = " + (Property.isSensitive(key) ? "" : entry.getValue())); - } - - monitorSwappiness(); - } - - /** - * - */ - public static void monitorSwappiness() { - SimpleTimer.getInstance().schedule(new Runnable() { - @Override - public void run() { - try { - String procFile = "/proc/sys/vm/swappiness"; - File swappiness = new File(procFile); - if (swappiness.exists() && swappiness.canRead()) { - InputStream is = new FileInputStream(procFile); - try { - byte[] buffer = new byte[10]; - int bytes = is.read(buffer); - String setting = new String(buffer, 0, bytes); - setting = setting.trim(); - if (bytes > 0 && Integer.parseInt(setting) > 10) { - log.warn("System swappiness setting is greater than ten (" + setting + ") which can cause time-sensitive operations to be delayed. " - + " Accumulo is time sensitive because it needs to maintain distributed lock agreement."); - } - } finally { - is.close(); - } - } - } catch (Throwable t) { - log.error(t, t); - } - } - }, 1000, 10 * 60 * 1000); - } - - public static void waitForZookeeperAndHdfs(VolumeManager fs) { - log.info("Attempting to talk to zookeeper"); - while (true) { - try { - ZooReaderWriter.getInstance().getChildren(Constants.ZROOT); - break; - } catch (InterruptedException e) { - // ignored - } catch (KeeperException ex) { - log.info("Waiting for accumulo to be initialized"); - UtilWaitThread.sleep(1000); - } - } - log.info("Zookeeper connected and initialized, attemping to talk to HDFS"); - long sleep = 1000; - while (true) { - try { - if (fs.isReady()) - break; - log.warn("Waiting for the NameNode to leave safemode"); - } catch (IOException ex) { - log.warn("Unable to connect to HDFS"); - } - log.info("Sleeping " + sleep / 1000. + " seconds"); - UtilWaitThread.sleep(sleep); - sleep = Math.min(60 * 1000, sleep * 2); - } - log.info("Connected to HDFS"); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/java/org/apache/accumulo/server/ServerConstants.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/src/main/java/org/apache/accumulo/server/ServerConstants.java deleted file mode 100644 index 9e0ac39..0000000 --- a/server/src/main/java/org/apache/accumulo/server/ServerConstants.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.server; - -import java.io.IOException; - -import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.metadata.MetadataTable; -import org.apache.accumulo.core.metadata.RootTable; -import org.apache.accumulo.core.util.CachedConfiguration; -import org.apache.accumulo.server.conf.ServerConfiguration; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; - -public class ServerConstants { - - // versions should never be negative - public static final Integer WIRE_VERSION = 2; - - /** - * current version reflects the addition of a separate root table (ACCUMULO-1481) - */ - public static final int DATA_VERSION = 6; - public static final int PREV_DATA_VERSION = 5; - - private static String[] baseDirs = null; - private static String defaultBaseDir = null; - - public static synchronized String getDefaultBaseDir() { - if (defaultBaseDir == null) { - String singleNamespace = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_DFS_DIR); - String dfsUri = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_DFS_URI); - String baseDir; - - if (dfsUri == null || dfsUri.isEmpty()) { - Configuration hadoopConfig = CachedConfiguration.getInstance(); - try { - baseDir = FileSystem.get(hadoopConfig).getUri().toString() + singleNamespace; - } catch (IOException e) { - throw new RuntimeException(e); - } - } else { - if (!dfsUri.contains(":")) - throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_DFS_URI.getKey() + " got " + dfsUri); - baseDir = dfsUri + singleNamespace; - } - - defaultBaseDir = new Path(baseDir).toString(); - - } - - return defaultBaseDir; - } - - // these are functions to delay loading the Accumulo configuration unless we must - public static synchronized String[] getBaseDirs() { - if (baseDirs == null) { - String singleNamespace = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_DFS_DIR); - String ns = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_VOLUMES); - - if (ns == null || ns.isEmpty()) { - baseDirs = new String[] {getDefaultBaseDir()}; - } else { - String namespaces[] = ns.split(","); - for (String namespace : namespaces) { - if (!namespace.contains(":")) { - throw new IllegalArgumentException("Expected fully qualified URI for " + Property.INSTANCE_VOLUMES.getKey() + " got " + namespace); - } - } - baseDirs = prefix(namespaces, singleNamespace); - } - } - - return baseDirs; - } - - public static String[] prefix(String bases[], String suffix) { - if (suffix.startsWith("/")) - suffix = suffix.substring(1); - String result[] = new String[bases.length]; - for (int i = 0; i < bases.length; i++) { - result[i] = bases[i] + "/" + suffix; - } - return result; - } - - public static final String TABLE_DIR = "tables"; - public static final String RECOVERY_DIR = "recovery"; - public static final String WAL_DIR = "wal"; - - public static String[] getTablesDirs() { - return prefix(getBaseDirs(), TABLE_DIR); - } - - public static String[] getRecoveryDirs() { - return prefix(getBaseDirs(), RECOVERY_DIR); - } - - public static String[] getWalDirs() { - return prefix(getBaseDirs(), WAL_DIR); - } - - public static String[] getWalogArchives() { - return prefix(getBaseDirs(), "walogArchive"); - } - - public static Path getInstanceIdLocation() { - return new Path(getBaseDirs()[0], "instance_id"); - } - - public static Path getDataVersionLocation() { - return new Path(getBaseDirs()[0], "version"); - } - - public static String[] getRootTableDirs() { - return prefix(getTablesDirs(), RootTable.ID); - } - - public static String[] getMetadataTableDirs() { - return prefix(getTablesDirs(), MetadataTable.ID); - } -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/java/org/apache/accumulo/server/ServerOpts.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/accumulo/server/ServerOpts.java b/server/src/main/java/org/apache/accumulo/server/ServerOpts.java deleted file mode 100644 index 95fee8f..0000000 --- a/server/src/main/java/org/apache/accumulo/server/ServerOpts.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.server; - -import org.apache.accumulo.core.cli.Help; - -import com.beust.jcommander.Parameter; - -public class ServerOpts extends Help { - @Parameter(names={"-a", "--address"}, description = "address to bind to") - String address = null; - - public String getAddress() { - if (address != null) - return address; - return "0.0.0.0"; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/java/org/apache/accumulo/server/cli/ClientOnDefaultTable.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/accumulo/server/cli/ClientOnDefaultTable.java b/server/src/main/java/org/apache/accumulo/server/cli/ClientOnDefaultTable.java deleted file mode 100644 index 53f5ac2..0000000 --- a/server/src/main/java/org/apache/accumulo/server/cli/ClientOnDefaultTable.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.server.cli; - -import org.apache.accumulo.core.client.Instance; -import org.apache.accumulo.core.client.ZooKeeperInstance; -import org.apache.accumulo.core.client.mock.MockInstance; -import org.apache.accumulo.server.client.HdfsZooInstance; - -public class ClientOnDefaultTable extends org.apache.accumulo.core.cli.ClientOnDefaultTable { - { - principal = "root"; - } - - @Override - synchronized public Instance getInstance() { - if (cachedInstance != null) - return cachedInstance; - - if (mock) - return cachedInstance = new MockInstance(instance); - if (instance == null) { - return cachedInstance = HdfsZooInstance.getInstance(); - } - return cachedInstance = new ZooKeeperInstance(this.instance, this.zookeepers); - } - public ClientOnDefaultTable(String table) { - super(table); - } -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/java/org/apache/accumulo/server/cli/ClientOnRequiredTable.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/accumulo/server/cli/ClientOnRequiredTable.java b/server/src/main/java/org/apache/accumulo/server/cli/ClientOnRequiredTable.java deleted file mode 100644 index e9e9bf1..0000000 --- a/server/src/main/java/org/apache/accumulo/server/cli/ClientOnRequiredTable.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.server.cli; - -import org.apache.accumulo.core.client.Instance; -import org.apache.accumulo.core.client.ZooKeeperInstance; -import org.apache.accumulo.core.client.mock.MockInstance; -import org.apache.accumulo.server.client.HdfsZooInstance; - -public class ClientOnRequiredTable extends org.apache.accumulo.core.cli.ClientOnRequiredTable { - { - principal = "root"; - } - - @Override - synchronized public Instance getInstance() { - if (cachedInstance != null) - return cachedInstance; - - if (mock) - return cachedInstance = new MockInstance(instance); - if (instance == null) { - return cachedInstance = HdfsZooInstance.getInstance(); - } - return cachedInstance = new ZooKeeperInstance(this.instance, this.zookeepers); - } -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/598821cd/server/src/main/java/org/apache/accumulo/server/cli/ClientOpts.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/accumulo/server/cli/ClientOpts.java b/server/src/main/java/org/apache/accumulo/server/cli/ClientOpts.java deleted file mode 100644 index 6f3516a..0000000 --- a/server/src/main/java/org/apache/accumulo/server/cli/ClientOpts.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.server.cli; - -import org.apache.accumulo.core.client.Instance; -import org.apache.accumulo.core.client.ZooKeeperInstance; -import org.apache.accumulo.core.client.mock.MockInstance; -import org.apache.accumulo.server.client.HdfsZooInstance; - -public class ClientOpts extends org.apache.accumulo.core.cli.ClientOpts { - - { - principal = "root"; - } - - @Override - public Instance getInstance() { - if (mock) - return new MockInstance(instance); - if (instance == null) { - return HdfsZooInstance.getInstance(); - } - return new ZooKeeperInstance(this.instance, this.zookeepers); - } -}