trafodion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lium...@apache.org
Subject [05/15] trafodion git commit: use coprocessor to set storage policy
Date Tue, 26 Jun 2018 01:22:37 GMT
http://git-wip-us.apache.org/repos/asf/trafodion/blob/c5bab335/core/sqf/src/seatrans/hbase-trx/src/main/protobuf/TrxRegion.proto
----------------------------------------------------------------------
diff --git a/core/sqf/src/seatrans/hbase-trx/src/main/protobuf/TrxRegion.proto b/core/sqf/src/seatrans/hbase-trx/src/main/protobuf/TrxRegion.proto
index b3edbfa..357b15b 100755
--- a/core/sqf/src/seatrans/hbase-trx/src/main/protobuf/TrxRegion.proto
+++ b/core/sqf/src/seatrans/hbase-trx/src/main/protobuf/TrxRegion.proto
@@ -421,6 +421,16 @@ message TrafEstimateRowCountResponse {
   optional bool hasException = 7;
 }
 
+message TrafSetStoragePolicyRequest {
+  required  string path = 1;
+  required  string policy = 2;
+}
+
+message TrafSetStoragePolicyResponse {
+  required bool status = 1;
+  required string exception = 2;
+}
+
 message TransactionalAggregateRequest {
   /** The request passed to the TransactionalAggregateService consists of three parts
    *  (1) the (canonical) classname of the ColumnInterpreter implementation
@@ -543,4 +553,7 @@ service TrxRegionService {
     returns (TransactionalAggregateResponse);
   rpc GetMedian (TransactionalAggregateRequest) 
     returns (TransactionalAggregateResponse);
+  rpc setStoragePolicy (TrafSetStoragePolicyRequest)
+    returns (TrafSetStoragePolicyResponse);
+
 }

http://git-wip-us.apache.org/repos/asf/trafodion/blob/c5bab335/core/sql/regress/seabase/DIFF002.KNOWN
----------------------------------------------------------------------
diff --git a/core/sql/regress/seabase/DIFF002.KNOWN b/core/sql/regress/seabase/DIFF002.KNOWN
deleted file mode 100644
index cea6aa4..0000000
--- a/core/sql/regress/seabase/DIFF002.KNOWN
+++ /dev/null
@@ -1,48 +0,0 @@
-1320c1320,1323
-< --- SQL operation complete.
----
-> *** ERROR[8448] Unable to access Hbase interface. Call to ExpHbaseInterface::create()
returned error HBASE_CREATE_ERROR(701). Cause: java.io.IOException: invoke set storage policy
error : null
-> org.TRAFODION.sql.HBaseClient.createk(HBaseClient.java:620).
-> 
-> --- SQL operation failed with errors.
-1324,1357c1327,1329
-< CREATE TABLE TRAFODION.T002SCH.CREATEOPTIONS
-<  (
-<  A INT DEFAULT NULL
-<  )
-<  ATTRIBUTES ALIGNED FORMAT
-<  HBASE_OPTIONS
-<  (
-<  MAX_VERSIONS = '3',
-<  MIN_VERSIONS = '2',
-<  TIME_TO_LIVE = '100',
-<  BLOCKCACHE = 'false',
-<  IN_MEMORY = 'true',
-<  COMPRESSION = 'GZ',
-<  BLOOMFILTER = 'ROWCOL',
-<  BLOCKSIZE = '10000',
-<  DATA_BLOCK_ENCODING = 'DIFF',
-<  CACHE_BLOOMS_ON_WRITE = 'false',
-<  CACHE_DATA_ON_WRITE = 'false',
-<  CACHE_INDEXES_ON_WRITE = 'false',
-<  COMPACT_COMPRESSION = 'GZ',
-<  PREFIX_LENGTH_KEY = '10',
-<  EVICT_BLOCKS_ON_CLOSE = 'true',
-<  KEEP_DELETED_CELLS = 'false',
-<  REPLICATION_SCOPE = '0',
-<  MAX_FILESIZE = '4000000',
-<  COMPACT = 'true',
-<  DURABILITY = 'async_wal',
-<  MEMSTORE_FLUSH_SIZE = '2000000',
-<  SPLIT_POLICY = 'org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy',
-<  CACHE_DATA_IN_L1 = 'false',
-<  HDFS_STORAGE_POLICY = 'hot',
-<  PREFETCH_BLOCKS_ON_OPEN = 'false'
-<  )
-< ;
----
-> *** ERROR[4082] Object TRAFODION.T002SCH.CREATEOPTIONS does not exist or is inaccessible.
-> 
-> *** ERROR[8822] The statement was not prepared.
-1359d1330
-< --- SQL operation complete.

http://git-wip-us.apache.org/repos/asf/trafodion/blob/c5bab335/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
----------------------------------------------------------------------
diff --git a/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java b/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
index 2cdfa49..5c161a7 100644
--- a/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
+++ b/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
@@ -583,38 +583,12 @@ public class HBaseClient {
             {
               if(setDescRet.storagePolicyChanged())
               {
-                 //change the HDFS storage policy
-                 //get the HBase table path
-                 String hbaseRoot = config.get("hbase.rootdir");
-                 FileSystem fs = FileSystem.get(config);
-                 //Construct the HDFS dir
-                 //find out if namespace is there
-                 String[] parts = tblName.split(":");
-                 String namespacestr="";
-
-                 //guess the path pattern
-                 //different HBase version may have different path pattern
-                 //There is no interface to get this information using HBase User API
-                 //Since it is HBase internal behavior
-                 //At present, before HBase 2.0 release and before HBASE-19858 released in
HBase 1.5.0
-                 //Trafodion here need a trick to guess
-                 String fullPath = hbaseRoot + "/data/" ;
-                 String fullPath2 = hbaseRoot + "/data/default/";
-
-                 //check if fullPath2 exist
-                 if(fs.exists(new Path(fullPath2)))
-                    fullPath = fullPath2;
-
-                 if(parts.length >1) //have namespace
-                   fullPath = fullPath + parts[0] + "/" + parts[1];
-                 else
-                   fullPath = fullPath + tblName;
-
-                 if (logger.isDebugEnabled()) logger.debug("createk table fullPath is " +
fullPath);
-
-                 admin.close(); //close here, invokeSetStoragePolicy may throw exception
-
-                 invokeSetStoragePolicy(fs, fullPath, setDescRet.storagePolicy_ ) ;
+                 Object tableOptionsStoragePolicy[] = new Object[HBASE_HDFS_STORAGE_POLICY+1];
+                 for(int i=0; i<HBASE_HDFS_STORAGE_POLICY; i++)
+                   tableOptionsStoragePolicy[i]="";
+                 tableOptionsStoragePolicy[HBASE_HDFS_STORAGE_POLICY]=(String)setDescRet.storagePolicy_
;
+                 tableOptionsStoragePolicy[HBASE_NAME]=(String)tblName;
+                 alter(tblName,tableOptionsStoragePolicy,transID);
               }
             }
             else
@@ -622,35 +596,6 @@ public class HBaseClient {
         return true;
     }
 
-    private static void invokeSetStoragePolicy(final FileSystem fs, final String pathstr,
-      final String storagePolicy) 
-       throws IOException {
-        Path path = new Path(pathstr);
-        Method m = null;
-        try {
-            m = fs.getClass().getDeclaredMethod("setStoragePolicy",
-            new Class<?>[] { Path.class, String.class });
-            m.setAccessible(true);
-        } catch (NoSuchMethodException e) {
-            m = null;
-            throw new IOException("FileSystem doesn't support setStoragePolicy");
-        } catch (SecurityException e) {
-          m = null; 
-          throw new IOException("No access to setStoragePolicy on FileSystem from the SecurityManager");
-        }
-        if (m != null) {
-          try {
-            m.invoke(fs, path, storagePolicy);
-            if (logger.isDebugEnabled()) {
-              logger.debug("Set storagePolicy=" + storagePolicy + " for path=" + path);
-            }
-          } catch (Exception e) {
-               logger.error("invoke set storage policy error : " + e);
-               throw new IOException(e);
-          }
-        }
-    }
-
     public boolean registerTruncateOnAbort(String tblName, long transID)
         throws MasterNotRunningException, IOException {
 
@@ -688,7 +633,6 @@ public class HBaseClient {
         Admin admin = getConnection().getAdmin();
         HTableDescriptor htblDesc = admin.getTableDescriptor(TableName.valueOf(tblName));
      
         HColumnDescriptor[] families = htblDesc.getColumnFamilies();
-
         String colFam = (String)tableOptions[HBASE_NAME];
         if (colFam == null)
             return true; // must have col fam name
@@ -720,13 +664,18 @@ public class HBaseClient {
                 return true; // col fam already exists
         }
         else {
-            if (colDesc == null)
+            if (colDesc == null )
+            {
+               if( (String)tableOptions[HBASE_HDFS_STORAGE_POLICY] == null || (String)tableOptions[HBASE_HDFS_STORAGE_POLICY]==""
)
                 return true; // colDesc must exist
+            }
+            else {
 
-            int defaultVersionsValue = colDesc.getMaxVersions(); 
+              int defaultVersionsValue = colDesc.getMaxVersions(); 
 
-            status = 
+              status = 
                 setDescriptors(tableOptions,htblDesc /*out*/,colDesc /*out*/, defaultVersionsValue);
+           }
         }
 
             if (transID != 0) {


Mime
View raw message