incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [31/50] [abbrv] git commit: Used TSONProtocol instead of thrift binary protocol and cleaned up some unused code
Date Sun, 03 Nov 2013 15:20:25 GMT
Used TSONProtocol instead of thrift binary protocol and cleaned up some unused code


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/46ce5dc3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/46ce5dc3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/46ce5dc3

Branch: refs/heads/0.3.0-lucene-upgrade
Commit: 46ce5dc362c518e2c736328bc4601b2aaf29b2b9
Parents: ac70144
Author: Rahul Challapalli <challapallirahul@gmail.com>
Authored: Tue Oct 29 23:23:14 2013 -0700
Committer: Rahul Challapalli <challapallirahul@gmail.com>
Committed: Tue Oct 29 23:32:16 2013 -0700

----------------------------------------------------------------------
 blur-289.patch                                  | 151 +++++++++++++++++++
 .../clusterstatus/ZookeeperClusterStatus.java   |  77 +---------
 .../clusterstatus/ZookeeperPathConstants.java   |   4 -
 3 files changed, 157 insertions(+), 75 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46ce5dc3/blur-289.patch
----------------------------------------------------------------------
diff --git a/blur-289.patch b/blur-289.patch
new file mode 100644
index 0000000..7cb04cb
--- /dev/null
+++ b/blur-289.patch
@@ -0,0 +1,151 @@
+From 12e7afe3e630783fbafacd09967d25d89643e1bd Mon Sep 17 00:00:00 2001
+From: Rahul Challapalli <challapallirahul@gmail.com>
+Date: Sun, 27 Oct 2013 23:41:43 -0700
+Subject: [PATCH] Modified the way in which TableDescriptor object is
+ serialized and stored in zookeeper
+
+---
+ .../clusterstatus/ZookeeperClusterStatus.java      | 90 +++++++++++++++-------
+ 1 file changed, 64 insertions(+), 26 deletions(-)
+
+diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
+index 9a0f420..0ce0d55 100644
+--- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
++++ b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
+@@ -37,6 +37,10 @@ import org.apache.blur.BlurConfiguration;
+ import org.apache.blur.log.Log;
+ import org.apache.blur.log.LogFactory;
+ import org.apache.blur.lucene.search.FairSimilarity;
++import org.apache.blur.thirdparty.thrift_0_9_0.TDeserializer;
++import org.apache.blur.thirdparty.thrift_0_9_0.TException;
++import org.apache.blur.thirdparty.thrift_0_9_0.TSerializer;
++import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TBinaryProtocol;
+ import org.apache.blur.thrift.generated.TableDescriptor;
+ import org.apache.blur.utils.BlurConstants;
+ import org.apache.blur.utils.BlurUtil;
+@@ -409,6 +413,52 @@ public class ZookeeperClusterStatus extends ClusterStatus {
+     TableDescriptor tableDescriptor = new TableDescriptor();
+     try {
+       checkIfOpen();
++      
++      TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory());
++      String blurTablePath = ZookeeperPathConstants.getTablePath(cluster, table);
++      byte[] bytes = getData(blurTablePath);
++      
++      if (bytes == null || bytes.length == 0) {
++        /*
++         * table descriptor is stored in an older format where we manually
++         * serialized each field into a different zookeeper node
++         * so we fetch it using old code and serialize it again with thrift protocol
++         */
++        getOldTableDescriptor(useCache, cluster, table, tableDescriptor);
++        
++        // store it using thrift protocol
++        byte[] newFormatBytes = serializeTableDescriptor(tableDescriptor);
++        BlurUtil.createPath(_zk, blurTablePath, newFormatBytes);
++        
++        // remove old child nodes
++        List<String> list = _zk.getChildren(blurTablePath, false);
++        for (String p : list) {
++          BlurUtil.removeAll(_zk, blurTablePath + "/" + p);
++        }
++      } else {
++        deserializer.deserialize(tableDescriptor, bytes);
++      }
++      
++      
++    } catch (TException e) { 
++      throw new RuntimeException(e);
++    } catch (KeeperException e) {
++      throw new RuntimeException(e);
++    } catch (InterruptedException e) {
++      throw new RuntimeException(e);
++    } finally {
++      long e = System.nanoTime();
++      LOG.debug("trace getTableDescriptor took [" + (e - s) / 1000000.0 + " ms]");
++    }
++    tableDescriptor.cluster = cluster;
++    _tableDescriptorCache.put(table, tableDescriptor);
++    return tableDescriptor;
++  }
++
++  private TableDescriptor getOldTableDescriptor(boolean useCache, String cluster, String
table, TableDescriptor tableDescriptor) {
++    long s = System.nanoTime();
++    try {
++      
+       NullPointerException npe = null;
+       LOOP: for (int i = 0; i < 10; i++) {
+         npe = null;
+@@ -444,13 +494,12 @@ public class ZookeeperClusterStatus extends ClusterStatus {
+       throw new RuntimeException(e);
+     } finally {
+       long e = System.nanoTime();
+-      LOG.debug("trace getTableDescriptor took [" + (e - s) / 1000000.0 + " ms]");
++      LOG.debug("trace getOldTableDescriptor took [" + (e - s) / 1000000.0 + " ms]");
+     }
+-    tableDescriptor.cluster = cluster;
+-    _tableDescriptorCache.put(table, tableDescriptor);
++    
+     return tableDescriptor;
+   }
+-
++  
+   private boolean internalGetReadOnly(String tableReadOnlyPath) throws KeeperException,
InterruptedException {
+     Stat stat = _zk.exists(tableReadOnlyPath, false);
+     if (stat == null) {
+@@ -718,34 +767,14 @@ public class ZookeeperClusterStatus extends ClusterStatus {
+       String uri = BlurUtil.nullCheck(tableDescriptor.tableUri, "tableDescriptor.tableUri
cannot be null.");
+       int shardCount = BlurUtil.zeroCheck(tableDescriptor.shardCount,
+           "tableDescriptor.shardCount cannot be less than 1");
+-      Similarity similarity = BlurUtil.getInstance(tableDescriptor.similarityClass, Similarity.class);
+-      boolean blockCaching = tableDescriptor.blockCaching;
+-      Set<String> blockCachingFileTypes = tableDescriptor.blockCachingFileTypes;
+       String blurTablePath = ZookeeperPathConstants.getTablePath(cluster, table);
+ 
+       if (_zk.exists(blurTablePath, false) != null) {
+         throw new IOException("Table [" + table + "] already exists.");
+       }
+       BlurUtil.setupFileSystem(uri, shardCount);
+-      BlurUtil.createPath(_zk, blurTablePath, null);
+-      BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableColumnsToPreCache(cluster,
table),
+-          toBytes(tableDescriptor.preCacheCols));
+-      BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableUriPath(cluster, table), uri.getBytes());
+-      BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableShardCountPath(cluster, table),
+-          Integer.toString(shardCount).getBytes());
+-      BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableSimilarityPath(cluster, table),
similarity.getClass()
+-          .getName().getBytes());
+-      BlurUtil.createPath(_zk, ZookeeperPathConstants.getLockPath(cluster, table), null);
+-      BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableFieldNamesPath(cluster, table),
null);
+-      if (tableDescriptor.readOnly) {
+-        BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableReadOnlyPath(cluster, table),
null);
+-      }
+-      if (blockCaching) {
+-        BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableBlockCachingPath(cluster,
table), null);
+-      }
+-      BlurUtil.createPath(_zk, ZookeeperPathConstants.getTableBlockCachingFileTypesPath(cluster,
table),
+-          toBytes(blockCachingFileTypes));
+-      BlurUtil.createPath(_zk, ZookeeperPathConstants.getTablePropertiesPath(cluster, table),
toBytes(tableDescriptor.getTableProperties()));
++      byte[] bytes = serializeTableDescriptor(tableDescriptor); 
++      BlurUtil.createPath(_zk, blurTablePath, bytes);
+     } catch (IOException e) {
+       throw new RuntimeException(e);
+     } catch (KeeperException e) {
+@@ -758,6 +787,15 @@ public class ZookeeperClusterStatus extends ClusterStatus {
+     }
+   }
+ 
++  private byte[] serializeTableDescriptor(TableDescriptor td) {
++    try{
++      TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory());
++      return serializer.serialize(td);
++    } catch (TException e) {
++      throw new RuntimeException(e);
++    }
++  }
++  
+   private void assignTableUri(TableDescriptor tableDescriptor) {
+     if (tableDescriptor.getTableUri() != null) {
+       return;
+-- 
+1.7.12.4 (Apple Git-37)
+

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46ce5dc3/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
index 0ce0d55..3cf948a 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java
@@ -21,7 +21,6 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -40,7 +39,7 @@ import org.apache.blur.lucene.search.FairSimilarity;
 import org.apache.blur.thirdparty.thrift_0_9_0.TDeserializer;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thirdparty.thrift_0_9_0.TSerializer;
-import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TBinaryProtocol;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TJSONProtocol;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.BlurUtil;
@@ -50,7 +49,6 @@ import org.apache.blur.zookeeper.WatchNodeData;
 import org.apache.blur.zookeeper.WatchNodeExistance;
 import org.apache.blur.zookeeper.ZkUtils;
 import org.apache.blur.zookeeper.ZooKeeperLockManager;
-import org.apache.lucene.search.similarities.Similarity;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.WatchedEvent;
@@ -414,7 +412,6 @@ public class ZookeeperClusterStatus extends ClusterStatus {
     try {
       checkIfOpen();
       
-      TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory());
       String blurTablePath = ZookeeperPathConstants.getTablePath(cluster, table);
       byte[] bytes = getData(blurTablePath);
       
@@ -424,22 +421,19 @@ public class ZookeeperClusterStatus extends ClusterStatus {
          * serialized each field into a different zookeeper node
          * so we fetch it using old code and serialize it again with thrift protocol
          */
+        LOG.info("The schema of Table [{0}] was stored in an older format. Now converting
it to the new format", table);
         getOldTableDescriptor(useCache, cluster, table, tableDescriptor);
         
+        BlurUtil.removeAll(_zk, blurTablePath);
+        
         // store it using thrift protocol
         byte[] newFormatBytes = serializeTableDescriptor(tableDescriptor);
         BlurUtil.createPath(_zk, blurTablePath, newFormatBytes);
         
-        // remove old child nodes
-        List<String> list = _zk.getChildren(blurTablePath, false);
-        for (String p : list) {
-          BlurUtil.removeAll(_zk, blurTablePath + "/" + p);
-        }
       } else {
+        TDeserializer deserializer = new TDeserializer(new TJSONProtocol.Factory());
         deserializer.deserialize(tableDescriptor, bytes);
       }
-      
-      
     } catch (TException e) { 
       throw new RuntimeException(e);
     } catch (KeeperException e) {
@@ -472,7 +466,6 @@ public class ZookeeperClusterStatus extends ClusterStatus {
           tableDescriptor.readOnly = internalGetReadOnly(ZookeeperPathConstants.getTableReadOnlyPath(cluster,
table));
           tableDescriptor.preCacheCols = toList(getData(ZookeeperPathConstants
               .getTableColumnsToPreCache(cluster, table)));
-          tableDescriptor.tableProperties = toMap(getData(ZookeeperPathConstants.getTablePropertiesPath(cluster,
table)));
           byte[] data = getData(ZookeeperPathConstants.getTableSimilarityPath(cluster, table));
           if (data != null) {
             tableDescriptor.similarityClass = new String(data);
@@ -521,20 +514,6 @@ public class ZookeeperClusterStatus extends ClusterStatus {
     return list;
   }
 
-  private static byte[] toBytes(List<String> list) {
-    if (list == null) {
-      return null;
-    }
-    StringBuilder builder = new StringBuilder();
-    for (String s : list) {
-      if (builder.length() != 0) {
-        builder.append(',');
-      }
-      builder.append(s);
-    }
-    return builder.toString().getBytes();
-  }
-
   private void updateEnabled(boolean useCache, TableDescriptor tableDescriptor, String cluster,
String table) {
     if (tableDescriptor != null) {
       tableDescriptor.setEnabled(isEnabled(useCache, cluster, table));
@@ -768,7 +747,6 @@ public class ZookeeperClusterStatus extends ClusterStatus {
       int shardCount = BlurUtil.zeroCheck(tableDescriptor.shardCount,
           "tableDescriptor.shardCount cannot be less than 1");
       String blurTablePath = ZookeeperPathConstants.getTablePath(cluster, table);
-
       if (_zk.exists(blurTablePath, false) != null) {
         throw new IOException("Table [" + table + "] already exists.");
       }
@@ -789,7 +767,7 @@ public class ZookeeperClusterStatus extends ClusterStatus {
 
   private byte[] serializeTableDescriptor(TableDescriptor td) {
     try{
-      TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory());
+      TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
       return serializer.serialize(td);
     } catch (TException e) {
       throw new RuntimeException(e);
@@ -891,49 +869,6 @@ public class ZookeeperClusterStatus extends ClusterStatus {
     }
   }
 
-  private static byte[] toBytes(Set<String> blockCachingFileTypes) {
-    if (blockCachingFileTypes == null || blockCachingFileTypes.isEmpty()) {
-      return null;
-    }
-    StringBuilder builder = new StringBuilder();
-    for (String type : blockCachingFileTypes) {
-      builder.append(type).append(',');
-    }
-    return builder.substring(0, builder.length() - 1).getBytes();
-  }
-  
-  private static byte[] toBytes(Map<String,String> properties) {
-    if (properties == null || properties.isEmpty()) {
-      return null;
-    }
-    StringBuilder builder = new StringBuilder();
-    Set<Entry<String, String>> entrySet = properties.entrySet();
-    for (Entry<String, String> entry : entrySet) {
-      String key = entry.getKey();
-      String value = entry.getValue();
-      String keyValue = key+"->"+value;
-      builder.append(keyValue).append(',');
-    }
-    
-    return builder.substring(0, builder.length() - 1).getBytes();
-  }
-  
-  private static Map<String,String> toMap(byte[] mapBytes) {
-    Map<String,String> properties = new HashMap<String, String>();
-    
-    if (mapBytes == null) {
-      return null;
-    }
-    String str = new String(mapBytes);
-    String[] keyValuePairs = str.split(",");
-    for (String keyValue : keyValuePairs) {
-      String[] split = keyValue.split("->");
-      properties.put(split[0].trim(), split[1].trim());
-    }
-    
-    return properties;
-  }
-
   @Override
   public boolean isOpen() {
     return _running.get();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46ce5dc3/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java
b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java
index 11ab6e5..5278aca 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperPathConstants.java
@@ -107,8 +107,4 @@ public class ZookeeperPathConstants {
     return getClusterPath(cluster) + "/layout";
   }
 
-  public static String getTablePropertiesPath(String cluster, String table) {
-    return getTablePath(cluster, table) + "/tableproperties";
-  }
-
 }


Mime
View raw message