lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yo...@apache.org
Subject svn commit: r1211075 - in /lucene/dev/branches/solrcloud/solr: core/src/java/org/apache/solr/cloud/ solrj/src/java/org/apache/solr/common/cloud/
Date Tue, 06 Dec 2011 19:57:50 GMT
Author: yonik
Date: Tue Dec  6 19:57:49 2011
New Revision: 1211075

URL: http://svn.apache.org/viewvc?rev=1211075&view=rev
Log:
simplify json serialization/deserialization, remove extra String creations, move CoreState+CoreAssignments
to common

Added:
    lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CoreAssignment.java
  (contents, props changed)
      - copied, changed from r1210956, lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/CoreAssignment.java
    lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CoreState.java
  (contents, props changed)
      - copied, changed from r1210956, lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/CoreState.java
Removed:
    lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/CoreAssignment.java
    lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/CoreState.java
Modified:
    lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/NodeStateWatcher.java
    lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/Overseer.java
    lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/ZkController.java
    lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CloudState.java
    lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java

Modified: lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/NodeStateWatcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/NodeStateWatcher.java?rev=1211075&r1=1211074&r2=1211075&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/NodeStateWatcher.java
(original)
+++ lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/NodeStateWatcher.java
Tue Dec  6 19:57:49 2011
@@ -25,6 +25,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.solr.common.cloud.CoreState;
 import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.WatchedEvent;

Modified: lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1211075&r1=1211074&r2=1211075&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/Overseer.java Tue
Dec  6 19:57:49 2011
@@ -30,13 +30,7 @@ import java.util.Set;
 
 import org.apache.solr.cloud.NodeStateWatcher.NodeStateChangeListener;
 import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.CloudState;
-import org.apache.solr.common.cloud.CloudStateUtility;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.cloud.ZooKeeperException;
+import org.apache.solr.common.cloud.*;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.WatchedEvent;

Modified: lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1211075&r1=1211074&r2=1211075&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/ZkController.java
(original)
+++ lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/ZkController.java
Tue Dec  6 19:57:49 2011
@@ -31,13 +31,7 @@ import java.util.regex.Pattern;
 
 import org.apache.solr.cloud.RecoveryStrat.OnFinish;
 import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.CloudState;
-import org.apache.solr.common.cloud.OnReconnect;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.cloud.ZooKeeperException;
+import org.apache.solr.common.cloud.*;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.CoreDescriptor;

Modified: lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CloudState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CloudState.java?rev=1211075&r1=1211074&r2=1211075&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CloudState.java
(original)
+++ lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CloudState.java
Tue Dec  6 19:57:49 2011
@@ -18,16 +18,15 @@ package org.apache.solr.common.cloud;
  */
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.noggit.CharArr;
-import org.apache.noggit.JSONUtil;
-import org.apache.noggit.ObjectBuilder;
+import java.io.InputStreamReader;
+import java.util.*;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.UnicodeUtil;
+import org.apache.noggit.*;
+import org.apache.solr.common.cloud.CoreAssignment;
+import org.apache.solr.common.cloud.CoreState;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -35,7 +34,7 @@ import org.slf4j.LoggerFactory;
 // quasi immutable :(
 public class CloudState {
 	protected static Logger log = LoggerFactory.getLogger(CloudState.class);
-	private final Map<String, Map<String,Slice>> collectionStates;
+	private final Map<String, Map<String,Slice>> collectionStates;  // Map<collectionName,
Map<sliceName,Slice>>
 	private final Set<String> liveNodes;
 
 	public CloudState() {
@@ -122,7 +121,7 @@ public class CloudState {
       return new CloudState(liveNodes, Collections.<String, Map<String,Slice>>emptyMap());
     }
     
-    LinkedHashMap<String, Object> stateMap = (LinkedHashMap<String, Object>)
ObjectBuilder.fromJSON(new String(bytes, "utf-8"));
+    LinkedHashMap<String, Object> stateMap = (LinkedHashMap<String, Object>)
fromJSON(bytes);
     HashMap<String,Map<String, Slice>> state = new HashMap<String,Map<String,Slice>>();
 
     for(String collectionName: stateMap.keySet()){
@@ -142,8 +141,64 @@ public class CloudState {
     return new CloudState(liveNodes, state);
 	}
 
+  private static class CloudJSONWriter extends JSONWriter {
+    public CloudJSONWriter(CharArr out, int indent) {
+      super(out, indent);
+    }
+    @Override
+    public void handleUnknownClass(Object o) {
+      if (o instanceof Slice) {
+        setIndentSize(2); // this is big enough, we currently always want indenting
+        write( ((Slice)o).getShards() );
+      } else if (o instanceof ZkNodeProps) {
+        write( ((ZkNodeProps) o).getProperties() );
+      } else if (o instanceof CloudState) {
+        setIndentSize(2); // this is big enough, we currently always want indenting
+        write( ((CloudState)o).getCollectionStates() );
+      } else if (o instanceof CoreAssignment) {
+        write(((CoreAssignment) o).getProperties());
+      } else if (o instanceof CoreState) {
+        write(((CoreState) o).getProperties());
+      } else {
+        super.handleUnknownClass(o);
+      }
+    }
+  }
+
   public static byte[] store(CloudState state)
       throws IOException {
+    return toJSON(state);
+  }
+
+  // convenience methods... should these go somewhere else?
+
+  public static byte[] toJSON(Object o) {
+    CharArr out = new CharArr();
+    new CloudJSONWriter(out, 2).write(o); // indentation by default
+    return toUTF8(out);
+  }
+
+  public static byte[] toUTF8(CharArr out) {
+    BytesRef br = new BytesRef(out);
+    return Arrays.copyOf(br.bytes, br.length);
+  }
+
+  public static Object fromJSON(byte[] utf8) {
+    // convert directly from bytes to chars
+    // and parse directly from that instead of going through
+    // intermediate strings or readers
+    CharsRef chars = new CharsRef();
+    UnicodeUtil.UTF8toUTF16(utf8, 0, utf8.length, chars);   // TODO: this method currently
oversizes the array
+    JSONParser parser = new JSONParser(chars.chars, chars.offset, chars.length);
+    try {
+      return ObjectBuilder.getVal(parser);
+    } catch (IOException e) {
+      throw new RuntimeException(e); // should never happen w/o using real IO
+    }
+  }
+
+  public static byte[] store0(CloudState state)
+      throws IOException {
     CharArr out = new CharArr();
     out.append(JSONUtil.OBJECT_START);
     int collCount = state.getCollections().size();

Copied: lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CoreAssignment.java
(from r1210956, lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/CoreAssignment.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CoreAssignment.java?p2=lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CoreAssignment.java&p1=lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/CoreAssignment.java&r1=1210956&r2=1211075&rev=1211075&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/CoreAssignment.java
(original)
+++ lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CoreAssignment.java
Tue Dec  6 19:57:49 2011
@@ -1,4 +1,4 @@
-package org.apache.solr.cloud;
+package org.apache.solr.common.cloud;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -18,31 +18,15 @@ package org.apache.solr.cloud;
  */
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.noggit.CharArr;
-import org.apache.noggit.JSONUtil;
-import org.apache.noggit.ObjectBuilder;
+import java.util.*;
 
 public class CoreAssignment {
 
   private static String COLLECTION="_collection";
   private static String CORE="_core";
 
-  public String getCoreName() {
-    return properties.get(CORE);
-  }
-
-  public String getCollectionName() {
-    return properties.get(COLLECTION);
-  }
-
   private final Map<String, String> properties;
-  
+
   public Map<String,String> getProperties() {
     return properties;
   }
@@ -58,34 +42,29 @@ public class CoreAssignment {
     props.put(CORE, coreName);
     this.properties = Collections.unmodifiableMap(props);
   }
-  
-  public static byte[] tobytes(CoreAssignment... assignments) throws IOException {
-    CharArr out = new CharArr();
-    out.append(JSONUtil.ARRAY_START);
-    boolean first = true;
-    for (CoreAssignment assignment : assignments) {
-      if (first) {
-        first = false;
-      } else {
-        out.append(JSONUtil.VALUE_SEPARATOR);
-      }
-      out.append(JSONUtil.toJSON(assignment.properties));
-      
-    }
-    
-    out.append(JSONUtil.ARRAY_END);
 
-    return out.toString().getBytes("utf-8");
+  public String getCoreName() {
+    return properties.get(CORE);
+  }
+
+  public String getCollectionName() {
+    return properties.get(COLLECTION);
+  }
+
+  public static byte[] tobytes(CoreAssignment... assignments) throws IOException {
+    return CloudState.toJSON(assignments);
   }
   
   public static CoreAssignment[] fromBytes(byte[] bytes) throws IOException {
-    ArrayList<CoreAssignment> states = new ArrayList<CoreAssignment>(); 
-    List<Map<String, String>> stateMaps = (List<Map<String, String>>)ObjectBuilder.fromJSON(new
String(bytes,"utf-8"));
+    List<Map<String, String>> stateMaps = (List<Map<String, String>>)
CloudState.fromJSON(bytes);
+
+    CoreAssignment[] states = new CoreAssignment[stateMaps.size()];
+    int i = 0;
     for (Map<String,String> state : stateMaps) {
-      states.add(new CoreAssignment(state));
+      states[i++] = new CoreAssignment(state);
     }
     
-    return states.toArray(new CoreAssignment[states.size()]);
+    return states;
   }
   
   @Override

Copied: lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CoreState.java
(from r1210956, lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/CoreState.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CoreState.java?p2=lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CoreState.java&p1=lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/CoreState.java&r1=1210956&r2=1211075&rev=1211075&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/CoreState.java
(original)
+++ lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/CoreState.java
Tue Dec  6 19:57:49 2011
@@ -1,4 +1,4 @@
-package org.apache.solr.cloud;
+package org.apache.solr.common.cloud;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -18,40 +18,18 @@ package org.apache.solr.cloud;
  */
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.noggit.CharArr;
-import org.apache.noggit.JSONUtil;
-import org.apache.noggit.ObjectBuilder;
+import java.util.*;
 
 public class CoreState {
-
-  
   private static String COLLECTION="_collection";
   private static String CORE="_core";
 
-  public String getCoreName() {
-    return properties.get(CORE);
-  }
-
-  public String getCollectionName() {
-    return properties.get(COLLECTION);
-  }
-
   private final Map<String, String> properties;
-  
-  public Map<String,String> getProperties() {
-    return properties;
-  }
 
   private CoreState(Map<String, String> props) {
     this.properties = Collections.unmodifiableMap(props);
   }
-  
+
   public CoreState(String coreName, String collectionName, Map<String,String> properties)
{
     HashMap<String,String> props = new HashMap<String,String>();
     props.putAll(properties);
@@ -59,34 +37,33 @@ public class CoreState {
     props.put(CORE, coreName);
     this.properties = Collections.unmodifiableMap(props);
   }
-  
-  public static byte[] tobytes(CoreState... states) throws IOException {
-    CharArr out = new CharArr();
-    out.append(JSONUtil.ARRAY_START);
-    boolean first = true;
-    for (CoreState state : states) {
-      if (first) {
-        first = false;
-      } else {
-        out.append(JSONUtil.VALUE_SEPARATOR);
-      }
-      out.append(JSONUtil.toJSON(state.properties));
-      
-    }
-    
-    out.append(JSONUtil.ARRAY_END);
 
-    return out.toString().getBytes("utf-8");
+  public String getCoreName() {
+    return properties.get(CORE);
+  }
+
+  public String getCollectionName() {
+    return properties.get(COLLECTION);
+  }
+
+  public Map<String,String> getProperties() {
+    return properties;
+  }
+
+  public static byte[] tobytes(CoreState... states) throws IOException {
+    return CloudState.toJSON(states);
   }
   
   public static CoreState[] fromBytes(byte[] bytes) throws IOException {
-    ArrayList<CoreState> states = new ArrayList<CoreState>(); 
-    List<Map<String, String>> stateMaps = (List<Map<String, String>>)ObjectBuilder.fromJSON(new
String(bytes,"utf-8"));
+    List<Map<String, String>> stateMaps = (List<Map<String, String>>)
CloudState.fromJSON(bytes);
+
+    CoreState[] states = new CoreState[stateMaps.size()];
+    int i = 0;
     for (Map<String,String> state : stateMaps) {
-      states.add(new CoreState(state));
+      states[i++] = new CoreState(state);
     }
-    
-    return states.toArray(new CoreState[states.size()]);
+
+    return states;
   }
   
   @Override

Modified: lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java?rev=1211075&r1=1211074&r2=1211075&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java
(original)
+++ lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java
Tue Dec  6 19:57:49 2011
@@ -49,14 +49,18 @@ public class ZkNodeProps  {
   public Set<String> keySet() {
     return Collections.unmodifiableSet(propMap.keySet());
   }
-  
+
+  public Map<String,String> getProperties() {
+    return Collections.unmodifiableMap(propMap);
+  }
+
   public static ZkNodeProps load(byte[] bytes) throws IOException {
-    Map<String, String> props = (Map<String, String>) ObjectBuilder.fromJSON(new
String(bytes, "utf-8"));
+    Map<String, String> props = (Map<String, String>) CloudState.fromJSON(bytes);
     return new ZkNodeProps(props);
   }
 
   public byte[] store() throws IOException {
-    return new String(JSONUtil.toJSON(this.propMap)).getBytes("utf-8");
+    return CloudState.toJSON(propMap);
   }
   
   public String get(String key) {



Mime
View raw message