incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [4/9] git commit: Changing the create table call to be idempotent as well as faster. It longer creates all the shard dirs in hdfs, the shard servers now do that as they bring the shards online for the first time.
Date Thu, 29 Jan 2015 15:15:39 GMT
Changing the create table call to be idempotent as well as faster.  It longer creates all the
shard dirs in hdfs, the shard servers now do that as they bring the shards online for the
first time.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/ce2bc4d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/ce2bc4d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/ce2bc4d5

Branch: refs/heads/master
Commit: ce2bc4d5d50b9e65f7fcc2c0446dce8a14d46b92
Parents: 2dacede
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Jan 29 09:42:48 2015 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Jan 29 09:42:48 2015 -0500

----------------------------------------------------------------------
 .../org/apache/blur/manager/IndexServer.java    |  9 -----
 .../AbstractDistributedIndexServer.java         | 42 --------------------
 .../indexserver/DistributedIndexServer.java     | 13 +++++-
 .../manager/indexserver/LocalIndexServer.java   | 21 ----------
 .../java/org/apache/blur/thrift/TableAdmin.java | 18 +++++++--
 .../blur/thrift/ThriftBlurShardServer.java      |  1 -
 .../blur/command/ShardCommandManagerTest.java   |  5 ---
 .../apache/blur/thrift/BlurClusterTestBase.java |  1 +
 .../blur/thrift/BlurClusterTestNoSecurity.java  | 13 ------
 .../java/org/apache/blur/thrift/FacetTests.java | 11 +----
 .../org/apache/blur/thrift/SuiteCluster.java    |  1 -
 11 files changed, 30 insertions(+), 105 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java
index ab5aad7..9275df4 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/IndexServer.java
@@ -57,15 +57,6 @@ public interface IndexServer extends Closeable {
   // Table Meta Data
 
   /**
-   * The shard list for a given table.
-   * 
-   * @param table
-   *          the table name.
-   * @return the list of shards.
-   */
-  List<String> getShardList(String table);
-
-  /**
    * Gets the current nodes name.
    * 
    * @return

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java
index f50b58e..d1e7500 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java
@@ -22,24 +22,15 @@ import static org.apache.blur.metrics.MetricsConstants.INDEX_MEMORY_USAGE;
 import static org.apache.blur.metrics.MetricsConstants.ORG_APACHE_BLUR;
 import static org.apache.blur.metrics.MetricsConstants.SEGMENT_COUNT;
 import static org.apache.blur.metrics.MetricsConstants.TABLE_COUNT;
-import static org.apache.blur.utils.BlurConstants.SHARD_PREFIX;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.concurrent.atomic.AtomicLong;
 
-import org.apache.blur.log.Log;
-import org.apache.blur.log.LogFactory;
 import org.apache.blur.manager.clusterstatus.ClusterStatus;
 import org.apache.blur.metrics.AtomicLongGauge;
 import org.apache.blur.server.TableContext;
-import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.trace.Trace;
-import org.apache.blur.trace.Tracer;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.ContentSummary;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
@@ -48,8 +39,6 @@ import com.yammer.metrics.core.MetricName;
 
 public abstract class AbstractDistributedIndexServer extends AbstractIndexServer {
 
-  private static final Log LOG = LogFactory.getLog(AbstractDistributedIndexServer.class);
-
   protected final ClusterStatus _clusterStatus;
   protected final Configuration _configuration;
   protected final String _nodeName;
@@ -92,37 +81,6 @@ public abstract class AbstractDistributedIndexServer extends AbstractIndexServer
     return contentSummary.getLength();
   }
 
-  @Override
-  public final List<String> getShardList(String table) {
-    checkTable(table);
-    Tracer trace = Trace.trace("filesystem - getShardList", Trace.param("table", table));
-    List<String> result = new ArrayList<String>();
-    try {
-      TableContext tableContext = getTableContext(table);
-      TableDescriptor descriptor = tableContext.getDescriptor();
-      Path tablePath = new Path(descriptor.tableUri);
-      FileSystem fileSystem = tablePath.getFileSystem(_configuration);
-      if (!fileSystem.exists(tablePath)) {
-        LOG.error("Table [{0}] is missing, defined location [{1}]", table, tablePath.toUri());
-        throw new RuntimeException("Table [" + table + "] is missing, defined location ["
+ tablePath.toUri() + "]");
-      }
-      FileStatus[] listStatus = fileSystem.listStatus(tablePath);
-      for (FileStatus status : listStatus) {
-        if (status.isDir()) {
-          String name = status.getPath().getName();
-          if (name.startsWith(SHARD_PREFIX)) {
-            result.add(name);
-          }
-        }
-      }
-      return result;
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    } finally {
-      trace.done();
-    }
-  }
-
   protected final TableContext getTableContext(final String table) {
     return TableContext.create(_clusterStatus.getTableDescriptor(true, _clusterStatus.getCluster(true,
table), table));
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
index c087ed8..79a5b3c 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
@@ -57,6 +57,7 @@ import org.apache.blur.store.hdfs_v2.JoinDirectory;
 import org.apache.blur.thrift.generated.ShardState;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.ShardUtil;
 import org.apache.blur.zookeeper.WatchChildren;
 import org.apache.blur.zookeeper.WatchChildren.OnChange;
 import org.apache.blur.zookeeper.ZookeeperPathConstants;
@@ -602,7 +603,8 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer
{
       throw new RuntimeException("Table [" + table + "] is not found.");
     }
     List<String> onlineShardServerList = _clusterStatus.getOnlineShardServers(false,
cluster);
-    List<String> shardList = getShardList(table);
+    TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(false, cluster, table);
+    List<String> shardList = generateShardList(tableDescriptor);
 
     String shutdownPath = ZookeeperPathConstants.getShutdownPath(cluster);
     if (isShuttingDown(shutdownPath)) {
@@ -625,6 +627,15 @@ public class DistributedIndexServer extends AbstractDistributedIndexServer
{
     return shardsToServeCache;
   }
 
+  private List<String> generateShardList(TableDescriptor tableDescriptor) {
+    int shardCount = tableDescriptor.getShardCount();
+    List<String> list = new ArrayList<String>();
+    for (int i = 0; i < shardCount; i++) {
+      list.add(ShardUtil.getShardName(i));
+    }
+    return list;
+  }
+
   private boolean isShuttingDown(String shutdownPath) {
     try {
       Stat stat = _zookeeper.exists(shutdownPath, false);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
index 8cce939..54a06f6 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
@@ -23,8 +23,6 @@ import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.SortedSet;
@@ -173,25 +171,6 @@ public class LocalIndexServer extends AbstractIndexServer {
   }
 
   @Override
-  public List<String> getShardList(String table) {
-    try {
-      List<String> result = new ArrayList<String>();
-      Path tablePath = _tableContext.getTablePath();
-      File tableFile = new File(new File(tablePath.toUri()), table);
-      if (tableFile.isDirectory()) {
-        for (File f : tableFile.listFiles()) {
-          if (f.isDirectory()) {
-            result.add(f.getName());
-          }
-        }
-      }
-      return result;
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  @Override
   public String getNodeName() {
     return "localhost";
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
index 588c87c..69f74a2 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
@@ -48,7 +48,6 @@ import org.apache.blur.thrift.generated.ColumnDefinition;
 import org.apache.blur.thrift.generated.CommandDescriptor;
 import org.apache.blur.thrift.generated.Level;
 import org.apache.blur.thrift.generated.Metric;
-import org.apache.blur.thrift.generated.RowMutation;
 import org.apache.blur.thrift.generated.Schema;
 import org.apache.blur.thrift.generated.Selector;
 import org.apache.blur.thrift.generated.ShardState;
@@ -117,10 +116,23 @@ public abstract class TableAdmin implements Iface {
   @Override
   public final void createTable(TableDescriptor tableDescriptor) throws BlurException, TException
{
     try {
-      TableContext.clear(tableDescriptor.getName());
       ShardUtil.validateTableName(tableDescriptor.getName());
       assignClusterIfNull(tableDescriptor);
-      _clusterStatus.createTable(tableDescriptor);
+      List<String> tableList = _clusterStatus.getTableList(false, tableDescriptor.getCluster());
+      if (!tableList.contains(tableDescriptor.getName())) {
+        TableContext.clear(tableDescriptor.getName());
+        _clusterStatus.createTable(tableDescriptor);
+      } else {
+        TableDescriptor existing = _clusterStatus.getTableDescriptor(false, tableDescriptor.getCluster(),
+            tableDescriptor.getName());
+        if (existing.equals(tableDescriptor)) {
+          LOG.warn("Table [{0}] has already exists, but tried to create same table a second
time.",
+              tableDescriptor.getName());
+        } else {
+          LOG.warn("Table [{0}] has already exists.", tableDescriptor.getName());
+          throw new BException("Table [{0}] has already exists.", tableDescriptor.getName());
+        }
+      }
     } catch (Exception e) {
       LOG.error("Unknown error during create of [table={0}, tableDescriptor={1}]", e, tableDescriptor.name,
           tableDescriptor);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index 6f7516c..cbdd2ce 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -58,7 +58,6 @@ import static org.apache.blur.utils.BlurUtil.quietClose;
 
 import java.io.Closeable;
 import java.io.File;
-import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.util.Map.Entry;
 import java.util.Set;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
index e9edf32..817718a 100644
--- a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
+++ b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
@@ -342,11 +342,6 @@ public class ShardCommandManagerTest {
       }
 
       @Override
-      public List<String> getShardList(String table) {
-        throw new RuntimeException("Not implemented.");
-      }
-
-      @Override
       public long getRowCount(String table) throws IOException {
         throw new RuntimeException("Not implemented.");
       }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
index a52e81d..61b79ee 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
@@ -323,6 +323,7 @@ public abstract class BlurClusterTestBase {
     columnDefinition.setFieldLessIndexed(true);
     columnDefinition.setFieldType("string");
     columnDefinition.setSortable(true);
+    columnDefinition.setMultiValueField(false);
     columnDefinition.setProperties(new HashMap<String, String>());
     client.addColumnDefinition(tableName, columnDefinition);
     long s = System.nanoTime();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java
b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java
index a7dc5dd..1504f01 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java
@@ -16,26 +16,13 @@
  */
 package org.apache.blur.thrift;
 
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
-import java.util.Random;
-import java.util.UUID;
 
-import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thrift.generated.Blur.Iface;
-import org.apache.blur.thrift.generated.BlurException;
-import org.apache.blur.thrift.generated.RecordMutation;
 import org.apache.blur.thrift.generated.RowMutation;
 import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.thrift.generated.TableStats;
-import org.apache.blur.thrift.util.BlurThriftHelper;
 import org.apache.blur.user.User;
-import org.junit.Test;
 
 public class BlurClusterTestNoSecurity extends BlurClusterTestBase {
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/test/java/org/apache/blur/thrift/FacetTests.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/FacetTests.java b/blur-core/src/test/java/org/apache/blur/thrift/FacetTests.java
index 433fa16..d6f858e 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/FacetTests.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/FacetTests.java
@@ -15,22 +15,15 @@ package org.apache.blur.thrift;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.util.List;
 
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thrift.generated.BlurException;
-import org.apache.blur.thrift.generated.BlurQuery;
 import org.apache.blur.thrift.generated.BlurResults;
-import org.apache.blur.thrift.generated.Facet;
-import org.apache.blur.thrift.generated.Query;
-import org.apache.blur.thrift.generated.Selector;
-import org.apache.blur.thrift.generated.Blur.Iface;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
 import org.junit.Test;
 
 public class FacetTests extends BaseClusterTest {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce2bc4d5/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java b/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java
index 7768c6c..46b9173 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/SuiteCluster.java
@@ -42,7 +42,6 @@ public class SuiteCluster {
     }
   }
 
-  @SuppressWarnings({ "unchecked", "rawtypes" })
   public static void setupMiniCluster() throws IOException {
     File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile();
     testDirectory.mkdirs();


Mime
View raw message