incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [3/3] git commit: Fixing empty schema issue that seems to happen when new field definitions are added to a table or when new fields are added via MR.
Date Fri, 06 Dec 2013 15:55:13 GMT
Fixing empty schema issue that seems to happen when new field definitions are added to a table
or when new fields are added via MR.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/043e52e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/043e52e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/043e52e5

Branch: refs/heads/master
Commit: 043e52e5c55e6d0989ecd3e6a26042d3d699efab
Parents: bc688a4
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Fri Dec 6 10:53:55 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Fri Dec 6 10:55:05 2013 -0500

----------------------------------------------------------------------
 .../org/apache/blur/manager/IndexManager.java   | 42 ----------------
 .../org/apache/blur/server/TableContext.java    |  2 +-
 .../blur/thrift/BlurControllerServer.java       | 31 ------------
 .../org/apache/blur/thrift/BlurShardServer.java |  8 +--
 .../java/org/apache/blur/thrift/TableAdmin.java | 52 ++++++++++++++++++++
 .../apache/blur/manager/IndexManagerTest.java   | 15 ------
 .../org/apache/blur/thrift/BlurClusterTest.java | 25 ++++++++++
 .../apache/blur/analysis/BaseFieldManager.java  |  7 ++-
 .../org/apache/blur/analysis/FieldManager.java  |  2 +
 9 files changed, 86 insertions(+), 98 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index 8804697..6cd9f8e 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -45,7 +45,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLongArray;
 
 import org.apache.blur.analysis.FieldManager;
-import org.apache.blur.analysis.FieldTypeDefinition;
 import org.apache.blur.concurrent.Executors;
 import org.apache.blur.index.ExitableReader;
 import org.apache.blur.index.ExitableReader.ExitingReaderException;
@@ -69,7 +68,6 @@ import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.BlurQuery;
 import org.apache.blur.thrift.generated.BlurQueryStatus;
 import org.apache.blur.thrift.generated.Column;
-import org.apache.blur.thrift.generated.ColumnDefinition;
 import org.apache.blur.thrift.generated.ErrorType;
 import org.apache.blur.thrift.generated.FetchResult;
 import org.apache.blur.thrift.generated.FetchRowResult;
@@ -81,7 +79,6 @@ import org.apache.blur.thrift.generated.RecordMutationType;
 import org.apache.blur.thrift.generated.Row;
 import org.apache.blur.thrift.generated.RowMutation;
 import org.apache.blur.thrift.generated.RowMutationType;
-import org.apache.blur.thrift.generated.Schema;
 import org.apache.blur.thrift.generated.ScoreType;
 import org.apache.blur.thrift.generated.Selector;
 import org.apache.blur.trace.Trace;
@@ -818,45 +815,6 @@ public class IndexManager {
     return new Term(columnFamily + "." + columnName, value);
   }
 
-  public Schema schema(String table) throws IOException {
-    TableContext tableContext = getTableContext(table);
-    FieldManager fieldManager = tableContext.getFieldManager();
-    Schema schema = new Schema().setTable(table);
-    schema.setFamilies(new HashMap<String, Map<String, ColumnDefinition>>());
-    Set<String> fieldNames = fieldManager.getFieldNames();
-    INNER: for (String fieldName : fieldNames) {
-      FieldTypeDefinition fieldTypeDefinition = fieldManager.getFieldTypeDefinition(fieldName);
-      if (fieldTypeDefinition == null) {
-        continue INNER;
-      }
-      String columnName = fieldTypeDefinition.getColumnName();
-      String columnFamily = fieldTypeDefinition.getFamily();
-      String subColumnName = fieldTypeDefinition.getSubColumnName();
-      Map<String, ColumnDefinition> map = schema.getFamilies().get(columnFamily);
-      if (map == null) {
-        map = new HashMap<String, ColumnDefinition>();
-        schema.putToFamilies(columnFamily, map);
-      }
-      if (subColumnName == null) {
-        map.put(columnName, getColumnDefinition(fieldTypeDefinition));
-      } else {
-        map.put(columnName + "." + subColumnName, getColumnDefinition(fieldTypeDefinition));
-      }
-    }
-    return schema;
-  }
-
-  private static ColumnDefinition getColumnDefinition(FieldTypeDefinition fieldTypeDefinition)
{
-    ColumnDefinition columnDefinition = new ColumnDefinition();
-    columnDefinition.setFamily(fieldTypeDefinition.getFamily());
-    columnDefinition.setColumnName(fieldTypeDefinition.getColumnName());
-    columnDefinition.setSubColumnName(fieldTypeDefinition.getSubColumnName());
-    columnDefinition.setFieldLessIndexed(fieldTypeDefinition.isFieldLessIndexed());
-    columnDefinition.setFieldType(fieldTypeDefinition.getFieldType());
-    columnDefinition.setProperties(fieldTypeDefinition.getProperties());
-    return columnDefinition;
-  }
-
   public void mutate(final RowMutation mutation) throws BlurException, IOException {
     long s = System.nanoTime();
     doMutate(mutation);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-core/src/main/java/org/apache/blur/server/TableContext.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
index d361625..d0d54e6 100644
--- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java
+++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
@@ -134,7 +134,7 @@ public class TableContext {
       HdfsFieldManager hdfsFieldManager = new HdfsFieldManager(SUPER, new NoStopWordStandardAnalyzer(),
storagePath,
           configuration, strict, defaultMissingFieldType, defaultMissingFieldLessIndexing,
defaultMissingFieldProps);
       loadCustomTypes(tableContext, blurConfiguration, hdfsFieldManager);
-      hdfsFieldManager.load();
+      hdfsFieldManager.loadFromStorage();
       tableContext.fieldManager = hdfsFieldManager;
     } catch (IOException e) {
       throw new RuntimeException(e);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
index 03a3134..71780f4 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
@@ -802,37 +802,6 @@ public class BlurControllerServer extends TableAdmin implements Iface
{
   }
 
   @Override
-  public Schema schema(final String table) throws BlurException, TException {
-    checkTable(table);
-    try {
-      return scatterGather(getCluster(table), new BlurCommand<Schema>() {
-        @Override
-        public Schema call(Client client) throws BlurException, TException {
-          return client.schema(table);
-        }
-      }, new Merger<Schema>() {
-        @Override
-        public Schema merge(BlurExecutorCompletionService<Schema> service) throws BlurException
{
-          Schema result = null;
-          while (service.getRemainingCount() > 0) {
-            Future<Schema> future = service.poll(_defaultParallelCallTimeout, TimeUnit.MILLISECONDS,
true, table);
-            Schema schema = service.getResultThrowException(future, table);
-            if (result == null) {
-              result = schema;
-            } else {
-              result = BlurControllerServer.merge(result, schema);
-            }
-          }
-          return result;
-        }
-      });
-    } catch (Exception e) {
-      LOG.error("Unknown error while trying to schema table [{0}]", e, table);
-      throw new BException("Unknown error while trying to schema table [{0}]", e, table);
-    }
-  }
-
-  @Override
   public List<String> terms(final String table, final String columnFamily, final String
columnName,
       final String startWith, final short size) throws BlurException, TException {
     checkTable(table);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
index 2c24ad9..1397ba7 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
@@ -286,14 +286,8 @@ public class BlurShardServer extends TableAdmin implements Iface {
 
   @Override
   public Schema schema(String table) throws BlurException, TException {
-    checkTable(_cluster, table);
     resetSearchers();
-    try {
-      return _indexManager.schema(table);
-    } catch (Exception e) {
-      LOG.error("Unknown error while trying to get schema for table [{0}={1}]", e, "table",
table);
-      throw new BException(e.getMessage(), e);
-    }
+    return super.schema(table);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
index d487916..95f9275 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
@@ -25,6 +25,7 @@ import java.util.Set;
 
 import org.apache.blur.BlurConfiguration;
 import org.apache.blur.analysis.FieldManager;
+import org.apache.blur.analysis.FieldTypeDefinition;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.manager.clusterstatus.ClusterStatus;
@@ -34,6 +35,7 @@ import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.ColumnDefinition;
 import org.apache.blur.thrift.generated.Metric;
+import org.apache.blur.thrift.generated.Schema;
 import org.apache.blur.thrift.generated.Selector;
 import org.apache.blur.thrift.generated.ShardState;
 import org.apache.blur.thrift.generated.TableDescriptor;
@@ -517,4 +519,54 @@ public abstract class TableAdmin implements Iface {
   public void setMaxRecordsPerRowFetchRequest(int _maxRecordsPerRowFetchRequest) {
     this._maxRecordsPerRowFetchRequest = _maxRecordsPerRowFetchRequest;
   }
+
+  @Override
+  public Schema schema(String table) throws BlurException, TException {
+    checkTable(table);
+    try {
+      TableContext tableContext = getTableContext(table);
+      FieldManager fieldManager = tableContext.getFieldManager();
+      fieldManager.loadFromStorage();
+      Schema schema = new Schema().setTable(table);
+      schema.setFamilies(new HashMap<String, Map<String, ColumnDefinition>>());
+      Set<String> fieldNames = fieldManager.getFieldNames();
+      INNER: for (String fieldName : fieldNames) {
+        FieldTypeDefinition fieldTypeDefinition = fieldManager.getFieldTypeDefinition(fieldName);
+        if (fieldTypeDefinition == null) {
+          continue INNER;
+        }
+        String columnName = fieldTypeDefinition.getColumnName();
+        String columnFamily = fieldTypeDefinition.getFamily();
+        String subColumnName = fieldTypeDefinition.getSubColumnName();
+        Map<String, ColumnDefinition> map = schema.getFamilies().get(columnFamily);
+        if (map == null) {
+          map = new HashMap<String, ColumnDefinition>();
+          schema.putToFamilies(columnFamily, map);
+        }
+        if (subColumnName == null) {
+          map.put(columnName, getColumnDefinition(fieldTypeDefinition));
+        } else {
+          map.put(columnName + "." + subColumnName, getColumnDefinition(fieldTypeDefinition));
+        }
+      }
+      return schema;
+    } catch (Exception e) {
+      throw new BException("Unknown error while trying to get schema for table [{0}]", table);
+    }
+  }
+
+  private static ColumnDefinition getColumnDefinition(FieldTypeDefinition fieldTypeDefinition)
{
+    ColumnDefinition columnDefinition = new ColumnDefinition();
+    columnDefinition.setFamily(fieldTypeDefinition.getFamily());
+    columnDefinition.setColumnName(fieldTypeDefinition.getColumnName());
+    columnDefinition.setSubColumnName(fieldTypeDefinition.getSubColumnName());
+    columnDefinition.setFieldLessIndexed(fieldTypeDefinition.isFieldLessIndexed());
+    columnDefinition.setFieldType(fieldTypeDefinition.getFieldType());
+    columnDefinition.setProperties(fieldTypeDefinition.getProperties());
+    return columnDefinition;
+  }
+
+  private TableContext getTableContext(final String table) {
+    return TableContext.create(_clusterStatus.getTableDescriptor(true, _clusterStatus.getCluster(true,
table), table));
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
index 55cf481..851df4f 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
@@ -41,9 +41,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Map;
 import java.util.Random;
-import java.util.TreeSet;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicLongArray;
 
@@ -56,7 +54,6 @@ import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.BlurQuery;
 import org.apache.blur.thrift.generated.BlurResult;
 import org.apache.blur.thrift.generated.Column;
-import org.apache.blur.thrift.generated.ColumnDefinition;
 import org.apache.blur.thrift.generated.Facet;
 import org.apache.blur.thrift.generated.FetchRecordResult;
 import org.apache.blur.thrift.generated.FetchResult;
@@ -67,7 +64,6 @@ import org.apache.blur.thrift.generated.RecordMutation;
 import org.apache.blur.thrift.generated.Row;
 import org.apache.blur.thrift.generated.RowMutation;
 import org.apache.blur.thrift.generated.RowMutationType;
-import org.apache.blur.thrift.generated.Schema;
 import org.apache.blur.thrift.generated.ScoreType;
 import org.apache.blur.thrift.generated.Selector;
 import org.apache.blur.thrift.generated.TableDescriptor;
@@ -715,17 +711,6 @@ public class IndexManagerTest {
   }
 
   @Test
-  public void testSchema() throws Exception {
-    Schema schema = indexManager.schema(TABLE);
-    assertEquals(TABLE, schema.table);
-    Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies();
-    assertEquals(new TreeSet<String>(Arrays.asList(FAMILY, FAMILY2)), new TreeSet<String>(families.keySet()));
-    assertEquals(new TreeSet<String>(Arrays.asList("testcol1", "testcol2", "testcol3",
"testcol12", "testcol13")),
-        new TreeSet<String>(families.get(FAMILY).keySet()));
-    assertEquals(new TreeSet<String>(Arrays.asList("testcol18")), new TreeSet<String>(families.get(FAMILY2).keySet()));
-  }
-
-  @Test
   public void testQuerySuperQueryTrue() throws Exception {
     BlurQuery blurQuery = new BlurQuery();
     blurQuery.query = new Query();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
index 6b07d04..59201d0 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
@@ -40,7 +40,9 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.blur.MiniCluster;
 import org.apache.blur.TestType;
+import org.apache.blur.analysis.FieldManager;
 import org.apache.blur.manager.IndexManager;
+import org.apache.blur.server.TableContext;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thrift.generated.Blur;
 import org.apache.blur.thrift.generated.Blur.Iface;
@@ -48,6 +50,7 @@ import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.BlurQuery;
 import org.apache.blur.thrift.generated.BlurResult;
 import org.apache.blur.thrift.generated.BlurResults;
+import org.apache.blur.thrift.generated.ColumnDefinition;
 import org.apache.blur.thrift.generated.ErrorType;
 import org.apache.blur.thrift.generated.FetchResult;
 import org.apache.blur.thrift.generated.Query;
@@ -112,6 +115,7 @@ public class BlurClusterTest {
       KeeperException {
     testCreateTable();
     testLoadTable();
+    testForEmptySchema();
     testQueryWithSelector();
     testBatchFetch();
     testQueryCancel();
@@ -122,6 +126,27 @@ public class BlurClusterTest {
     testCreateTableWithCustomType();
   }
 
+  private void testForEmptySchema() throws BlurException, TException, IOException {
+    Blur.Iface client = getClient();
+    Schema schema = client.schema("test");
+    Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies();
+    assertTrue(!families.isEmpty());
+    int size = families.size();
+    System.out.println(size);
+
+    TableContext tableContext = TableContext.create(client.describe("test"));
+    FieldManager fieldManager = tableContext.getFieldManager();
+
+    assertTrue(fieldManager.addColumnDefinition("test-family", "test-column", null, false,
"string", null));
+
+    TableContext.clear();
+    Schema newschema = client.schema("test");
+    Map<String, Map<String, ColumnDefinition>> newfamilies = newschema.getFamilies();
+    assertTrue(!newfamilies.isEmpty());
+    int newsize = newfamilies.size();
+    assertEquals(size + 1, newsize);
+  }
+
   private void testCreateTableWithCustomType() throws IOException, BlurException, TException
{
     Blur.Iface client = getClient();
     TableDescriptor tableDescriptor = new TableDescriptor();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
index 0121d72..9c974c5 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
@@ -181,10 +181,13 @@ public abstract class BaseFieldManager extends FieldManager {
     return new TreeSet<String>(_fieldNameToDefMap.keySet());
   }
 
-  public void load() throws IOException {
+  @Override
+  public void loadFromStorage() throws IOException {
     List<String> fieldNamesToLoad = getFieldNamesToLoad();
     for (String fieldName : fieldNamesToLoad) {
-      tryToLoad(fieldName);
+      if (!_fieldNameToDefMap.containsKey(fieldName)) {
+        tryToLoad(fieldName);  
+      }
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/043e52e5/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
index 050e6cc..b6b36fd 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
@@ -271,4 +271,6 @@ public abstract class FieldManager {
   
   public abstract String resolveField(String field);
 
+  public abstract void loadFromStorage() throws IOException;
+
 }


Mime
View raw message