incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/2] git commit: Fixed some schema issues.
Date Tue, 20 Aug 2013 14:41:51 GMT
Updated Branches:
  refs/heads/master dccd15276 -> c7385fb29


Fixed some schema issues.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/510de991
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/510de991
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/510de991

Branch: refs/heads/master
Commit: 510de9917c1db85a2c573c15de0c2612691f7f22
Parents: dccd152
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Aug 20 10:18:52 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Aug 20 10:18:52 2013 -0400

----------------------------------------------------------------------
 .../org/apache/blur/manager/IndexManager.java   | 87 +++++++++-----------
 .../org/apache/blur/server/TableContext.java    |  4 +-
 .../apache/blur/analysis/BaseFieldManager.java  | 15 ++++
 .../org/apache/blur/analysis/FieldManager.java  |  7 ++
 .../apache/blur/analysis/HdfsFieldManager.java  | 16 ++++
 .../blur/analysis/BaseFieldManagerTest.java     |  5 ++
 .../blur/lucene/search/SuperParserTest.java     |  6 ++
 .../blur/shell/AddColumnDefinitionCommand.java  |  2 +-
 .../apache/blur/shell/SchemaTableCommand.java   | 31 ++++++-
 9 files changed, 118 insertions(+), 55 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index eba9b29..f4ace8e 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -96,7 +96,6 @@ import org.apache.lucene.document.Document;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.index.StoredFieldVisitor;
@@ -279,12 +278,11 @@ public class IndexManager {
     }
 
     TableContext context = getTableContext(table);
-    Filter preFilter = QueryParserUtil.parseFilter(table, query.recordFilter, false, fieldManager,
-        _filterCache, context);
-    Filter postFilter = QueryParserUtil.parseFilter(table, query.rowFilter, true, fieldManager,
-        _filterCache, context);
-    return QueryParserUtil.parseQuery(query.query, query.rowQuery, fieldManager, postFilter,
-        preFilter, getScoreType(query.scoreType), context);
+    Filter preFilter = QueryParserUtil.parseFilter(table, query.recordFilter, false, fieldManager,
_filterCache,
+        context);
+    Filter postFilter = QueryParserUtil.parseFilter(table, query.rowFilter, true, fieldManager,
_filterCache, context);
+    return QueryParserUtil.parseQuery(query.query, query.rowQuery, fieldManager, postFilter,
preFilter,
+        getScoreType(query.scoreType), context);
   }
 
   private void populateSelector(String table, Selector selector) throws IOException, BlurException
{
@@ -392,10 +390,10 @@ public class IndexManager {
       org.apache.blur.thrift.generated.Query simpleQuery = blurQuery.query;
       Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.recordFilter, false,
fieldManager,
           _filterCache, context);
-      Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.rowFilter, true,
fieldManager,
-          _filterCache, context);
-      Query userQuery = QueryParserUtil.parseQuery(simpleQuery.query, simpleQuery.rowQuery,
fieldManager,
-          postFilter, preFilter, getScoreType(simpleQuery.scoreType), context);
+      Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.rowFilter, true,
fieldManager, _filterCache,
+          context);
+      Query userQuery = QueryParserUtil.parseQuery(simpleQuery.query, simpleQuery.rowQuery,
fieldManager, postFilter,
+          preFilter, getScoreType(simpleQuery.scoreType), context);
       Query facetedQuery = getFacetedQuery(blurQuery, userQuery, facetedCounts, fieldManager,
context, postFilter,
           preFilter);
       call = new SimpleQueryParallelCall(running, table, status, _indexServer, facetedQuery,
blurQuery.selector,
@@ -430,15 +428,16 @@ public class IndexManager {
     }
   }
 
-  public String parseQuery(String table, org.apache.blur.thrift.generated.Query simpleQuery)
throws ParseException, BlurException {
+  public String parseQuery(String table, org.apache.blur.thrift.generated.Query simpleQuery)
throws ParseException,
+      BlurException {
     TableContext context = getTableContext(table);
     FieldManager fieldManager = context.getFieldManager();
-    Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.recordFilter, false,
fieldManager,
-        _filterCache, context);
-    Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.rowFilter, true, fieldManager,
-        _filterCache, context);
-    Query userQuery = QueryParserUtil.parseQuery(simpleQuery.query, simpleQuery.rowQuery,
fieldManager,
-        postFilter, preFilter, getScoreType(simpleQuery.scoreType), context);
+    Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.recordFilter, false,
fieldManager, _filterCache,
+        context);
+    Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.rowFilter, true, fieldManager,
_filterCache,
+        context);
+    Query userQuery = QueryParserUtil.parseQuery(simpleQuery.query, simpleQuery.rowQuery,
fieldManager, postFilter,
+        preFilter, getScoreType(simpleQuery.scoreType), context);
     return userQuery.toString();
   }
 
@@ -459,8 +458,8 @@ public class IndexManager {
     int size = blurQuery.facets.size();
     Query[] queries = new Query[size];
     for (int i = 0; i < size; i++) {
-      queries[i] = QueryParserUtil.parseQuery(blurQuery.facets.get(i).queryStr, blurQuery.query.rowQuery,
-          fieldManager, postFilter, preFilter, ScoreType.CONSTANT, context);
+      queries[i] = QueryParserUtil.parseQuery(blurQuery.facets.get(i).queryStr, blurQuery.query.rowQuery,
fieldManager,
+          postFilter, preFilter, ScoreType.CONSTANT, context);
     }
     return queries;
   }
@@ -744,36 +743,26 @@ public class IndexManager {
   public Schema schema(String table) throws IOException {
     TableContext tableContext = getTableContext(table);
     FieldManager fieldManager = tableContext.getFieldManager();
-    
     Schema schema = new Schema().setTable(table);
-    schema.setFamilies(new HashMap<String, Map<String,ColumnDefinition>>());
-    Map<String, BlurIndex> blurIndexes = _indexServer.getIndexes(table);
-    for (BlurIndex blurIndex : blurIndexes.values()) {
-      IndexSearcherClosable searcher = blurIndex.getIndexReader();
-      try {
-        FieldInfos mergedFieldInfos = MultiFields.getMergedFieldInfos(searcher.getIndexReader());
-        INNER:
-        for (FieldInfo fieldInfo : mergedFieldInfos) {
-          String fieldName = fieldInfo.name;
-          FieldTypeDefinition fieldTypeDefinition = fieldManager.getFieldTypeDefinition(fieldName);
-          if (fieldTypeDefinition == null) {
-            continue INNER;
-          }
-          int index = fieldName.indexOf('.');
-          if (index > 0) {
-            String columnFamily = fieldName.substring(0, index);
-            String column = fieldName.substring(index + 1);
-            Map<String, ColumnDefinition> map = schema.getFamilies().get(columnFamily);
-            if (map == null) {
-              map = new HashMap<String, ColumnDefinition>();
-              schema.putToFamilies(columnFamily, map);
-            }
-            map.put(column, getColumnDefinition(fieldTypeDefinition));
-          }
-        }
-      } finally {
-        // this will allow for closing of index
-        searcher.close();
+    schema.setFamilies(new HashMap<String, Map<String, ColumnDefinition>>());
+    Set<String> fieldNames = fieldManager.getFieldNames();
+    INNER: for (String fieldName : fieldNames) {
+      FieldTypeDefinition fieldTypeDefinition = fieldManager.getFieldTypeDefinition(fieldName);
+      if (fieldTypeDefinition == null) {
+        continue INNER;
+      }
+      String columnName = fieldTypeDefinition.getColumnName();
+      String columnFamily = fieldTypeDefinition.getFamily();
+      String subColumnName = fieldTypeDefinition.getSubColumnName();
+      Map<String, ColumnDefinition> map = schema.getFamilies().get(columnFamily);
+      if (map == null) {
+        map = new HashMap<String, ColumnDefinition>();
+        schema.putToFamilies(columnFamily, map);
+      }
+      if (subColumnName == null) {
+        map.put(columnName, getColumnDefinition(fieldTypeDefinition));
+      } else {
+        map.put(columnName + "." + subColumnName, getColumnDefinition(fieldTypeDefinition));
       }
     }
     return schema;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-core/src/main/java/org/apache/blur/server/TableContext.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
index cc2d541..24fe308 100644
--- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java
+++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
@@ -117,8 +117,10 @@ public class TableContext {
 
     Path storagePath = new Path(tableContext.tablePath, TYPES);
     try {
-      tableContext.fieldManager = new HdfsFieldManager(SUPER, new NoStopWordStandardAnalyzer(),
storagePath,
+      HdfsFieldManager hdfsFieldManager = new HdfsFieldManager(SUPER, new NoStopWordStandardAnalyzer(),
storagePath,
           configuration, strict, defaultMissingFieldType, defaultMissingFieldLessIndexing,
defaultMissingFieldProps);
+      hdfsFieldManager.load();
+      tableContext.fieldManager = hdfsFieldManager;
     } catch (IOException e) {
       throw new RuntimeException(e);
     }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
index 7d2165d..92779ac 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
@@ -155,6 +156,18 @@ public abstract class BaseFieldManager extends FieldManager {
   }
 
   @Override
+  public Set<String> getFieldNames() throws IOException {
+    return new TreeSet<String>(_fieldNameToDefMap.keySet());
+  }
+
+  public void load() throws IOException {
+    List<String> fieldNamesToLoad = getFieldNamesToLoad();
+    for (String fieldName : fieldNamesToLoad) {
+      tryToLoad(fieldName);
+    }
+  }
+
+  @Override
   public void registerType(Class<? extends FieldTypeDefinition> c) {
     try {
       FieldTypeDefinition fieldTypeDefinition = c.newInstance();
@@ -387,6 +400,8 @@ public abstract class BaseFieldManager extends FieldManager {
 
   protected abstract void tryToLoad(String fieldName) throws IOException;
 
+  protected abstract List<String> getFieldNamesToLoad() throws IOException;
+
   private Set<String> getConcurrentSet() {
     return Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
index bd2033c..b8cc592 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
@@ -19,6 +19,7 @@ package org.apache.blur.analysis;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.blur.thrift.generated.Record;
 import org.apache.lucene.analysis.Analyzer;
@@ -250,6 +251,12 @@ public abstract class FieldManager {
    */
   public abstract Query getCustomQuery(String field, String text) throws IOException;
 
+  /**
+   * Register a {@link FieldTypeDefinition} into this field manager.
+   * @param c the class.
+   */
   public abstract void registerType(Class<? extends FieldTypeDefinition> c);
+  
+  public abstract Set<String> getFieldNames() throws IOException;
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
index 71f1434..b56faa1 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
@@ -17,7 +17,9 @@ package org.apache.blur.analysis;
  * limitations under the License.
  */
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
@@ -29,6 +31,7 @@ import org.apache.blur.log.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.lucene.analysis.Analyzer;
@@ -83,6 +86,18 @@ public class HdfsFieldManager extends BaseFieldManager {
     _configuration = configuration;
     _fileSystem = _storagePath.getFileSystem(_configuration);
   }
+  
+  @Override
+  protected List<String> getFieldNamesToLoad() throws IOException {
+    FileStatus[] listStatus = _fileSystem.listStatus(_storagePath);
+    List<String> fieldNames = new ArrayList<String>();
+    for (FileStatus fileStatus : listStatus) {
+      if (!fileStatus.isDir()) {
+        fieldNames.add(fileStatus.getPath().getName());  
+      }
+    }
+    return fieldNames;
+  }
 
   @Override
   protected boolean tryToStore(FieldTypeDefinition fieldTypeDefinition, String fieldName)
throws IOException {
@@ -192,4 +207,5 @@ public class HdfsFieldManager extends BaseFieldManager {
   public static void setLock(Lock lock) {
     _lock = lock;
   }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java
----------------------------------------------------------------------
diff --git a/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java b/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java
index 917338c..bee4b88 100644
--- a/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java
+++ b/blur-query/src/test/java/org/apache/blur/analysis/BaseFieldManagerTest.java
@@ -185,6 +185,11 @@ public class BaseFieldManagerTest {
       protected void tryToLoad(String field) {
 
       }
+      
+      @Override
+      protected List<String> getFieldNamesToLoad() throws IOException {
+        return new ArrayList<String>();
+      }
     };
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
----------------------------------------------------------------------
diff --git a/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java b/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
index 12f5029..9ec2784 100644
--- a/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
+++ b/blur-query/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
@@ -6,6 +6,7 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.blur.analysis.BaseFieldManager;
@@ -61,6 +62,11 @@ public class SuperParserTest {
       protected void tryToLoad(String fieldName) {
 
       }
+
+      @Override
+      protected List<String> getFieldNamesToLoad() throws IOException {
+        return new ArrayList<String>();
+      }
     };
 
     fieldManager.addColumnDefinitionInt("a", "id_i");

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java
b/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java
index 8d7ef16..98b5a2c 100644
--- a/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java
+++ b/blur-shell/src/main/java/org/apache/blur/shell/AddColumnDefinitionCommand.java
@@ -48,7 +48,7 @@ public class AddColumnDefinitionCommand extends Command implements TableFirstArg
     columnDefinition.setColumnName(args[3]);
     columnDefinition.setFieldType(args[4]);
     if (cmd.hasOption("s")) {
-      columnDefinition.setColumnName(cmd.getOptionValue("s"));
+      columnDefinition.setSubColumnName(cmd.getOptionValue("s"));
     }
     if (cmd.hasOption("F")) {
       columnDefinition.setFieldLessIndexed(true);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/510de991/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java
index 67606c3..b33655d 100644
--- a/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java
+++ b/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java
@@ -20,6 +20,10 @@ package org.apache.blur.shell;
 
 import java.io.PrintWriter;
 import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
 
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thrift.generated.Blur;
@@ -37,15 +41,34 @@ public class SchemaTableCommand extends Command implements TableFirstArgCommand
     String tablename = args[1];
 
     Schema schema = client.schema(tablename);
+    out.println(schema);
     out.println(schema.getTable());
     Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies();
-    for (String cf : families.keySet()) {
+    Set<String> familyNames = new TreeSet<String>(families.keySet());
+    for (String cf : familyNames) {
       out.println("family : " + cf);
       Map<String, ColumnDefinition> columns = families.get(cf);
-      for (String c : columns.keySet()) {
+      Set<String> columnNames = new TreeSet<String>(columns.keySet());
+      for (String c : columnNames) {
         ColumnDefinition columnDefinition = columns.get(c);
-//        out.println("\tcolumn : " + c);
-        out.println("\tcolumn : " + columnDefinition);
+        out.println("\tcolumn   : " + columnDefinition.getColumnName());
+        String fieldType = columnDefinition.getFieldType();
+        Map<String, String> properties = columnDefinition.getProperties();
+        String subColumnName = columnDefinition.getSubColumnName();
+        if (subColumnName != null) {
+          out.println(  "\t\t\tsubName   : " + subColumnName);
+          out.println(  "\t\t\tfieldType : " + fieldType);
+          Map<String, String> props = new TreeMap<String, String>(properties);
+          for (Entry<String, String> e : props.entrySet()) {
+            out.println("\t\t\tprop      : " + e);
+          }
+        } else {
+          out.println(  "\t\tfieldType : " + fieldType);
+          Map<String, String> props = new TreeMap<String, String>(properties);
+          for (Entry<String, String> e : props.entrySet()) {
+            out.println("\t\tprop      : " + e);
+          }
+        }
       }
     }
   }


Mime
View raw message