incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Fixed BLUR-299 and BLUR-300
Date Wed, 20 Nov 2013 15:50:57 GMT
Updated Branches:
  refs/heads/apache-blur-0.2 06121ffd5 -> 1266c8169


Fixed BLUR-299 and BLUR-300


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/1266c816
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/1266c816
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/1266c816

Branch: refs/heads/apache-blur-0.2
Commit: 1266c81699a8e4025bce2e45148189a15f5f6b86
Parents: 06121ff
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Wed Nov 20 10:50:57 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Wed Nov 20 10:50:57 2013 -0500

----------------------------------------------------------------------
 .../org/apache/blur/server/TableContext.java    | 30 +++++++++++-------
 .../java/org/apache/blur/thrift/TableAdmin.java |  4 ++-
 .../test/java/org/apache/blur/MiniCluster.java  |  1 +
 .../src/test/java/org/apache/blur/TestType.java | 28 +++++++++++++++++
 .../org/apache/blur/thrift/BlurClusterTest.java | 33 +++++++++++++++++---
 .../apache/blur/analysis/HdfsFieldManager.java  | 11 ++++++-
 6 files changed, 88 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/1266c816/blur-core/src/main/java/org/apache/blur/server/TableContext.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
index 1e22ce6..d361625 100644
--- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java
+++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
@@ -83,7 +83,6 @@ public class TableContext {
     cache.clear();
   }
 
-  @SuppressWarnings("unchecked")
   public static TableContext create(TableDescriptor tableDescriptor) {
     if (tableDescriptor == null) {
       throw new NullPointerException("TableDescriptor can not be null.");
@@ -134,23 +133,40 @@ public class TableContext {
     try {
       HdfsFieldManager hdfsFieldManager = new HdfsFieldManager(SUPER, new NoStopWordStandardAnalyzer(),
storagePath,
           configuration, strict, defaultMissingFieldType, defaultMissingFieldLessIndexing,
defaultMissingFieldProps);
+      loadCustomTypes(tableContext, blurConfiguration, hdfsFieldManager);
       hdfsFieldManager.load();
       tableContext.fieldManager = hdfsFieldManager;
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
 
+    Class<?> c1 = configuration.getClass(BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE,
+        KeepOnlyLastCommitDeletionPolicy.class);
+    tableContext.indexDeletionPolicy = (IndexDeletionPolicy) configure(ReflectionUtils.newInstance(c1,
configuration),
+        tableContext);
+    Class<?> c2 = configuration.getClass(BLUR_SHARD_INDEX_SIMILARITY, DefaultSimilarity.class);
+    tableContext.similarity = (Similarity) configure(ReflectionUtils.newInstance(c2, configuration),
tableContext);
+
+    cache.put(name, tableContext);
+    return tableContext;
+  }
+
+  @SuppressWarnings("unchecked")
+  private static void loadCustomTypes(TableContext tableContext, BlurConfiguration blurConfiguration,
+      FieldManager fieldManager) {
     Set<Entry<String, String>> entrySet = blurConfiguration.getProperties().entrySet();
+    TableDescriptor descriptor = tableContext.descriptor;
     for (Entry<String, String> entry : entrySet) {
       String key = entry.getKey();
       if (key.startsWith(BLUR_FIELDTYPE)) {
         String className = entry.getValue();
+        descriptor.putToTableProperties(key, className);
         LOG.info("Attempting to load new type [{0}]", className);
         Class<? extends FieldTypeDefinition> clazz;
         try {
           clazz = (Class<? extends FieldTypeDefinition>) Class.forName(className);
           FieldTypeDefinition fieldTypeDefinition = clazz.newInstance();
-          tableContext.fieldManager.registerType(clazz);
+          fieldManager.registerType(clazz);
           LOG.info("Sucessfully loaded new type [{0}] with name [{1}]", className, fieldTypeDefinition.getName());
         } catch (ClassNotFoundException e) {
           LOG.error("The field type definition class [{0}] was not found.  Check the classpath.",
e, className);
@@ -161,16 +177,6 @@ public class TableContext {
         }
       }
     }
-
-    Class<?> c1 = configuration.getClass(BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE,
-        KeepOnlyLastCommitDeletionPolicy.class);
-    tableContext.indexDeletionPolicy = (IndexDeletionPolicy) configure(ReflectionUtils.newInstance(c1,
configuration),
-        tableContext);
-    Class<?> c2 = configuration.getClass(BLUR_SHARD_INDEX_SIMILARITY, DefaultSimilarity.class);
-    tableContext.similarity = (Similarity) configure(ReflectionUtils.newInstance(c2, configuration),
tableContext);
-
-    cache.put(name, tableContext);
-    return tableContext;
   }
 
   private static Map<String, String> emptyIfNull(Map<String, String> defaultMissingFieldProps)
{

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/1266c816/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
index cac2818..9bd68a3 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
@@ -370,7 +370,9 @@ public abstract class TableAdmin implements Iface {
       if (cluster == null) {
         throw new BException("Table [" + table + "] not found.");
       }
-      return _clusterStatus.getTableDescriptor(true, cluster, table);
+      TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(true, cluster,
table);
+      TableContext tableContext = TableContext.create(tableDescriptor);
+      return tableContext.getDescriptor();
     } catch (Exception e) {
       LOG.error("Unknown error while trying to describe a table [" + table + "].", e);
       throw new BException("Unknown error while trying to describe a table [" + table + "].",
e);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/1266c816/blur-core/src/test/java/org/apache/blur/MiniCluster.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
index d8a40fb..bd8af0c 100644
--- a/blur-core/src/test/java/org/apache/blur/MiniCluster.java
+++ b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
@@ -223,6 +223,7 @@ public class MiniCluster {
     configuration.setLong(BLUR_SHARD_SAFEMODEDELAY, 5000);
     configuration.setInt(BLUR_GUI_CONTROLLER_PORT, -1);
     configuration.setInt(BLUR_GUI_SHARD_PORT, -1);
+    configuration.set("blur.fieldtype.customtype1", TestType.class.getName());
 
     return configuration;
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/1266c816/blur-core/src/test/java/org/apache/blur/TestType.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/TestType.java b/blur-core/src/test/java/org/apache/blur/TestType.java
new file mode 100644
index 0000000..9575102
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/TestType.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur;
+
+import org.apache.blur.analysis.type.StringFieldTypeDefinition;
+
+public class TestType extends StringFieldTypeDefinition {
+
+  @Override
+  public String getName() {
+    return "test_type";
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/1266c816/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
index eb96e78..ed8ae00 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.File;
@@ -31,12 +32,14 @@ import java.lang.management.MemoryUsage;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.blur.MiniCluster;
+import org.apache.blur.TestType;
 import org.apache.blur.manager.IndexManager;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thrift.generated.Blur;
@@ -116,6 +119,26 @@ public class BlurClusterTest {
     testTestShardFailover();
     testTermsList();
     testCreateDisableAndRemoveTable();
+    testCreateTableWithCustomType();
+  }
+
+  private void testCreateTableWithCustomType() throws IOException, BlurException, TException
{
+    Blur.Iface client = getClient();
+    TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.setName("test_type");
+    tableDescriptor.setShardCount(1);
+    tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/test_type");
+    client.createTable(tableDescriptor);
+    List<String> tableList = client.tableList();
+    assertTrue(tableList.contains("test_type"));
+
+    client.disableTable("test_type");
+
+    client.enableTable("test_type");
+
+    TableDescriptor describe = client.describe("test_type");
+    Map<String, String> tableProperties = describe.getTableProperties();
+    assertEquals(TestType.class.getName(), tableProperties.get("blur.fieldtype.customtype1"));
   }
 
   public void testCreateTable() throws BlurException, TException, IOException {
@@ -167,7 +190,7 @@ public class BlurClusterTest {
     Schema schema = client.schema("test");
     assertFalse(schema.getFamilies().isEmpty());
   }
-  
+
   private void testQueryWithSelector() throws BlurException, TException {
     Iface client = getClient();
     BlurQuery blurQueryRow = new BlurQuery();
@@ -177,15 +200,15 @@ public class BlurClusterTest {
     blurQueryRow.setUseCacheIfPresent(false);
     blurQueryRow.setCacheResult(false);
     blurQueryRow.setSelector(new Selector());
-    
+
     BlurResults resultsRow = client.query("test", blurQueryRow);
-//    assertRowResults(resultsRow);
+    // assertRowResults(resultsRow);
     assertEquals(100, resultsRow.getTotalResults());
-    
+
     for (BlurResult blurResult : resultsRow.getResults()) {
       System.out.println(blurResult);
     }
-    
+
     System.out.println();
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/1266c816/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
index a293c91..daf43ee 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.lucene.analysis.Analyzer;
 
 public class HdfsFieldManager extends BaseFieldManager {
@@ -95,7 +96,15 @@ public class HdfsFieldManager extends BaseFieldManager {
     if (!_fileSystem.exists(_storagePath)) {
       return EMPTY_LIST;
     }
-    FileStatus[] listStatus = _fileSystem.listStatus(_storagePath);
+    FileStatus[] listStatus = _fileSystem.listStatus(_storagePath, new PathFilter() {
+      @Override
+      public boolean accept(Path path) {
+        if (path.getName().endsWith(".tmp")) {
+          return false;
+        }
+        return true;
+      }
+    });
     if (listStatus == null) {
       return EMPTY_LIST;
     }


Mime
View raw message