incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Fixed BLUR-250 and partial fix for BLUR-251
Date Fri, 04 Oct 2013 13:20:34 GMT
Updated Branches:
  refs/heads/apache-blur-0.2 e9b42ee09 -> 48617571e


Fixed BLUR-250 and partial fix for BLUR-251


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/48617571
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/48617571
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/48617571

Branch: refs/heads/apache-blur-0.2
Commit: 48617571ebb0ae5cb2051d0a01e8858d439ad458
Parents: e9b42ee
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Fri Oct 4 09:20:20 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Fri Oct 4 09:20:20 2013 -0400

----------------------------------------------------------------------
 .../org/apache/blur/server/TableContext.java    | 75 ++++++++++++++++-
 .../blur/thrift/ThriftBlurShardServer.java      |  3 +
 .../apache/blur/server/TableContextTest.java    | 86 ++++++++++++++++++++
 .../java/org/apache/blur/server/TestType.java   | 12 +++
 .../org/apache/blur/utils/BlurConstants.java    |  2 +
 5 files changed, 174 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/48617571/blur-core/src/main/java/org/apache/blur/server/TableContext.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
index 24fe308..5b85e2e 100644
--- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java
+++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
@@ -16,6 +16,7 @@ package org.apache.blur.server;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import static org.apache.blur.utils.BlurConstants.BLUR_FIELDTYPE;
 import static org.apache.blur.utils.BlurConstants.BLUR_SAHRD_INDEX_SIMILARITY;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_COMMITS;
@@ -26,10 +27,13 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.blur.BlurConfiguration;
 import org.apache.blur.analysis.FieldManager;
+import org.apache.blur.analysis.FieldTypeDefinition;
 import org.apache.blur.analysis.HdfsFieldManager;
 import org.apache.blur.analysis.NoStopWordStandardAnalyzer;
 import org.apache.blur.log.Log;
@@ -52,6 +56,10 @@ public class TableContext {
   private static final String LOGS = "logs";
   private static final String TYPES = "types";
 
+  private static ConcurrentHashMap<String, TableContext> cache = new ConcurrentHashMap<String,
TableContext>();
+  private static Configuration systemConfiguration;
+  private static BlurConfiguration systemBlurConfiguration;
+
   private Path tablePath;
   private Path walTablePath;
   private String defaultFieldName;
@@ -65,8 +73,7 @@ public class TableContext {
   private ScoreType defaultScoreType;
   private Term defaultPrimeDocTerm;
   private FieldManager fieldManager;
-
-  private static ConcurrentHashMap<String, TableContext> cache = new ConcurrentHashMap<String,
TableContext>();
+  private BlurConfiguration blurConfiguration;
 
   protected TableContext() {
 
@@ -76,6 +83,7 @@ public class TableContext {
     cache.clear();
   }
 
+  @SuppressWarnings("unchecked")
   public static TableContext create(TableDescriptor tableDescriptor) {
     if (tableDescriptor == null) {
       throw new NullPointerException("TableDescriptor can not be null.");
@@ -84,22 +92,29 @@ public class TableContext {
     if (name == null) {
       throw new NullPointerException("Table name in the TableDescriptor can not be null.");
     }
+    String tableUri = tableDescriptor.getTableUri();
+    if (tableUri == null) {
+      throw new NullPointerException("Table uri in the TableDescriptor can not be null.");
+    }
     TableContext tableContext = cache.get(name);
     if (tableContext != null) {
       return tableContext;
     }
     LOG.info("Creating table context for table [{0}]", name);
-    Configuration configuration = new Configuration();
+    Configuration configuration = getSystemConfiguration();
+    BlurConfiguration blurConfiguration = getSystemBlurConfiguration();
     Map<String, String> tableProperties = tableDescriptor.getTableProperties();
     if (tableProperties != null) {
       for (Entry<String, String> prop : tableProperties.entrySet()) {
         configuration.set(prop.getKey(), prop.getValue());
+        blurConfiguration.set(prop.getKey(), prop.getValue());
       }
     }
 
     tableContext = new TableContext();
     tableContext.configuration = configuration;
-    tableContext.tablePath = new Path(tableDescriptor.getTableUri());
+    tableContext.blurConfiguration = blurConfiguration;
+    tableContext.tablePath = new Path(tableUri);
     tableContext.walTablePath = new Path(tableContext.tablePath, LOGS);
 
     tableContext.defaultFieldName = SUPER;
@@ -125,6 +140,28 @@ public class TableContext {
       throw new RuntimeException(e);
     }
 
+    Set<Entry<String, String>> entrySet = blurConfiguration.getProperties().entrySet();
+    for (Entry<String, String> entry : entrySet) {
+      String key = entry.getKey();
+      if (key.startsWith(BLUR_FIELDTYPE)) {
+        String className = entry.getValue();
+        LOG.info("Attempting to load new type [{0}]", className);
+        Class<? extends FieldTypeDefinition> clazz;
+        try {
+          clazz = (Class<? extends FieldTypeDefinition>) Class.forName(className);
+          FieldTypeDefinition fieldTypeDefinition = clazz.newInstance();
+          tableContext.fieldManager.registerType(clazz);
+          LOG.info("Sucessfully loaded new type [{0}] with name [{1}]", className, fieldTypeDefinition.getName());
+        } catch (ClassNotFoundException e) {
+          LOG.error("The field type definition class [{0}] was not found.  Check the classpath.",
e, className);
+        } catch (InstantiationException e) {
+          LOG.error("Could not create the field type definition [{0}].", e, className);
+        } catch (IllegalAccessException e) {
+          LOG.error("Unknown exception while trying to load field type definition [{0}].",
e, className);
+        }
+      }
+    }
+
     Class<?> c1 = configuration.getClass(BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE,
         KeepOnlyLastCommitDeletionPolicy.class);
     tableContext.indexDeletionPolicy = (IndexDeletionPolicy) configure(ReflectionUtils.newInstance(c1,
configuration),
@@ -205,4 +242,34 @@ public class TableContext {
   public long getTimeBetweenWALSyncsNanos() {
     return TimeUnit.MILLISECONDS.toNanos(10);
   }
+
+  public BlurConfiguration getBlurConfiguration() {
+    return blurConfiguration;
+  }
+
+  public static synchronized Configuration getSystemConfiguration() {
+    if (systemConfiguration == null) {
+      systemConfiguration = new Configuration();
+    }
+    return systemConfiguration;
+  }
+
+  public static void setSystemConfiguration(Configuration systemConfiguration) {
+    TableContext.systemConfiguration = systemConfiguration;
+  }
+
+  public static synchronized BlurConfiguration getSystemBlurConfiguration() {
+    if (systemBlurConfiguration == null) {
+      try {
+        systemBlurConfiguration = new BlurConfiguration();
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+    return systemBlurConfiguration;
+  }
+
+  public static void setSystemBlurConfiguration(BlurConfiguration systemBlurConfiguration)
{
+    TableContext.systemBlurConfiguration = systemBlurConfiguration;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/48617571/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index 4a89cff..4c7d133 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -69,6 +69,7 @@ import org.apache.blur.manager.writer.BlurIndexRefresher;
 import org.apache.blur.metrics.JSONReporter;
 import org.apache.blur.metrics.ReporterSetup;
 import org.apache.blur.server.ShardServerEventHandler;
+import org.apache.blur.server.TableContext;
 import org.apache.blur.store.BlockCacheDirectoryFactory;
 import org.apache.blur.store.BlockCacheDirectoryFactoryV1;
 import org.apache.blur.store.BlockCacheDirectoryFactoryV2;
@@ -117,6 +118,8 @@ public class ThriftBlurShardServer extends ThriftServer {
   public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration,
boolean randomPort)
       throws Exception {
     Configuration config = new Configuration();
+    TableContext.setSystemBlurConfiguration(configuration);
+    TableContext.setSystemConfiguration(config);
 
     String bindAddress = configuration.get(BLUR_SHARD_BIND_ADDRESS);
     int bindPort = configuration.getInt(BLUR_SHARD_BIND_PORT, -1);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/48617571/blur-core/src/test/java/org/apache/blur/server/TableContextTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/server/TableContextTest.java b/blur-core/src/test/java/org/apache/blur/server/TableContextTest.java
new file mode 100644
index 0000000..45e3c91
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/server/TableContextTest.java
@@ -0,0 +1,86 @@
+package org.apache.blur.server;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.blur.BlurConfiguration;
+import org.apache.blur.analysis.FieldManager;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.blur.utils.BlurConstants;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TableContextTest {
+
+  private static final File TMPDIR = new File("./target/tmp");
+  private File base;
+  private String name = "testtable-TableContextTest";
+  private File file;
+
+  @Before
+  public void setup() {
+    TableContext.clear();
+    base = new File(TMPDIR, "TableContextTest");
+    rm(base);
+    file = new File(base, name);
+    file.mkdirs();
+  }
+
+  @Test
+  public void testLoadingNewTypeWhenNotDefined() throws IOException {
+    TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.setName(name);
+    tableDescriptor.setTableUri(file.toURI().toString());
+    TableContext context = TableContext.create(tableDescriptor);
+
+    FieldManager fieldManager = context.getFieldManager();
+    try {
+      fieldManager.addColumnDefinition("fam", "col", null, false, "test", null);
+      fail("should fail because new type is not loaded.");
+    } catch (IllegalArgumentException e) {
+
+    }
+  }
+
+  @Test
+  public void testLoadingNewTypeWhenDefinedFromTableDescriptor() throws IOException {
+    TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.setName(name);
+    tableDescriptor.setTableUri(file.toURI().toString());
+    tableDescriptor.putToTableProperties(BlurConstants.BLUR_FIELDTYPE + "test", TestType.class.getName());
+
+    TableContext context = TableContext.create(tableDescriptor);
+    FieldManager fieldManager = context.getFieldManager();
+
+    fieldManager.addColumnDefinition("fam", "col", null, false, "test", null);
+  }
+  
+  @Test
+  public void testLoadingNewTypeWhenDefinedFromBlurConfigure() throws IOException {
+    BlurConfiguration blurConfiguration = new BlurConfiguration();
+    blurConfiguration.set(BlurConstants.BLUR_FIELDTYPE + "test", TestType.class.getName());
+    
+    TableContext.setSystemBlurConfiguration(blurConfiguration);
+    
+    TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.setName(name);
+    tableDescriptor.setTableUri(file.toURI().toString());
+
+    TableContext context = TableContext.create(tableDescriptor);
+    FieldManager fieldManager = context.getFieldManager();
+
+    fieldManager.addColumnDefinition("fam", "col", null, false, "test", null);
+  }
+
+  private void rm(File file) {
+    if (file.isDirectory()) {
+      for (File f : file.listFiles()) {
+        rm(f);
+      }
+    }
+    file.delete();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/48617571/blur-core/src/test/java/org/apache/blur/server/TestType.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/server/TestType.java b/blur-core/src/test/java/org/apache/blur/server/TestType.java
new file mode 100644
index 0000000..6c2e6ab
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/server/TestType.java
@@ -0,0 +1,12 @@
+package org.apache.blur.server;
+
+import org.apache.blur.analysis.type.TextFieldTypeDefinition;
+
+public class TestType extends TextFieldTypeDefinition {
+
+  @Override
+  public String getName() {
+    return "test";
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/48617571/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index 1439aa1..ecc1b41 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -72,6 +72,8 @@ public class BlurConstants {
   public static final String BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE = "blur.shard.index.deletion.policy.maxage";
   public static final String BLUR_ZOOKEEPER_SYSTEM_TIME_TOLERANCE = "blur.zookeeper.system.time.tolerance";
   public static final String BLUR_SAHRD_INDEX_SIMILARITY = "blur.sahrd.index.similarity";
+  
+  public static final String BLUR_FIELDTYPE = "blur.fieldtype.";
 
   public static final String BLUR_SHARD_TIME_BETWEEN_COMMITS = "blur.shard.time.between.commits";
   public static final String BLUR_SHARD_TIME_BETWEEN_REFRESHS = "blur.shard.time.between.refreshs";


Mime
View raw message