tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject tajo git commit: TAJO-1867: TajoMaster should handle the change of ${tajo.root}.
Date Fri, 18 Sep 2015 01:34:06 GMT
Repository: tajo
Updated Branches:
  refs/heads/branch-0.11.0 646cec155 -> 855127f2c


TAJO-1867: TajoMaster should handle the change of ${tajo.root}.

Closes #767


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/855127f2
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/855127f2
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/855127f2

Branch: refs/heads/branch-0.11.0
Commit: 855127f2c65fdfeb6efddc82d6565f15dc78ff46
Parents: 646cec1
Author: Hyunsik Choi <hyunsik@apache.org>
Authored: Thu Sep 17 18:23:39 2015 -0700
Committer: Hyunsik Choi <hyunsik@apache.org>
Committed: Thu Sep 17 18:33:51 2015 -0700

----------------------------------------------------------------------
 CHANGES                                         |  5 +++
 .../src/main/proto/CatalogProtos.proto          |  6 +--
 .../org/apache/tajo/catalog/CatalogServer.java  |  4 +-
 .../tajo/catalog/store/AbstractDBStore.java     |  4 +-
 .../org/apache/tajo/catalog/TestCatalog.java    |  5 +--
 .../TestCatalogAgainstCaseSensitivity.java      |  7 +---
 .../tajo/catalog/TestCatalogExceptions.java     |  8 +---
 .../java/org/apache/tajo/master/TajoMaster.java | 41 ++++++++++++++++++--
 .../apache/tajo/master/exec/DDLExecutor.java    |  2 +-
 9 files changed, 55 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 79537a8..d0111ba 100644
--- a/CHANGES
+++ b/CHANGES
@@ -38,6 +38,8 @@ Release 0.11.0 - unreleased
 
   IMPROVEMENT
 
+    TAJO-1867: TajoMaster should handle the change of ${tajo.root}. (hyunsik)
+
     TAJO-1134: start-tajo.sh should display WEB UI URL and TajoMaster RPC 
     address. (Contributed YeonSu Han, committed by hyunsik)
 
@@ -274,6 +276,9 @@ Release 0.11.0 - unreleased
     TAJO-1846: Python temp directory path should be selected differently based 
     on user platform. (Contributed by Dongkyu Hwangbo, Committed by jihoon)
 
+    TAJO-1848: ShutdownHook in TajoMaster can throw NPE if serviceInit().
+    (hyunsik)
+
     TAJO-1851: Can not release a different rack task. (jinho)
 
     TAJO-1830: Fix race condition in HdfsServiceTracker. (jinho)

http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index cfac82f..cec528a 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@ -292,13 +292,9 @@ message AlterTablespaceProto {
     LOCATION = 0;
   }
 
-  message SetLocation {
-    required string uri = 1;
-  }
-
   message AlterTablespaceCommand {
     required AlterTablespaceType type = 1;
-    optional SetLocation location = 2;
+    optional string location = 2;
   }
 }
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
index 8f4e0e8..a60ddda 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
@@ -374,11 +374,11 @@ public class CatalogServer extends AbstractService {
           for (AlterTablespaceCommand command : request.getCommandList()) {
             if (command.getType() == AlterTablespaceProto.AlterTablespaceType.LOCATION) {
               try {
-                URI uri = URI.create(command.getLocation().getUri());
+                URI uri = URI.create(command.getLocation());
                 Preconditions.checkArgument(uri.getScheme() != null);
               } catch (Exception e) {
                 throw new ServiceException("ALTER TABLESPACE's LOCATION must be a URI form
(scheme:///.../), but "
-                    + command.getLocation().getUri());
+                    + command.getLocation());
               }
             }
           }

http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
index 0f07149..c4d1828 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
@@ -544,12 +544,12 @@ public abstract class AbstractDBStore extends CatalogConstants implements
Catalo
     if (alterProto.getCommandList().size() == 1) {
       AlterTablespaceCommand command = alterProto.getCommand(0);
       if (command.getType() == AlterTablespaceProto.AlterTablespaceType.LOCATION) {
-        AlterTablespaceProto.SetLocation setLocation = command.getLocation();
+        final String uri = command.getLocation();
         try {
           String sql = "UPDATE " + TB_SPACES + " SET SPACE_URI=? WHERE SPACE_NAME=?";
 
           pstmt = conn.prepareStatement(sql);
-          pstmt.setString(1, setLocation.getUri());
+          pstmt.setString(1, uri);
           pstmt.setString(2, alterProto.getSpaceName());
           pstmt.executeUpdate();
         } catch (SQLException se) {

http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
index 8720105..a5e4861 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
@@ -47,7 +47,6 @@ import java.util.*;
 import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
 import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto;
 import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType;
-import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.SetLocation;
 import static org.junit.Assert.*;
 
 public class TestCatalog {
@@ -99,7 +98,7 @@ public class TestCatalog {
     AlterTablespaceProto.AlterTablespaceCommand.Builder commandBuilder =
         AlterTablespaceProto.AlterTablespaceCommand.newBuilder();
     commandBuilder.setType(AlterTablespaceType.LOCATION);
-    commandBuilder.setLocation(SetLocation.newBuilder().setUri("hdfs://zzz.com/warehouse"));
+    commandBuilder.setLocation("hdfs://zzz.com/warehouse");
     AlterTablespaceProto.Builder alter = AlterTablespaceProto.newBuilder();
     alter.setSpaceName("space1");
     alter.addCommand(commandBuilder.build());
@@ -122,7 +121,7 @@ public class TestCatalog {
     // ALTER TABLESPACE space1 LOCATION 'hdfs://zzz.com/warehouse';
     commandBuilder = AlterTablespaceProto.AlterTablespaceCommand.newBuilder();
     commandBuilder.setType(AlterTablespaceType.LOCATION);
-    commandBuilder.setLocation(SetLocation.newBuilder().setUri("hdfs://www.com/warehouse"));
+    commandBuilder.setLocation("hdfs://www.com/warehouse");
     alter = AlterTablespaceProto.newBuilder();
     alter.setSpaceName("space2");
     alter.addCommand(commandBuilder.build());

http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
index bfff6b4..594f5ce 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
@@ -22,7 +22,6 @@ import org.apache.tajo.catalog.partition.PartitionDesc;
 import org.apache.tajo.catalog.proto.CatalogProtos.*;
 import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand;
 import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType;
-import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.SetLocation;
 import org.apache.tajo.catalog.statistics.TableStats;
 import org.apache.tajo.common.TajoDataTypes.Type;
 import org.apache.tajo.exception.UndefinedPartitionException;
@@ -68,16 +67,14 @@ public class TestCatalogAgainstCaseSensitivity {
         addCommand(
             AlterTablespaceCommand.newBuilder().
                 setType(AlterTablespaceType.LOCATION).
-                setLocation(SetLocation.newBuilder()
-                    .setUri("hdfs://zzz.com/warehouse"))).build());
+                setLocation("hdfs://zzz.com/warehouse")).build());
 
     catalog.alterTablespace(AlterTablespaceProto.newBuilder().
         setSpaceName("SpAcE1").
         addCommand(
             AlterTablespaceCommand.newBuilder().
                 setType(AlterTablespaceType.LOCATION).
-                setLocation(SetLocation.newBuilder()
-                    .setUri("hdfs://zzz.com/warehouse"))).build());
+                setLocation("hdfs://zzz.com/warehouse")).build());
 
     Set<TablespaceProto> tablespaceProtos = new HashSet<>();
     for (String tablespaceName : catalog.getAllTablespaceNames()) {

http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java
b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java
index 7be34d8..6586cec 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java
@@ -24,7 +24,6 @@ import org.apache.tajo.catalog.partition.PartitionDesc;
 import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto;
 import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand;
 import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType;
-import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.SetLocation;
 import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
 import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto;
 import org.apache.tajo.catalog.proto.CatalogProtos.UpdateTableStatsProto;
@@ -37,7 +36,6 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.net.URI;
-import java.util.UUID;
 
 public class TestCatalogExceptions {
 
@@ -85,8 +83,7 @@ public class TestCatalogExceptions {
         addCommand(
             AlterTablespaceCommand.newBuilder().
                 setType(AlterTablespaceType.LOCATION).
-                setLocation(SetLocation.newBuilder()
-                    .setUri("hdfs:"))).build());
+                setLocation("hdfs:")).build());
   }
 
   @Test(expected = UndefinedTablespaceException.class)
@@ -96,8 +93,7 @@ public class TestCatalogExceptions {
         addCommand(
             AlterTablespaceCommand.newBuilder().
                 setType(AlterTablespaceType.LOCATION).
-                setLocation(SetLocation.newBuilder()
-                    .setUri("hdfs://zzz.com/warehouse"))).build());
+                setLocation("hdfs://zzz.com/warehouse")).build());
   }
 
   @Test(expected = DuplicateDatabaseException.class)

http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
index 326bc86..2f37cee 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
@@ -34,17 +34,20 @@ import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.RackResolver;
 import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.tajo.algebra.AlterTablespace;
 import org.apache.tajo.catalog.CatalogServer;
 import org.apache.tajo.catalog.CatalogService;
 import org.apache.tajo.catalog.FunctionDesc;
 import org.apache.tajo.catalog.LocalCatalogWrapper;
+import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand;
 import org.apache.tajo.catalog.store.AbstractDBStore;
 import org.apache.tajo.catalog.store.DerbyStore;
 import org.apache.tajo.conf.TajoConf;
 import org.apache.tajo.conf.TajoConf.ConfVars;
 import org.apache.tajo.engine.function.FunctionLoader;
-import org.apache.tajo.exception.DuplicateDatabaseException;
-import org.apache.tajo.exception.DuplicateTablespaceException;
+import org.apache.tajo.exception.*;
 import org.apache.tajo.function.FunctionSignature;
 import org.apache.tajo.master.rm.TajoResourceManager;
 import org.apache.tajo.metrics.ClusterResourceMetricSet;
@@ -382,7 +385,39 @@ public class TajoMaster extends CompositeService {
   private void checkBaseTBSpaceAndDatabase()
       throws IOException, DuplicateDatabaseException, DuplicateTablespaceException {
 
-    if (!catalog.existTablespace(DEFAULT_TABLESPACE_NAME)) {
+    if (catalog.existTablespace(DEFAULT_TABLESPACE_NAME)) { // if default tablespace already
exists
+
+      CatalogProtos.TablespaceProto tablespace = null;
+      try {
+        tablespace = catalog.getTablespace(DEFAULT_TABLESPACE_NAME);
+      } catch (UndefinedTablespaceException e) {
+        throw new TajoInternalError(e);
+      }
+
+      // if warehouse directory and the location of default tablespace are different from
each other
+      if (!tablespace.getUri().equals(context.getConf().getVar(ConfVars.WAREHOUSE_DIR)))
{
+        AlterTablespaceCommand.Builder alterCommand =
+            AlterTablespaceCommand.newBuilder()
+                .setType(AlterTablespaceProto.AlterTablespaceType.LOCATION)
+                .setLocation(context.getConf().getVar(ConfVars.WAREHOUSE_DIR));
+
+        AlterTablespaceProto alterTablespace = AlterTablespaceProto.newBuilder()
+            .setSpaceName(DEFAULT_TABLESPACE_NAME)
+            .addCommand(alterCommand).build();
+
+        // update the location of default tablespace
+        try {
+          catalog.alterTablespace(alterTablespace);
+        } catch (TajoException e) {
+          throw new TajoInternalError(e);
+        }
+
+        LOG.warn(
+            "The location of default tablespace has been changed. " +
+            "You may not accept existing managed tables stored in the previous default tablespace");
+      }
+
+    } else if (!catalog.existTablespace(DEFAULT_TABLESPACE_NAME)) { // if the default tablespace
does not exists
       catalog.createTablespace(DEFAULT_TABLESPACE_NAME, context.getConf().getVar(ConfVars.WAREHOUSE_DIR));
     } else {
       LOG.info(String.format("Default tablespace (%s) is already prepared.", DEFAULT_TABLESPACE_NAME));

http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
index 9f80802..c5d5838 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
@@ -206,7 +206,7 @@ public class DDLExecutor {
       AlterTablespaceProto.AlterTablespaceCommand.Builder commandBuilder =
           AlterTablespaceProto.AlterTablespaceCommand.newBuilder();
       commandBuilder.setType(AlterTablespaceProto.AlterTablespaceType.LOCATION);
-      commandBuilder.setLocation(AlterTablespaceProto.SetLocation.newBuilder().setUri(alterTablespace.getLocation()));
+      commandBuilder.setLocation(alterTablespace.getLocation());
       commandBuilder.build();
       builder.addCommand(commandBuilder);
     } else {


Mime
View raw message