Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id E6AD7200AE3 for ; Thu, 5 May 2016 01:38:24 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id E562D160A02; Wed, 4 May 2016 23:38:24 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id C0F891609FF for ; Thu, 5 May 2016 01:38:23 +0200 (CEST) Received: (qmail 38365 invoked by uid 500); 4 May 2016 23:38:23 -0000 Mailing-List: contact commits-help@drill.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: commits@drill.apache.org Delivered-To: mailing list commits@drill.apache.org Received: (qmail 38335 invoked by uid 99); 4 May 2016 23:38:23 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 04 May 2016 23:38:22 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C4F30DFF73; Wed, 4 May 2016 23:38:22 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: parthc@apache.org To: commits@drill.apache.org Date: Wed, 04 May 2016 23:38:24 -0000 Message-Id: <14390e1e74ab4091ab05b809c4f6e7b6@git.apache.org> In-Reply-To: <2f901a5f67bb4cc6a8cb5d94aa7f1605@git.apache.org> References: <2f901a5f67bb4cc6a8cb5d94aa7f1605@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [3/4] drill git commit: DRILL-4132 Ability to submit simple type of physical plan directly to EndPoint DrillBit for execution. There are multiple changes to achieve this: 1. During physical planning split single plan into multiple based on the number of archived-at: Wed, 04 May 2016 23:38:25 -0000 http://git-wip-us.apache.org/repos/asf/drill/blob/6bba69d4/protocol/src/main/java/org/apache/drill/exec/proto/SchemaExecProtos.java ---------------------------------------------------------------------- diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaExecProtos.java b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaExecProtos.java index 1775f30..f996868 100644 --- a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaExecProtos.java +++ b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaExecProtos.java @@ -42,6 +42,9 @@ public final class SchemaExecProtos output.writeInt32(2, message.getMajorFragmentId(), false); if(message.hasMinorFragmentId()) output.writeInt32(3, message.getMinorFragmentId(), false); + if(message.hasParentQueryId()) + output.writeObject(4, message.getParentQueryId(), org.apache.drill.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); + } public boolean isInitialized(org.apache.drill.exec.proto.ExecProtos.FragmentHandle message) { @@ -91,6 +94,10 @@ public final class SchemaExecProtos case 3: builder.setMinorFragmentId(input.readInt32()); break; + case 4: + builder.setParentQueryId(input.mergeObject(org.apache.drill.exec.proto.UserBitShared.QueryId.newBuilder(), org.apache.drill.exec.proto.SchemaUserBitShared.QueryId.MERGE)); + + break; default: input.handleUnknownField(number, this); } @@ -134,6 +141,7 @@ public final class SchemaExecProtos case 1: return "queryId"; case 2: return "majorFragmentId"; case 3: return "minorFragmentId"; + case 4: return "parentQueryId"; default: return null; } } @@ -148,6 +156,7 @@ public final class SchemaExecProtos fieldMap.put("queryId", 1); fieldMap.put("majorFragmentId", 2); fieldMap.put("minorFragmentId", 3); + fieldMap.put("parentQueryId", 4); } } http://git-wip-us.apache.org/repos/asf/drill/blob/6bba69d4/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserProtos.java ---------------------------------------------------------------------- diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserProtos.java b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserProtos.java index 6fc43bb..10764be 100644 --- a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserProtos.java +++ b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserProtos.java @@ -549,6 +549,9 @@ public final class SchemaUserProtos output.writeEnum(2, message.getType().getNumber(), false); if(message.hasPlan()) output.writeString(3, message.getPlan(), false); + for(org.apache.drill.exec.proto.BitControl.PlanFragment fragments : message.getFragmentsList()) + output.writeObject(4, fragments, org.apache.drill.exec.proto.SchemaBitControl.PlanFragment.WRITE, true); + } public boolean isInitialized(org.apache.drill.exec.proto.UserProtos.RunQuery message) { @@ -597,6 +600,10 @@ public final class SchemaUserProtos case 3: builder.setPlan(input.readString()); break; + case 4: + builder.addFragments(input.mergeObject(org.apache.drill.exec.proto.BitControl.PlanFragment.newBuilder(), org.apache.drill.exec.proto.SchemaBitControl.PlanFragment.MERGE)); + + break; default: input.handleUnknownField(number, this); } @@ -640,6 +647,7 @@ public final class SchemaUserProtos case 1: return "resultsMode"; case 2: return "type"; case 3: return "plan"; + case 4: return "fragments"; default: return null; } } @@ -654,6 +662,270 @@ public final class SchemaUserProtos fieldMap.put("resultsMode", 1); fieldMap.put("type", 2); fieldMap.put("plan", 3); + fieldMap.put("fragments", 4); + } + } + + public static final class GetQueryPlanFragments + { + public static final org.apache.drill.exec.proto.SchemaUserProtos.GetQueryPlanFragments.MessageSchema WRITE = + new org.apache.drill.exec.proto.SchemaUserProtos.GetQueryPlanFragments.MessageSchema(); + public static final org.apache.drill.exec.proto.SchemaUserProtos.GetQueryPlanFragments.BuilderSchema MERGE = + new org.apache.drill.exec.proto.SchemaUserProtos.GetQueryPlanFragments.BuilderSchema(); + + public static class MessageSchema implements com.dyuproject.protostuff.Schema + { + public void writeTo(com.dyuproject.protostuff.Output output, org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments message) throws java.io.IOException + { + if(message.hasQuery()) + output.writeString(1, message.getQuery(), false); + if(message.hasType()) + output.writeEnum(2, message.getType().getNumber(), false); + if(message.hasSplitPlan()) + output.writeBool(3, message.getSplitPlan(), false); + } + public boolean isInitialized(org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments message) + { + return message.isInitialized(); + } + public java.lang.String getFieldName(int number) + { + return org.apache.drill.exec.proto.SchemaUserProtos.GetQueryPlanFragments.getFieldName(number); + } + public int getFieldNumber(java.lang.String name) + { + return org.apache.drill.exec.proto.SchemaUserProtos.GetQueryPlanFragments.getFieldNumber(name); + } + public java.lang.Class typeClass() + { + return org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments.class; + } + public java.lang.String messageName() + { + return org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments.class.getSimpleName(); + } + public java.lang.String messageFullName() + { + return org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments.class.getName(); + } + //unused + public void mergeFrom(com.dyuproject.protostuff.Input input, org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments message) throws java.io.IOException {} + public org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments newMessage() { return null; } + } + public static class BuilderSchema implements com.dyuproject.protostuff.Schema + { + public void mergeFrom(com.dyuproject.protostuff.Input input, org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments.Builder builder) throws java.io.IOException + { + for(int number = input.readFieldNumber(this);; number = input.readFieldNumber(this)) + { + switch(number) + { + case 0: + return; + case 1: + builder.setQuery(input.readString()); + break; + case 2: + builder.setType(org.apache.drill.exec.proto.UserBitShared.QueryType.valueOf(input.readEnum())); + break; + case 3: + builder.setSplitPlan(input.readBool()); + break; + default: + input.handleUnknownField(number, this); + } + } + } + public boolean isInitialized(org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments.Builder builder) + { + return builder.isInitialized(); + } + public org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments.Builder newMessage() + { + return org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments.newBuilder(); + } + public java.lang.String getFieldName(int number) + { + return org.apache.drill.exec.proto.SchemaUserProtos.GetQueryPlanFragments.getFieldName(number); + } + public int getFieldNumber(java.lang.String name) + { + return org.apache.drill.exec.proto.SchemaUserProtos.GetQueryPlanFragments.getFieldNumber(name); + } + public java.lang.Class typeClass() + { + return org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments.Builder.class; + } + public java.lang.String messageName() + { + return org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments.class.getSimpleName(); + } + public java.lang.String messageFullName() + { + return org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments.class.getName(); + } + //unused + public void writeTo(com.dyuproject.protostuff.Output output, org.apache.drill.exec.proto.UserProtos.GetQueryPlanFragments.Builder builder) throws java.io.IOException {} + } + public static java.lang.String getFieldName(int number) + { + switch(number) + { + case 1: return "query"; + case 2: return "type"; + case 3: return "splitPlan"; + default: return null; + } + } + public static int getFieldNumber(java.lang.String name) + { + java.lang.Integer number = fieldMap.get(name); + return number == null ? 0 : number.intValue(); + } + private static final java.util.HashMap fieldMap = new java.util.HashMap(); + static + { + fieldMap.put("query", 1); + fieldMap.put("type", 2); + fieldMap.put("splitPlan", 3); + } + } + + public static final class QueryPlanFragments + { + public static final org.apache.drill.exec.proto.SchemaUserProtos.QueryPlanFragments.MessageSchema WRITE = + new org.apache.drill.exec.proto.SchemaUserProtos.QueryPlanFragments.MessageSchema(); + public static final org.apache.drill.exec.proto.SchemaUserProtos.QueryPlanFragments.BuilderSchema MERGE = + new org.apache.drill.exec.proto.SchemaUserProtos.QueryPlanFragments.BuilderSchema(); + + public static class MessageSchema implements com.dyuproject.protostuff.Schema + { + public void writeTo(com.dyuproject.protostuff.Output output, org.apache.drill.exec.proto.UserProtos.QueryPlanFragments message) throws java.io.IOException + { + if(message.hasStatus()) + output.writeEnum(1, message.getStatus().getNumber(), false); + if(message.hasQueryId()) + output.writeObject(2, message.getQueryId(), org.apache.drill.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); + + for(org.apache.drill.exec.proto.BitControl.PlanFragment fragments : message.getFragmentsList()) + output.writeObject(3, fragments, org.apache.drill.exec.proto.SchemaBitControl.PlanFragment.WRITE, true); + + if(message.hasError()) + output.writeObject(4, message.getError(), org.apache.drill.exec.proto.SchemaUserBitShared.DrillPBError.WRITE, false); + + } + public boolean isInitialized(org.apache.drill.exec.proto.UserProtos.QueryPlanFragments message) + { + return message.isInitialized(); + } + public java.lang.String getFieldName(int number) + { + return org.apache.drill.exec.proto.SchemaUserProtos.QueryPlanFragments.getFieldName(number); + } + public int getFieldNumber(java.lang.String name) + { + return org.apache.drill.exec.proto.SchemaUserProtos.QueryPlanFragments.getFieldNumber(name); + } + public java.lang.Class typeClass() + { + return org.apache.drill.exec.proto.UserProtos.QueryPlanFragments.class; + } + public java.lang.String messageName() + { + return org.apache.drill.exec.proto.UserProtos.QueryPlanFragments.class.getSimpleName(); + } + public java.lang.String messageFullName() + { + return org.apache.drill.exec.proto.UserProtos.QueryPlanFragments.class.getName(); + } + //unused + public void mergeFrom(com.dyuproject.protostuff.Input input, org.apache.drill.exec.proto.UserProtos.QueryPlanFragments message) throws java.io.IOException {} + public org.apache.drill.exec.proto.UserProtos.QueryPlanFragments newMessage() { return null; } + } + public static class BuilderSchema implements com.dyuproject.protostuff.Schema + { + public void mergeFrom(com.dyuproject.protostuff.Input input, org.apache.drill.exec.proto.UserProtos.QueryPlanFragments.Builder builder) throws java.io.IOException + { + for(int number = input.readFieldNumber(this);; number = input.readFieldNumber(this)) + { + switch(number) + { + case 0: + return; + case 1: + builder.setStatus(org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState.valueOf(input.readEnum())); + break; + case 2: + builder.setQueryId(input.mergeObject(org.apache.drill.exec.proto.UserBitShared.QueryId.newBuilder(), org.apache.drill.exec.proto.SchemaUserBitShared.QueryId.MERGE)); + + break; + case 3: + builder.addFragments(input.mergeObject(org.apache.drill.exec.proto.BitControl.PlanFragment.newBuilder(), org.apache.drill.exec.proto.SchemaBitControl.PlanFragment.MERGE)); + + break; + case 4: + builder.setError(input.mergeObject(org.apache.drill.exec.proto.UserBitShared.DrillPBError.newBuilder(), org.apache.drill.exec.proto.SchemaUserBitShared.DrillPBError.MERGE)); + + break; + default: + input.handleUnknownField(number, this); + } + } + } + public boolean isInitialized(org.apache.drill.exec.proto.UserProtos.QueryPlanFragments.Builder builder) + { + return builder.isInitialized(); + } + public org.apache.drill.exec.proto.UserProtos.QueryPlanFragments.Builder newMessage() + { + return org.apache.drill.exec.proto.UserProtos.QueryPlanFragments.newBuilder(); + } + public java.lang.String getFieldName(int number) + { + return org.apache.drill.exec.proto.SchemaUserProtos.QueryPlanFragments.getFieldName(number); + } + public int getFieldNumber(java.lang.String name) + { + return org.apache.drill.exec.proto.SchemaUserProtos.QueryPlanFragments.getFieldNumber(name); + } + public java.lang.Class typeClass() + { + return org.apache.drill.exec.proto.UserProtos.QueryPlanFragments.Builder.class; + } + public java.lang.String messageName() + { + return org.apache.drill.exec.proto.UserProtos.QueryPlanFragments.class.getSimpleName(); + } + public java.lang.String messageFullName() + { + return org.apache.drill.exec.proto.UserProtos.QueryPlanFragments.class.getName(); + } + //unused + public void writeTo(com.dyuproject.protostuff.Output output, org.apache.drill.exec.proto.UserProtos.QueryPlanFragments.Builder builder) throws java.io.IOException {} + } + public static java.lang.String getFieldName(int number) + { + switch(number) + { + case 1: return "status"; + case 2: return "queryId"; + case 3: return "fragments"; + case 4: return "error"; + default: return null; + } + } + public static int getFieldNumber(java.lang.String name) + { + java.lang.Integer number = fieldMap.get(name); + return number == null ? 0 : number.intValue(); + } + private static final java.util.HashMap fieldMap = new java.util.HashMap(); + static + { + fieldMap.put("status", 1); + fieldMap.put("queryId", 2); + fieldMap.put("fragments", 3); + fieldMap.put("error", 4); } } http://git-wip-us.apache.org/repos/asf/drill/blob/6bba69d4/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java ---------------------------------------------------------------------- diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java index b0489f7..cafdfdd 100644 --- a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java +++ b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java @@ -133,6 +133,10 @@ public final class UserBitShared { * PHYSICAL = 3; */ PHYSICAL(2, 3), + /** + * EXECUTION = 4; + */ + EXECUTION(3, 4), ; /** @@ -147,6 +151,10 @@ public final class UserBitShared { * PHYSICAL = 3; */ public static final int PHYSICAL_VALUE = 3; + /** + * EXECUTION = 4; + */ + public static final int EXECUTION_VALUE = 4; public final int getNumber() { return value; } @@ -156,6 +164,7 @@ public final class UserBitShared { case 1: return SQL; case 2: return LOGICAL; case 3: return PHYSICAL; + case 4: return EXECUTION; default: return null; } } @@ -20861,31 +20870,32 @@ public final class UserBitShared { "emas\030\003 \001(\003\"J\n\013MetricValue\022\021\n\tmetric_id\030\001" + " \001(\005\022\022\n\nlong_value\030\002 \001(\003\022\024\n\014double_value" + "\030\003 \001(\001*5\n\nRpcChannel\022\017\n\013BIT_CONTROL\020\000\022\014\n" + - "\010BIT_DATA\020\001\022\010\n\004USER\020\002*/\n\tQueryType\022\007\n\003SQ" + - "L\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010PHYSICAL\020\003*\207\001\n\rFragm" + - "entState\022\013\n\007SENDING\020\000\022\027\n\023AWAITING_ALLOCA" + - "TION\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCAN", - "CELLED\020\004\022\n\n\006FAILED\020\005\022\032\n\026CANCELLATION_REQ" + - "UESTED\020\006*\335\005\n\020CoreOperatorType\022\021\n\rSINGLE_" + - "SENDER\020\000\022\024\n\020BROADCAST_SENDER\020\001\022\n\n\006FILTER" + - "\020\002\022\022\n\016HASH_AGGREGATE\020\003\022\r\n\tHASH_JOIN\020\004\022\016\n" + - "\nMERGE_JOIN\020\005\022\031\n\025HASH_PARTITION_SENDER\020\006" + - "\022\t\n\005LIMIT\020\007\022\024\n\020MERGING_RECEIVER\020\010\022\034\n\030ORD" + - "ERED_PARTITION_SENDER\020\t\022\013\n\007PROJECT\020\n\022\026\n\022" + - "UNORDERED_RECEIVER\020\013\022\020\n\014RANGE_SENDER\020\014\022\n" + - "\n\006SCREEN\020\r\022\034\n\030SELECTION_VECTOR_REMOVER\020\016" + - "\022\027\n\023STREAMING_AGGREGATE\020\017\022\016\n\nTOP_N_SORT\020", - "\020\022\021\n\rEXTERNAL_SORT\020\021\022\t\n\005TRACE\020\022\022\t\n\005UNION" + - "\020\023\022\014\n\010OLD_SORT\020\024\022\032\n\026PARQUET_ROW_GROUP_SC" + - "AN\020\025\022\021\n\rHIVE_SUB_SCAN\020\026\022\025\n\021SYSTEM_TABLE_" + - "SCAN\020\027\022\021\n\rMOCK_SUB_SCAN\020\030\022\022\n\016PARQUET_WRI" + - "TER\020\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013TEXT_WRITE" + - "R\020\033\022\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON_SUB_SCAN\020" + - "\035\022\030\n\024INFO_SCHEMA_SUB_SCAN\020\036\022\023\n\017COMPLEX_T" + - "O_JSON\020\037\022\025\n\021PRODUCER_CONSUMER\020 \022\022\n\016HBASE" + - "_SUB_SCAN\020!\022\n\n\006WINDOW\020\"\022\024\n\020NESTED_LOOP_J" + - "OIN\020#\022\021\n\rAVRO_SUB_SCAN\020$B.\n\033org.apache.d", - "rill.exec.protoB\rUserBitSharedH\001" + "\010BIT_DATA\020\001\022\010\n\004USER\020\002*>\n\tQueryType\022\007\n\003SQ" + + "L\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010PHYSICAL\020\003\022\r\n\tEXECUT" + + "ION\020\004*\207\001\n\rFragmentState\022\013\n\007SENDING\020\000\022\027\n\023" + + "AWAITING_ALLOCATION\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FI", + "NISHED\020\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005\022\032\n\026C" + + "ANCELLATION_REQUESTED\020\006*\335\005\n\020CoreOperator" + + "Type\022\021\n\rSINGLE_SENDER\020\000\022\024\n\020BROADCAST_SEN" + + "DER\020\001\022\n\n\006FILTER\020\002\022\022\n\016HASH_AGGREGATE\020\003\022\r\n" + + "\tHASH_JOIN\020\004\022\016\n\nMERGE_JOIN\020\005\022\031\n\025HASH_PAR" + + "TITION_SENDER\020\006\022\t\n\005LIMIT\020\007\022\024\n\020MERGING_RE" + + "CEIVER\020\010\022\034\n\030ORDERED_PARTITION_SENDER\020\t\022\013" + + "\n\007PROJECT\020\n\022\026\n\022UNORDERED_RECEIVER\020\013\022\020\n\014R" + + "ANGE_SENDER\020\014\022\n\n\006SCREEN\020\r\022\034\n\030SELECTION_V" + + "ECTOR_REMOVER\020\016\022\027\n\023STREAMING_AGGREGATE\020\017", + "\022\016\n\nTOP_N_SORT\020\020\022\021\n\rEXTERNAL_SORT\020\021\022\t\n\005T" + + "RACE\020\022\022\t\n\005UNION\020\023\022\014\n\010OLD_SORT\020\024\022\032\n\026PARQU" + + "ET_ROW_GROUP_SCAN\020\025\022\021\n\rHIVE_SUB_SCAN\020\026\022\025" + + "\n\021SYSTEM_TABLE_SCAN\020\027\022\021\n\rMOCK_SUB_SCAN\020\030" + + "\022\022\n\016PARQUET_WRITER\020\031\022\023\n\017DIRECT_SUB_SCAN\020" + + "\032\022\017\n\013TEXT_WRITER\020\033\022\021\n\rTEXT_SUB_SCAN\020\034\022\021\n" + + "\rJSON_SUB_SCAN\020\035\022\030\n\024INFO_SCHEMA_SUB_SCAN" + + "\020\036\022\023\n\017COMPLEX_TO_JSON\020\037\022\025\n\021PRODUCER_CONS" + + "UMER\020 \022\022\n\016HBASE_SUB_SCAN\020!\022\n\n\006WINDOW\020\"\022\024" + + "\n\020NESTED_LOOP_JOIN\020#\022\021\n\rAVRO_SUB_SCAN\020$B", + ".\n\033org.apache.drill.exec.protoB\rUserBitS" + + "haredH\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {