hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From odiache...@apache.org
Subject incubator-hawq git commit: HAWQ-703. Serialize HCatalog Complex Types to plain text (as Hive profile). [Forced Update!]
Date Tue, 03 May 2016 23:54:03 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/HAWQ-703 b1db3991c -> 08007ba97 (forced update)


HAWQ-703. Serialize HCatalog Complex Types to plain text (as Hive profile).


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/08007ba9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/08007ba9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/08007ba9

Branch: refs/heads/HAWQ-703
Commit: 08007ba97e8a09308633fc8e43d937b7d0dd280e
Parents: 6717e7d b357dbc
Author: Oleksandr Diachenko <odiachenko@pivotal.io>
Authored: Mon May 2 13:00:44 2016 -0700
Committer: Oleksandr Diachenko <odiachenko@pivotal.io>
Committed: Tue May 3 16:53:51 2016 -0700

----------------------------------------------------------------------
 depends/libyarn/src/CMakeLists.txt              |   2 +-
 .../libyarn/src/libyarnclient/LibYarnClient.cpp |   6 +-
 pxf/Makefile                                    |   5 +-
 pxf/build.gradle                                |   5 +
 .../java/org/apache/hawq/pxf/api/Metadata.java  |  28 +-
 .../org/apache/hawq/pxf/api/package-info.java   |  23 ++
 .../hawq/pxf/api/utilities/Utilities.java       |  11 +-
 .../org/apache/hawq/pxf/api/MetadataTest.java   |  11 +
 .../hawq/pxf/plugins/hbase/package-info.java    |  23 ++
 .../hawq/pxf/plugins/hdfs/AvroResolver.java     |   4 +-
 .../hawq/pxf/plugins/hdfs/package-info.java     |  23 ++
 .../hawq/pxf/plugins/hive/package-info.java     |  23 ++
 .../hive/utilities/EnumHiveToHawqType.java      |   2 +-
 .../hive/utilities/HiveUtilitiesTest.java       |   8 +-
 .../pxf/service/MetadataResponseFormatter.java  |   1 +
 .../apache/hawq/pxf/service/package-info.java   |  23 ++
 .../pxf/service/utilities/ProtocolData.java     |   2 +-
 src/backend/executor/execMain.c                 |   2 +-
 .../gpopt/translate/CTranslatorUtils.cpp        |  10 +-
 .../communication/rmcomm_RM2RMSEG.c             |  13 +-
 src/backend/utils/misc/etc/gpcheck.cnf          |  48 ++-
 src/bin/psql/describe.c                         |   2 +
 tools/bin/gpcheck                               | 372 ++++++++++++++++++-
 tools/bin/gppylib/gpcheckutil.py                |  17 +-
 tools/bin/hawqpylib/hawqlib.py                  |  18 +
 tools/doc/gpcheck_help                          |  44 ++-
 tools/sbin/gpcheck_hostdump                     |  50 ++-
 27 files changed, 711 insertions(+), 65 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/08007ba9/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
----------------------------------------------------------------------
diff --cc pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
index 4cb7104,4fc510d..9e1c137
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Metadata.java
@@@ -75,22 -71,22 +75,26 @@@ public class Metadata 
       */
      public static class Field {
          private String name;
 -        private String type; // TODO: change to enum
 +        private EnumHawqType type; // field type which PXF exposes
-         private String sourceType; // filed type PXF reads from
++        private String sourceType; // field type PXF reads from
          private String[] modifiers; // type modifiers, optional field
  
-     public Field(String name, EnumHawqType type, String sourceType) {
-         if (StringUtils.isBlank(name) || StringUtils.isBlank(type.getTypeName())
-                 || StringUtils.isBlank(sourceType)) {
-             throw new IllegalArgumentException("Field name, type and source type cannot
be empty");
 -        public Field(String name, String type) {
 -
 -            if (StringUtils.isBlank(name) || StringUtils.isBlank(type)) {
 -                throw new IllegalArgumentException("Field name and type cannot be empty");
++        public Field(String name, EnumHawqType type, String sourceType) {
++            if (StringUtils.isBlank(name)
++                    || type == null
++                    || StringUtils.isBlank(sourceType)) {
++                throw new IllegalArgumentException(
++                        "Field name, type and source type cannot be empty");
+             }
 -
+             this.name = name;
+             this.type = type;
++            this.sourceType = sourceType;
          }
-         this.name = name;
-         this.type = type;
-         this.sourceType = sourceType;
-     }
-     public Field(String name, EnumHawqType type, String sourceType, String[] modifiers)
{
-         this(name, type, sourceType);
-         this.modifiers = modifiers;
+ 
 -        public Field(String name, String type, String[] modifiers) {
 -            this(name, type);
++        public Field(String name, EnumHawqType type, String sourceType,
++                String[] modifiers) {
++            this(name, type, sourceType);
+             this.modifiers = modifiers;
          }
  
          public String getName() {

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/08007ba9/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/MetadataTest.java
----------------------------------------------------------------------
diff --cc pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/MetadataTest.java
index d6e94e1,0000000..327a15b
mode 100644,000000..100644
--- a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/MetadataTest.java
+++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/MetadataTest.java
@@@ -1,49 -1,0 +1,60 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + * 
 + *   http://www.apache.org/licenses/LICENSE-2.0
 + * 
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +
 +package org.apache.hawq.pxf.api;
 +
 +import org.apache.hawq.pxf.api.Metadata;
 +import org.apache.hawq.pxf.api.utilities.EnumHawqType;
++
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.fail;
++
 +import org.junit.Test;
 +
 +public class MetadataTest {
 +
 +    @Test
 +    public void createFieldEmptyNameType() {
 +        try {
 +            Metadata.Field field = new Metadata.Field(null, null, null, null);
 +            fail("Empty name, type and source type shouldn't be allowed.");
 +        } catch (IllegalArgumentException e) {
 +            assertEquals("Field name, type and source type cannot be empty", e.getMessage());
 +        }
 +    }
 +
 +    @Test
++    public void createFieldNullType() {
++        try {
++            Metadata.Field field = new Metadata.Field("col1", null, "string");
++            fail("Empty name, type and source type shouldn't be allowed.");
++        } catch (IllegalArgumentException e) {
++            assertEquals("Field name, type and source type cannot be empty", e.getMessage());
++        }
++    }
++    @Test
 +    public void createItemEmptyNameType() {
 +        try {
 +            Metadata.Item item = new Metadata.Item(null, null);
 +            fail("Empty item name and path shouldn't be allowed.");
 +        } catch (IllegalArgumentException e) {
 +            assertEquals("Item or path name cannot be empty", e.getMessage());
 +        }
 +    }
 +}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/08007ba9/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/EnumHiveToHawqType.java
----------------------------------------------------------------------
diff --cc pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/EnumHiveToHawqType.java
index 13d402c,0000000..a9ff877
mode 100644,000000..100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/EnumHiveToHawqType.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/EnumHiveToHawqType.java
@@@ -1,112 -1,0 +1,112 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + * 
 + *   http://www.apache.org/licenses/LICENSE-2.0
 + * 
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +
 +package org.apache.hawq.pxf.plugins.hive.utilities;
 +
 +import org.apache.hawq.pxf.api.utilities.EnumHawqType;
 +import org.apache.hawq.pxf.api.UnsupportedTypeException;
 +
 +/**
 + * 
 + * Hive types, which are supported by plugin, mapped to HAWQ's types
 + * @see EnumHawqType
 + */
 +public enum EnumHiveToHawqType {
 +
 +    TinyintType("tinyint", EnumHawqType.Int2Type),
 +    SmallintType("smallint", EnumHawqType.Int2Type),
 +    IntType("int", EnumHawqType.Int4Type),
 +    BigintType("bigint", EnumHawqType.Int8Type),
 +    BooleanType("boolean", EnumHawqType.BoolType),
 +    FloatType("float", EnumHawqType.Float4Type),
 +    DoubleType("double", EnumHawqType.Float8Type),
 +    StringType("string", EnumHawqType.TextType),
 +    BinaryType("binary", EnumHawqType.ByteaType),
 +    TimestampType("timestamp", EnumHawqType.TimestampType),
 +    DateType("date", EnumHawqType.DateType),
 +    DecimalType("decimal", EnumHawqType.NumericType, "[(,)]"),
 +    VarcharType("varchar", EnumHawqType.VarcharType, "[(,)]"),
 +    CharType("char", EnumHawqType.BpcharType, "[(,)]"),
 +    ArrayType("array", EnumHawqType.TextType, "[<,>]"),
 +    MapType("map", EnumHawqType.TextType, "[<,>]"),
 +    StructType("struct", EnumHawqType.TextType, "[<,>]"),
 +    UnionType("uniontype", EnumHawqType.TextType, "[<,>]");
 +
 +    private String typeName;
 +    private EnumHawqType hawqType;
 +    private String splitExpression;
 +
 +    EnumHiveToHawqType(String typeName, EnumHawqType hawqType) {
 +        this.typeName = typeName;
 +        this.hawqType = hawqType;
 +    }
 +
 +    EnumHiveToHawqType(String typeName, EnumHawqType hawqType, String splitExpression) {
 +        this(typeName, hawqType);
 +        this.splitExpression = splitExpression;
 +    }
 +
 +    /**
 +     * 
 +     * @return name of type
 +     */
 +    public String getTypeName() {
 +        return this.typeName;
 +    }
 +
 +    /**
 +     * 
 +     * @return corresponding HAWQ type
 +     */
 +    public EnumHawqType getHawqType() {
 +        return this.hawqType;
 +    }
 +
 +    /**
 +     * 
 +     * @return split by expression
 +     */
 +    public String getSplitExpression() {
 +        return this.splitExpression;
 +    }
 +
 +    /**
 +     * Returns Hive to HAWQ type mapping entry for given Hive type 
 +     * 
 +     * @param hiveType full Hive type with modifiers, for example - decimal(10, 0), char(5),
binary, array<string>, map<string,float> etc
 +     * @return corresponding Hive to HAWQ type mapping entry
 +     * @throws UnsupportedTypeException if there is no corresponding HAWQ type
 +     */
 +    public static EnumHiveToHawqType getHiveToHawqType(String hiveType) {
 +        for (EnumHiveToHawqType t : values()) {
 +            String hiveTypeName = hiveType;
 +            if (t.getSplitExpression() != null) {
 +                String[] tokens = hiveType.split(t.getSplitExpression());
 +                hiveTypeName = tokens[0];
 +            }
 +
-             if (t.getTypeName().equals(hiveTypeName)) {
++            if (t.getTypeName().toLowerCase().equals(hiveTypeName.toLowerCase())) {
 +                return t;
 +            }
 +        }
 +        throw new UnsupportedTypeException("Unable to map Hive's type: "
 +                + hiveType + " to HAWQ's type");
 +    }
 +
 +}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/08007ba9/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
----------------------------------------------------------------------
diff --cc pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
index 1054d0d,466dedb..e9b024a
--- a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
+++ b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
@@@ -54,12 -54,6 +54,13 @@@ public class HiveUtilitiesTest 
          {"char(40)", "bpchar", "40"},
      };
  
 +    static String[][] complexTypes = {
-         {"array<string>", "text"},
-         {"map<string,float>", "text"},
-         {"struct<street:string,city:string,state:string,zip:int>", "text"},
++        {"ArraY<string>", "text"},
++        {"MaP<stRing, float>", "text"},
++        {"Struct<street:string, city:string, state:string, zip:int>", "text"},
++        {"UnionType<array<string>, string,int>", "text"}
 +    };
 +
      @Test
      public void mapHiveTypeUnsupported() throws Exception {
  
@@@ -157,37 -152,6 +158,38 @@@
      }
  
      @Test
 +    public void mapHiveTypeInvalidModifiers() throws Exception {
 +        String badHiveType = "decimal(abc, xyz)";
 +        hiveColumn = new FieldSchema("numericColumn", badHiveType, null);
 +        try {
 +            HiveUtilities.mapHiveType(hiveColumn);
 +            fail("should fail with bad modifiers error");
 +        } catch (UnsupportedTypeException e) {
 +            String errorMsg = "HAWQ does not support type " + badHiveType + " (Field numericColumn),
modifiers should be integers";
 +            assertEquals(errorMsg, e.getMessage());
 +        }
 +    }
 +
 +    @Test
 +    public void mapHiveTypeComplex() throws Exception {
 +        /*
 +         * array<dataType> -> text
 +         * map<keyDataType, valueDataType> -> text
 +         * struct<fieldName1:dataType, ..., fieldNameN:dataType> -> text
++         * uniontype<...> -> text
 +         */
 +        for (String[] line: complexTypes) {
 +            String hiveType = line[0];
 +            String expectedType = line[1];
 +            hiveColumn = new FieldSchema("field" + hiveType, hiveType, null);
 +            Metadata.Field result = HiveUtilities.mapHiveType(hiveColumn);
 +            assertEquals("field" + hiveType, result.getName());
 +            assertEquals(expectedType, result.getType().getTypeName());
 +            assertNull(result.getModifiers());
 +        }
 +    }
 +
 +    @Test
      public void parseTableQualifiedNameNoDbName() throws Exception {
          String name = "orphan";
          tblDesc = HiveUtilities.extractTableFromName(name);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/08007ba9/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/08007ba9/src/bin/psql/describe.c
----------------------------------------------------------------------


Mime
View raw message