hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shiv...@apache.org
Subject incubator-hawq git commit: HAWQ-1490. Added Demo PXF classes for standalone testing
Date Wed, 12 Jul 2017 18:08:29 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 80e8322c2 -> 2222f4b4a


HAWQ-1490. Added Demo PXF classes for standalone testing


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2222f4b4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2222f4b4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2222f4b4

Branch: refs/heads/master
Commit: 2222f4b4afa2fa8f3f862fb22aabbacc77e990f5
Parents: 80e8322
Author: John Gaskin <johntgaskin@gmail.com>
Authored: Wed Jul 12 11:08:16 2017 -0700
Committer: shivzone <shivram.mani@gmail.com>
Committed: Wed Jul 12 11:08:16 2017 -0700

----------------------------------------------------------------------
 .../hawq/pxf/api/examples/DemoAccessor.java     | 98 ++++++++++++++++++++
 .../hawq/pxf/api/examples/DemoFragmenter.java   | 59 ++++++++++++
 .../hawq/pxf/api/examples/DemoResolver.java     | 67 +++++++++++++
 .../hawq/pxf/api/examples/DemoTextResolver.java | 63 +++++++++++++
 .../apache/hawq/pxf/api/DemoAccessorTest.java   | 79 ++++++++++++++++
 .../apache/hawq/pxf/api/DemoResolverTest.java   | 70 ++++++++++++++
 6 files changed, 436 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoAccessor.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoAccessor.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoAccessor.java
new file mode 100644
index 0000000..bec3146
--- /dev/null
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoAccessor.java
@@ -0,0 +1,98 @@
+package org.apache.hawq.pxf.api.examples;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.hawq.pxf.api.OneRow;
+import org.apache.hawq.pxf.api.ReadAccessor;
+import org.apache.hawq.pxf.api.utilities.InputData;
+import org.apache.hawq.pxf.api.utilities.Plugin;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Internal interface that would defined the access to a file on HDFS, but in
+ * this case contains the data required.
+ *
+ * Demo implementation
+ */
+public class DemoAccessor extends Plugin implements ReadAccessor {
+
+    private static final Log LOG = LogFactory.getLog(DemoAccessor.class);
+    private int rowNumber;
+    private int fragmentNumber;
+    private static int NUM_ROWS = 2;
+
+    /**
+     * Constructs a DemoAccessor
+     *
+     * @param metaData the InputData
+     */
+    public DemoAccessor(InputData metaData) {
+        super(metaData);
+    }
+    @Override
+    public boolean openForRead() throws Exception {
+        /* no-op, because this plugin doesn't read a file. */
+        return true;
+    }
+
+    /**
+     * Read the next record
+     * The record contains as many fields as defined by the DDL schema.
+     *
+     * @return one row which corresponds to one record
+     */
+    @Override
+    public OneRow readNextObject() throws Exception {
+        /* return next row , <key=fragmentNo.rowNo, val=rowNo,text,fragmentNo>*/
+        /* check for EOF */
+        if (fragmentNumber > 0)
+            return null; /* signal EOF, close will be called */
+        int fragment = inputData.getDataFragment();
+        String fragmentMetadata = new String(inputData.getFragmentMetadata());
+        int colCount = inputData.getColumns();
+
+        /* generate row with (colCount) columns */
+        StringBuilder colValue = new StringBuilder(fragmentMetadata + " row" + (rowNumber+1));
+        for(int colIndex=1; colIndex<colCount; colIndex++) {
+            colValue.append(",").append("value" + colIndex);
+        }
+        OneRow row = new OneRow(fragment + "." + rowNumber, colValue.toString());
+
+        /* advance */
+        rowNumber += 1;
+        if (rowNumber == NUM_ROWS) {
+            rowNumber = 0;
+            fragmentNumber += 1;
+        }
+
+        /* return data */
+        return row;
+    }
+
+    /**
+     * close the reader. no action here
+     *
+     */
+    @Override
+    public void closeForRead() throws Exception {
+        /* Demo close doesn't do anything */
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoFragmenter.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoFragmenter.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoFragmenter.java
new file mode 100644
index 0000000..685fcf2
--- /dev/null
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoFragmenter.java
@@ -0,0 +1,59 @@
+package org.apache.hawq.pxf.api.examples;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.hawq.pxf.api.Fragmenter;
+import org.apache.hawq.pxf.api.Fragment;
+import org.apache.hawq.pxf.api.utilities.InputData;
+import java.util.List;
+
+/**
+ * Class that defines the splitting of a data resource into fragments that can be processed
in parallel
+ * getFragments() returns the fragments information of a given path (source name and location
of each fragment).
+ *
+ * Demo implementation
+ */
+
+public class DemoFragmenter extends Fragmenter{
+    /**
+     * Constructs the DemoFragmenter
+     *
+     * @param metaData
+     */
+    public DemoFragmenter(InputData metaData) {
+        super(metaData);
+    }
+
+    /**
+     * Provide metadata for each data partition of the given datasource
+     *
+     * @return list of fragments
+     */
+    @Override
+    public List<Fragment> getFragments() throws Exception {
+        String localhostname = java.net.InetAddress.getLocalHost().getHostName();
+        String[] localHosts = new String[]{localhostname, localhostname};
+        fragments.add(new Fragment(inputData.getDataSource() + ".1", localHosts, "fragment1".getBytes()));
+        fragments.add(new Fragment(inputData.getDataSource() + ".2", localHosts, "fragment2".getBytes()));
+        fragments.add(new Fragment(inputData.getDataSource() + ".3", localHosts, "fragment3".getBytes()));
+        return fragments;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoResolver.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoResolver.java
new file mode 100644
index 0000000..8780a19
--- /dev/null
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoResolver.java
@@ -0,0 +1,67 @@
+package org.apache.hawq.pxf.api.examples;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.hawq.pxf.api.OneField;
+import org.apache.hawq.pxf.api.OneRow;
+import org.apache.hawq.pxf.api.ReadResolver;
+import org.apache.hawq.pxf.api.utilities.InputData;
+import org.apache.hawq.pxf.api.utilities.Plugin;
+import java.util.LinkedList;
+import java.util.List;
+import static org.apache.hawq.pxf.api.io.DataType.INTEGER;
+import static org.apache.hawq.pxf.api.io.DataType.VARCHAR;
+
+/**
+ * Class that defines the deserializtion of one record brought from the external input data.
+ *
+ * Demo implementation that returns record custom format
+ */
+public class DemoResolver extends Plugin implements ReadResolver {
+    /**
+     * Constructs the DemoResolver
+     *
+     * @param metaData
+     */
+    public DemoResolver(InputData metaData) {
+        super(metaData);
+    }
+
+    /**
+     * Read the next record
+     * The record contains as many fields as defined by the DDL schema.
+     *
+     * @param row one record
+     * @return list of fields or columns
+     */
+    @Override
+    public List<OneField> getFields(OneRow row) throws Exception {
+        List<OneField> output = new LinkedList<OneField>();
+        Object data = row.getData();
+
+        /* break up the row into fields */
+        String[] fields = ((String) data).split(",");
+        for(int colIndex=0; colIndex<fields.length; colIndex++) {
+            output.add(new OneField(VARCHAR.getOID(), fields[colIndex]));
+        }
+
+        return output;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoTextResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoTextResolver.java
b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoTextResolver.java
new file mode 100644
index 0000000..772550e
--- /dev/null
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoTextResolver.java
@@ -0,0 +1,63 @@
+package org.apache.hawq.pxf.api.examples;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.hawq.pxf.api.OneField;
+import org.apache.hawq.pxf.api.OneRow;
+import org.apache.hawq.pxf.api.ReadResolver;
+import org.apache.hawq.pxf.api.utilities.InputData;
+import org.apache.hawq.pxf.api.utilities.Plugin;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import static org.apache.hawq.pxf.api.io.DataType.INTEGER;
+import static org.apache.hawq.pxf.api.io.DataType.VARCHAR;
+
+/**
+ * Class that defines the deserializtion of one record brought from the external input data.
+ *
+ * Demo implementation of resolver that returns text format
+ */
+public class DemoTextResolver extends Plugin implements ReadResolver {
+    /**
+     * Constructs the DemoResolver
+     *
+     * @param metaData
+     */
+    public DemoTextResolver(InputData metaData) {
+        super(metaData);
+    }
+
+    /**
+     * Read the next record
+     * The record contains as many fields as defined by the DDL schema.
+     *
+     * @param row one record
+     * @return the first column contains the entire text data
+     */
+    @Override
+    public List<OneField> getFields(OneRow row) throws Exception {
+        List<OneField> output = new LinkedList<OneField>();
+        Object data = row.getData();
+        output.add(new OneField(VARCHAR.getOID(), data));
+        return output;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoAccessorTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoAccessorTest.java b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoAccessorTest.java
new file mode 100755
index 0000000..36cbf21
--- /dev/null
+++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoAccessorTest.java
@@ -0,0 +1,79 @@
+package org.apache.hawq.pxf.api;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+
+
+import org.apache.hawq.pxf.api.examples.DemoAccessor;
+import org.apache.hawq.pxf.api.utilities.InputData;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({DemoAccessor.class}) // Enables mocking 'new' calls
+
+public class DemoAccessorTest {
+
+    @Mock InputData inputData;
+    DemoAccessor accessor;
+
+    @Before
+    public void setup() throws Exception {
+        accessor = new DemoAccessor(inputData);
+    }
+
+    @Test
+    public void testRowsWithSingleColumn() throws Exception {
+
+        when(inputData.getDataFragment()).thenReturn(0);
+        when(inputData.getFragmentMetadata()).thenReturn("fragment1".getBytes(), "fragment1".getBytes());
+        when(inputData.getColumns()).thenReturn(1);
+
+        int numRows = 2;
+        for (int i = 0; i < numRows; i++) {
+            OneRow row = accessor.readNextObject();
+            assertEquals(row.toString(),  "OneRow:0." + i + "->fragment1 row" + (i+1));
+        }
+        assertNull(accessor.readNextObject());
+    }
+
+    @Test
+    public void testRowsWithMultipleColumns() throws Exception {
+
+        when(inputData.getDataFragment()).thenReturn(0);
+        when(inputData.getFragmentMetadata()).thenReturn("fragment1".getBytes(), "fragment1".getBytes());
+        when(inputData.getColumns()).thenReturn(3);
+
+        int numRows = 2;
+        for (int i = 0; i < numRows; i++) {
+            OneRow row = accessor.readNextObject();
+            assertEquals(row.toString(),  "OneRow:0." + i + "->fragment1 row" + (i+1)
+ ",value1,value2");
+        }
+        assertNull(accessor.readNextObject());
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoResolverTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoResolverTest.java b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoResolverTest.java
new file mode 100755
index 0000000..dd22c87
--- /dev/null
+++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoResolverTest.java
@@ -0,0 +1,70 @@
+package org.apache.hawq.pxf.api;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.hawq.pxf.api.examples.DemoAccessor;
+import org.apache.hawq.pxf.api.examples.DemoResolver;
+import org.apache.hawq.pxf.api.examples.DemoTextResolver;
+import org.apache.hawq.pxf.api.utilities.InputData;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({DemoAccessor.class}) // Enables mocking 'new' calls
+
+public class DemoResolverTest {
+
+    @Mock InputData inputData;
+    DemoResolver customResolver;
+    DemoTextResolver textResolver;
+    OneRow row;
+
+    @Before
+    public void setup() throws Exception {
+        customResolver = new DemoResolver(inputData);
+        textResolver = new DemoTextResolver(inputData);
+        row = new OneRow("0.0","value1,value2");
+    }
+
+    @Test
+    public void testCustomData() throws Exception {
+
+        List<OneField> output = customResolver.getFields(row);
+        assertEquals("value1", output.get(0).toString());
+        assertEquals("value2", output.get(1).toString());
+    }
+
+    @Test
+    public void testTextData() throws Exception {
+
+        List<OneField> output = textResolver.getFields(row);
+        assertEquals("value1,value2", output.get(0).toString());
+
+    }
+}


Mime
View raw message