hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shiv...@apache.org
Subject incubator-hawq git commit: HAWQ-191. Remove Analyzer plugin from PXF
Date Mon, 14 Dec 2015 22:33:13 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 59264a537 -> de474ea5b


HAWQ-191. Remove Analyzer plugin from PXF


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/de474ea5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/de474ea5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/de474ea5

Branch: refs/heads/master
Commit: de474ea5b447610ad7e8fcfb2e9a0bfeda84a479
Parents: 59264a5
Author: Shivram Mani <shivram.mani@gmail.com>
Authored: Mon Dec 14 14:33:04 2015 -0800
Committer: Shivram Mani <shivram.mani@gmail.com>
Committed: Mon Dec 14 14:33:04 2015 -0800

----------------------------------------------------------------------
 pxf/build.gradle                                |   2 +-
 .../java/org/apache/hawq/pxf/api/Analyzer.java  |  57 ------
 .../org/apache/hawq/pxf/api/AnalyzerStats.java  | 117 ------------
 .../hawq/pxf/api/utilities/InputData.java       |  12 --
 .../apache/hawq/pxf/api/utilities/Plugin.java   |   2 +-
 .../hawq/pxf/plugins/hdfs/HdfsAnalyzer.java     | 184 -------------------
 .../hawq/pxf/service/AnalyzerFactory.java       |  37 ----
 .../hawq/pxf/service/rest/AnalyzerResource.java | 122 ------------
 .../pxf/service/rest/InvalidPathResource.java   |  50 ++++-
 .../pxf/service/utilities/ProtocolData.java     |   2 -
 .../src/main/resources/pxf-profiles-default.xml |   4 -
 11 files changed, 49 insertions(+), 540 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/de474ea5/pxf/build.gradle
----------------------------------------------------------------------
diff --git a/pxf/build.gradle b/pxf/build.gradle
index f50a6a3..c7b5669 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -218,7 +218,7 @@ project('pxf-service') {
 
 project('pxf-hdfs') {
     dependencies {
-        compile(project(':pxf-service')) //Yikes, HdfsAnalyzer is directly accessing the
bridge
+        compile(project(':pxf-service'))
         compile 'org.apache.avro:avro-mapred:1.7.4'
         compile "org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion"
     }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/de474ea5/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Analyzer.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Analyzer.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Analyzer.java
deleted file mode 100644
index 529de30..0000000
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Analyzer.java
+++ /dev/null
@@ -1,57 +0,0 @@
-package org.apache.hawq.pxf.api;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import org.apache.hawq.pxf.api.utilities.InputData;
-import org.apache.hawq.pxf.api.utilities.Plugin;
-
-/**
- * Abstract class that defines getting statistics for ANALYZE.
- * {@link #getEstimatedStats} returns statistics for a given path
- * (block size, number of blocks, number of tuples).
- * Used when calling ANALYZE on a PXF external table, to get
- * table's statistics that are used by the optimizer to plan queries.
- */
-public abstract class Analyzer extends Plugin {
-    /**
-     * Constructs an Analyzer.
-     *
-     * @param inputData the input data
-     */
-    public Analyzer(InputData inputData) {
-        super(inputData);
-    }
-
-    /**
-     * Gets the statistics for a given path.
-     * NOTE: It is highly recommended to implement an extremely fast logic
-     * that returns *estimated* statistics. Scanning all the data for exact
-     * statistics is considered bad practice.
-     *
-     * @param data the data source name (e.g, file, dir, wildcard, table name).
-     * @return AnalyzerStats the data statistics in json format.
-     * @throws Exception if fails to get stats
-     */
-    public AnalyzerStats getEstimatedStats(String data) throws Exception {
-        /* Return default values */
-        return new AnalyzerStats();
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/de474ea5/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/AnalyzerStats.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/AnalyzerStats.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/AnalyzerStats.java
deleted file mode 100644
index 8101af0..0000000
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/AnalyzerStats.java
+++ /dev/null
@@ -1,117 +0,0 @@
-package org.apache.hawq.pxf.api;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import org.codehaus.jackson.map.ObjectMapper;
-
-import java.io.IOException;
-
-/**
- * AnalyzerStats holds size statistics for a given path.
- */
-public class AnalyzerStats {
-
-    private static final long DEFAULT_BLOCK_SIZE = 67108864L; // 64MB (in bytes)
-    private static final long DEFAULT_NUMBER_OF_BLOCKS = 1L;
-    private static final long DEFAULT_NUMBER_OF_TUPLES = 1000000L;
-
-    private long blockSize; // block size (in bytes)
-    private long numberOfBlocks; // number of blocks
-    private long numberOfTuples; // number of tuples
-
-    /**
-     * Constructs an AnalyzerStats.
-     *
-     * @param blockSize block size (in bytes)
-     * @param numberOfBlocks number of blocks
-     * @param numberOfTuples number of tuples
-     */
-    public AnalyzerStats(long blockSize, long numberOfBlocks,
-                         long numberOfTuples) {
-        this.setBlockSize(blockSize);
-        this.setNumberOfBlocks(numberOfBlocks);
-        this.setNumberOfTuples(numberOfTuples);
-    }
-
-    /** Constructs an AnalyzerStats with the default values */
-    public AnalyzerStats() {
-        this(DEFAULT_BLOCK_SIZE, DEFAULT_NUMBER_OF_BLOCKS,
-                DEFAULT_NUMBER_OF_TUPLES);
-    }
-
-    /**
-     * Given an AnalyzerStats, serialize it in JSON to be used as the result
-     * string for HAWQ. An example result is as follows:
-     * {"PXFDataSourceStats":{"blockSize"
-     * :67108864,"numberOfBlocks":1,"numberOfTuples":5}}
-     *
-     * @param stats the data to be serialized
-     * @return the result in json format
-     * @throws IOException if converting to JSON format failed
-     */
-    public static String dataToJSON(AnalyzerStats stats) throws IOException {
-        ObjectMapper mapper = new ObjectMapper();
-        // mapper serializes all members of the class by default
-        return "{\"PXFDataSourceStats\":" + mapper.writeValueAsString(stats)
-                + "}";
-    }
-
-    /**
-     * Given a stats structure, convert it to be readable. Intended for
-     * debugging purposes only.
-     *
-     * @param stats the data to be stringify
-     * @param datapath the data path part of the original URI (e.g., table name,
-     *            *.csv, etc.)
-     * @return the stringified data
-     */
-    public static String dataToString(AnalyzerStats stats, String datapath) {
-        return "Statistics information for \"" + datapath + "\" "
-                + " Block Size: " + stats.blockSize + ", Number of blocks: "
-                + stats.numberOfBlocks + ", Number of tuples: "
-                + stats.numberOfTuples;
-    }
-
-    public long getBlockSize() {
-        return blockSize;
-    }
-
-    private void setBlockSize(long blockSize) {
-        this.blockSize = blockSize;
-    }
-
-    public long getNumberOfBlocks() {
-        return numberOfBlocks;
-    }
-
-    private void setNumberOfBlocks(long numberOfBlocks) {
-        this.numberOfBlocks = numberOfBlocks;
-    }
-
-    public long getNumberOfTuples() {
-        return numberOfTuples;
-    }
-
-    private void setNumberOfTuples(long numberOfTuples) {
-        this.numberOfTuples = numberOfTuples;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/de474ea5/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
index 4f95ed9..cbec312 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
@@ -45,7 +45,6 @@ public class InputData {
     protected String dataSource;
     protected String accessor;
     protected String resolver;
-    protected String analyzer;
     protected String fragmenter;
     protected String remoteLogin;
     protected String remoteSecret;
@@ -97,7 +96,6 @@ public class InputData {
         this.accessor = copy.accessor;
         this.resolver = copy.resolver;
         this.fragmenter = copy.fragmenter;
-        this.analyzer = copy.analyzer;
         this.remoteLogin = copy.remoteLogin;
         this.remoteSecret = copy.remoteSecret;
         this.threadSafe = copy.threadSafe;
@@ -274,16 +272,6 @@ public class InputData {
     }
 
     /**
-     * Returns the ClassName for the java class that was defined as Analyzer or
-     * null if no analyzer was defined.
-     *
-     * @return class name for Analyzer or null
-     */
-    public String getAnalyzer() {
-        return analyzer;
-    }
-
-    /**
      * Returns the contents of pxf_remote_service_login set in Hawq. Should the
      * user set it to an empty string this function will return null.
      *

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/de474ea5/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Plugin.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Plugin.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Plugin.java
index 18707c0..807c2db 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Plugin.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Plugin.java
@@ -22,7 +22,7 @@ package org.apache.hawq.pxf.api.utilities;
 
 
 /**
- * Base class for all plugin types (Accessor, Resolver, Fragmenter, Analyzer, ...).
+ * Base class for all plugin types (Accessor, Resolver, Fragmenter, ...).
  * Manages the meta data.
  */
 public class Plugin {

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/de474ea5/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/HdfsAnalyzer.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/HdfsAnalyzer.java
b/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/HdfsAnalyzer.java
deleted file mode 100644
index 08bfa15..0000000
--- a/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/HdfsAnalyzer.java
+++ /dev/null
@@ -1,184 +0,0 @@
-package org.apache.hawq.pxf.plugins.hdfs;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import org.apache.hawq.pxf.api.Analyzer;
-import org.apache.hawq.pxf.api.AnalyzerStats;
-import org.apache.hawq.pxf.api.ReadAccessor;
-import org.apache.hawq.pxf.api.utilities.InputData;
-import org.apache.hawq.pxf.service.ReadBridge;
-import org.apache.hawq.pxf.plugins.hdfs.utilities.HdfsUtilities;
-import org.apache.hawq.pxf.plugins.hdfs.utilities.PxfInputFormat;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-
-import java.io.IOException;
-import java.util.ArrayList;
-
-/**
- * Analyzer class for HDFS data resources
- *
- * Given an HDFS data source (a file, directory, or wild card pattern) return
- * statistics about it (number of blocks, number of tuples, etc.)
- */
-public class HdfsAnalyzer extends Analyzer {
-    private JobConf jobConf;
-    private FileSystem fs;
-    private Log Log;
-
-    /**
-     * Constructs an HdfsAnalyzer object.
-     *
-     * @param inputData all input parameters coming from the client
-     * @throws IOException if HDFS file system cannot be retrieved
-     */
-    public HdfsAnalyzer(InputData inputData) throws IOException {
-        super(inputData);
-        Log = LogFactory.getLog(HdfsAnalyzer.class);
-
-        jobConf = new JobConf(new Configuration(), HdfsAnalyzer.class);
-        fs = FileSystem.get(jobConf);
-    }
-
-    /**
-     * Collects a number of basic statistics based on an estimate. Statistics
-     * are: number of records, number of hdfs blocks and hdfs block size.
-     *
-     * @param datapath path is a data source URI that can appear as a file name,
-     *            a directory name or a wildcard pattern
-     * @return statistics in JSON format
-     * @throws Exception if path is wrong, its metadata cannot be retrieved from
-     *             file system, or if scanning the first block using the
-     *             accessor failed
-     */
-    @Override
-    public AnalyzerStats getEstimatedStats(String datapath) throws Exception {
-        long blockSize = 0;
-        long numberOfBlocks;
-        long dataSize = 0;
-        Path path = new Path(HdfsUtilities.absoluteDataPath(datapath));
-
-        ArrayList<InputSplit> splits = getSplits(path);
-
-        for (InputSplit split : splits) {
-            FileSplit fsp = (FileSplit) split;
-            dataSize += fsp.getLength();
-            if (blockSize == 0) {
-                Path filePath = fsp.getPath();
-                FileStatus fileStatus = fs.getFileStatus(filePath);
-                if (fileStatus.isFile()) {
-                    blockSize = fileStatus.getBlockSize();
-                }
-            }
-        }
-
-        // if no file is in path (only dirs), get default block size
-        if (blockSize == 0) {
-            blockSize = fs.getDefaultBlockSize(path);
-        }
-        numberOfBlocks = splits.size();
-
-        /*
-         * The estimate of the number of tuples in table is based on the
-         * actual number of tuples in the first block, multiplied by its
-         * size compared to the size of the whole data to be read.
-         * The calculation:
-         * Ratio of tuples to size = number of tuples in first block / first block size.
-         * Total of tuples = ratio * number of blocks * total block size.
-         */
-        long numberOfTuplesInBlock = getNumberOfTuplesInBlock(splits);
-        long numberOfTuples = 0;
-        if (!splits.isEmpty()) {
-            long blockLength = splits.get(0).getLength();
-            numberOfTuples = (long) Math.floor((((double) numberOfTuplesInBlock / blockLength)
* (dataSize)));
-        }
-        // AnalyzerStats stats = new AnalyzerStats(blockSize, numberOfBlocks,
-        AnalyzerStats stats = new AnalyzerStats(blockSize, numberOfBlocks,
-                numberOfTuples);
-
-        // print files size to log when in debug level
-        Log.debug(AnalyzerStats.dataToString(stats, path.toString()));
-
-        return stats;
-    }
-
-    /**
-     * Calculates the number of tuples in a split (block). Reads one block from
-     * HDFS. Exception during reading will filter upwards and handled in
-     * AnalyzerResource
-     */
-    private long getNumberOfTuplesInBlock(ArrayList<InputSplit> splits)
-            throws Exception {
-        long tuples = -1; /* default - if we are not able to read data */
-        ReadAccessor accessor;
-
-        if (splits.isEmpty()) {
-            return 0;
-        }
-
-        /*
-         * metadata information includes: file split's start, length and hosts
-         * (locations).
-         */
-        FileSplit firstSplit = (FileSplit) splits.get(0);
-        byte[] fragmentMetadata = HdfsUtilities.prepareFragmentMetadata(firstSplit);
-        inputData.setFragmentMetadata(fragmentMetadata);
-        inputData.setDataSource(firstSplit.getPath().toUri().getPath());
-        accessor = ReadBridge.getFileAccessor(inputData);
-
-        if (accessor.openForRead()) {
-            tuples = 0;
-            while (accessor.readNextObject() != null) {
-                tuples++;
-            }
-
-            accessor.closeForRead();
-        }
-        Log.debug("number of tuples in first block: " + tuples);
-
-        return tuples;
-    }
-
-    private ArrayList<InputSplit> getSplits(Path path) throws IOException {
-        PxfInputFormat fformat = new PxfInputFormat();
-        PxfInputFormat.setInputPaths(jobConf, path);
-        InputSplit[] splits = fformat.getSplits(jobConf, 1);
-        ArrayList<InputSplit> result = new ArrayList<InputSplit>();
-
-        // remove empty splits
-        if (splits != null) {
-            for (InputSplit split : splits) {
-                if (split.getLength() > 0) {
-                    result.add(split);
-                }
-            }
-        }
-
-        return result;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/de474ea5/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/AnalyzerFactory.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/AnalyzerFactory.java
b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/AnalyzerFactory.java
deleted file mode 100644
index ec95496..0000000
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/AnalyzerFactory.java
+++ /dev/null
@@ -1,37 +0,0 @@
-package org.apache.hawq.pxf.service;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import org.apache.hawq.pxf.api.Analyzer;
-import org.apache.hawq.pxf.api.utilities.InputData;
-import org.apache.hawq.pxf.service.utilities.Utilities;
-
-/*
- * Factory class for creation of Analyzer objects. The actual Analyzer object is "hidden"
behind
- * an Analyzer abstract class which is returned by the AnalyzerFactory.
- */
-public class AnalyzerFactory {
-    static public Analyzer create(InputData inputData) throws Exception {
-    	String analyzerName = inputData.getAnalyzer();
-
-        return (Analyzer) Utilities.createAnyInstance(InputData.class, analyzerName, inputData);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/de474ea5/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/AnalyzerResource.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/AnalyzerResource.java
b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/AnalyzerResource.java
deleted file mode 100644
index e41d81d..0000000
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/AnalyzerResource.java
+++ /dev/null
@@ -1,122 +0,0 @@
-package org.apache.hawq.pxf.service.rest;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import org.apache.hawq.pxf.api.Analyzer;
-import org.apache.hawq.pxf.api.AnalyzerStats;
-import org.apache.hawq.pxf.service.AnalyzerFactory;
-import org.apache.hawq.pxf.service.utilities.ProtocolData;
-import org.apache.hawq.pxf.service.utilities.SecuredHDFS;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import javax.servlet.ServletContext;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import java.io.IOException;
-import java.util.Map;
-
-/*
- * Class enhances the API of the WEBHDFS REST server.
- * Returns the data fragments that a data resource is made of, enabling parallel processing
of the data resource.
- * Example for querying API ANALYZER from a web client
- * curl -i "http://localhost:50070/pxf/v2/Analyzer/getEstimatedStats?path=/dir1/dir2/*txt"
- * /pxf/ is made part of the path when there is a webapp by that name in tcServer.
- */
-@Path("/" + Version.PXF_PROTOCOL_VERSION + "/Analyzer/")
-public class AnalyzerResource extends RestResource {
-    private Log Log;
-
-
-    public AnalyzerResource() throws IOException {
-        Log = LogFactory.getLog(AnalyzerResource.class);
-    }
-
-    /*
-     * Returns estimated statistics for the given path (data source).
-     * Example for querying API ANALYZER from a web client
-     * curl -i "http://localhost:50070/pxf/v2/Analyzer/getEstimatedStats?path=/dir1/dir2/*txt"
-     * A default answer, unless an analyzer implements getEstimatedStats, would be:
-     * {"PXFDataSourceStats":[{"blockSize":67108864,"numberOfBlocks":1000,"numberOfTuples":1000000}]}
-     * Currently only HDFS is implemented to calculate the block size and block number,
-     * and returns -1 for number of tuples.
-     * Example:
-     * {"PXFDataSourceStats":[{"blockSize":67108864,"numberOfBlocks":3,"numberOfTuples":-1}]}
-     *
-     * @param servletContext Servlet context contains attributes required by SecuredHDFS
-     * @param headers Holds HTTP headers from request
-     * @param path Holds URI path option used in this request
-     */
-    @GET
-    @Path("getEstimatedStats")
-    @Produces("application/json")
-    public Response getEstimatedStats(@Context ServletContext servletContext,
-                                      @Context final HttpHeaders headers,
-                                      @QueryParam("path") String path) throws Exception {
-
-        if (Log.isDebugEnabled()) {
-            StringBuilder startmsg = new StringBuilder("ANALYZER/getEstimatedStats started
for path \"" + path + "\"");
-            if (headers != null) {
-                for (String header : headers.getRequestHeaders().keySet()) {
-                    startmsg.append(" Header: ").append(header).append(" Value: ").append(headers.getRequestHeader(header));
-                }
-            }
-            Log.debug(startmsg);
-        }
-
-		/* Convert headers into a regular map */
-        Map<String, String> params = convertToCaseInsensitiveMap(headers.getRequestHeaders());
-
-        /* Store protocol level properties and verify */
-        final ProtocolData protData = new ProtocolData(params);
-        SecuredHDFS.verifyToken(protData, servletContext);
-        
-        /*
-         * Analyzer is a special case in which it is hard to tell if user didn't
-         * specify one, or specified a profile that doesn't include one, or it's
-         * an actual protocol violation. Since we can only test protocol level
-         * logic, we assume (like before) that it's a user error, which is the
-         * case in most likelihood. When analyzer module is removed in the near
-         * future, this assumption will go away with it.
-         */
-        if (protData.getAnalyzer() == null) {
-			throw new IllegalArgumentException(
-					"PXF 'Analyzer' class was not found. Please supply it in the LOCATION clause or use
it in a PXF profile in order to run ANALYZE on this table");
-        }
-        
-        /* Create an analyzer instance with API level parameters */
-        final Analyzer analyzer = AnalyzerFactory.create(protData);
-
-		/*
-         * Function queries the pxf Analyzer for the data fragments of the resource
-		 * The fragments are returned in a string formatted in JSON	 
-		 */
-        String jsonOutput = AnalyzerStats.dataToJSON(analyzer.getEstimatedStats(path));
-
-        return Response.ok(jsonOutput, MediaType.APPLICATION_JSON_TYPE).build();
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/de474ea5/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/InvalidPathResource.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/InvalidPathResource.java
b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/InvalidPathResource.java
index bf45381..1dee254 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/InvalidPathResource.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/InvalidPathResource.java
@@ -24,12 +24,17 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hawq.pxf.service.utilities.Utilities;
 
+import com.google.common.collect.ImmutableSet;
+import java.util.Arrays;
+import java.util.List;
+
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.PathSegment;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.ResponseBuilder;
 import javax.ws.rs.core.UriInfo;
@@ -52,6 +57,9 @@ public class InvalidPathResource {
     UriInfo rootUri;
 
     private static final Log Log = LogFactory.getLog(InvalidPathResource.class);
+    // Set of retired endpoints
+    private final ImmutableSet<String> retiredEndPoints = ImmutableSet.of(
+            "Analyzer");
 
     public InvalidPathResource() {
     }
@@ -114,10 +122,16 @@ public class InvalidPathResource {
         Log.debug("REST request: " + rootUri.getAbsolutePath() + ". " +
                 "Version " + version + ", supported version is " + Version.PXF_PROTOCOL_VERSION);
 
-        // if version is not of the format "v<number>" then it's not a version but
a wrong path
-        if (version.equals(Version.PXF_PROTOCOL_VERSION)  || !(version.matches("v[0-9]+")))
{
+        if(version.equals(Version.PXF_PROTOCOL_VERSION)) { // api with incorrect path
+            String endPoint = parseEndpoint(path);
+            if (retiredEndPoints.contains(endPoint)) {
+                errmsg = getRetiredPathMsg(endPoint);
+            } else {
+                errmsg = getUnknownPathMsg();
+            }
+        } else if(!(version.matches("v[0-9]+"))) { // api with version not of the format
"v<number>"
             errmsg = getUnknownPathMsg();
-        } else {
+        } else { // api with wrong version number
             errmsg = "Wrong version " + version + ", supported version is " + Version.PXF_PROTOCOL_VERSION;
         }
 
@@ -153,9 +167,39 @@ public class InvalidPathResource {
     }
 
     /**
+     * Parses the version part from the path.
+     * The the absolute path is
+     * http://<host>:<port>/pxf/<version>/<rest of path>
+     *
+     * path - the endpoint part after /pxf/
+     * returns the first element after /pxf/
+     */
+    private String parseEndpoint(String path) {
+        List<PathSegment> pathSegments = rootUri.getPathSegments();
+        if(pathSegments.size() < 2) {
+            return null;
+        }
+        return pathSegments.get(1).getPath();
+    }
+
+    /**
      * Returns unknown path message, with the path's special characters masked.
      */
     private String getUnknownPathMsg() {
         return "Unknown path \"" + Utilities.maskNonPrintables(rootUri.getAbsolutePath().toString())
+ "\"";
     }
+
+    /**
+     * Warn on recently retired paths
+     * eg: http://<host>:<port>/pxf/<version>/Analyzer/<rest of path>
+     *
+     * Returns message about path not being supported
+     */
+    private String getRetiredPathMsg(String endpoint) {
+        if("Analyzer".equals(endpoint)) {
+            return endpoint + " API is retired. Please use /Fragmenter/getFragmentsStats
instead";
+        } else {
+            return endpoint + " API is retired";
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/de474ea5/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
index 936df84..6ab224a 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
@@ -94,7 +94,6 @@ public class ProtocolData extends InputData {
         }
         accessor = getProperty("ACCESSOR");
         resolver = getProperty("RESOLVER");
-        analyzer = getOptionalProperty("ANALYZER");
         fragmenter = getOptionalProperty("FRAGMENTER");
         dataSource = getProperty("DATA-DIR");
 
@@ -142,7 +141,6 @@ public class ProtocolData extends InputData {
         this.accessor = copy.accessor;
         this.resolver = copy.resolver;
         this.fragmenter = copy.fragmenter;
-        this.analyzer = copy.analyzer;
         this.threadSafe = copy.threadSafe;
         this.remoteLogin = copy.remoteLogin;
         this.remoteSecret = copy.remoteSecret;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/de474ea5/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml b/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
index ba346a1..5b57d21 100644
--- a/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
+++ b/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
@@ -85,7 +85,6 @@ under the License.
             <fragmenter>org.apache.hawq.pxf.plugins.hdfs.HdfsDataFragmenter</fragmenter>
             <accessor>org.apache.hawq.pxf.plugins.hdfs.LineBreakAccessor</accessor>
             <resolver>org.apache.hawq.pxf.plugins.hdfs.StringPassResolver</resolver>
-            <analyzer>org.apache.hawq.pxf.plugins.hdfs.HdfsAnalyzer</analyzer>
         </plugins>
     </profile>
     <profile>
@@ -97,7 +96,6 @@ under the License.
             <fragmenter>org.apache.hawq.pxf.plugins.hdfs.HdfsDataFragmenter</fragmenter>
             <accessor>org.apache.hawq.pxf.plugins.hdfs.QuotedLineBreakAccessor</accessor>
             <resolver>org.apache.hawq.pxf.plugins.hdfs.StringPassResolver</resolver>
-            <analyzer>org.apache.hawq.pxf.plugins.hdfs.HdfsAnalyzer</analyzer>
         </plugins>
     </profile>
     <profile>
@@ -107,7 +105,6 @@ under the License.
             <fragmenter>org.apache.hawq.pxf.plugins.hdfs.HdfsDataFragmenter</fragmenter>
             <accessor>org.apache.hawq.pxf.plugins.hdfs.AvroFileAccessor</accessor>
             <resolver>org.apache.hawq.pxf.plugins.hdfs.AvroResolver</resolver>
-            <analyzer>org.apache.hawq.pxf.plugins.hdfs.HdfsAnalyzer</analyzer>
         </plugins>
     </profile>
     <profile>
@@ -120,7 +117,6 @@ under the License.
             <fragmenter>org.apache.hawq.pxf.plugins.hdfs.HdfsDataFragmenter</fragmenter>
             <accessor>org.apache.hawq.pxf.plugins.hdfs.SequenceFileAccessor</accessor>
             <resolver>org.apache.hawq.pxf.plugins.hdfs.WritableResolver</resolver>
-            <analyzer>org.apache.hawq.pxf.plugins.hdfs.HdfsAnalyzer</analyzer>
         </plugins>
     </profile>
     <profile>


Mime
View raw message