drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From par...@apache.org
Subject [03/15] drill git commit: DRILL-3243: Added CSG mods. Fixed field names. Removed old test files Added Parse_url and parse_query() functions Fix unit test
Date Tue, 01 Nov 2016 20:29:48 GMT
DRILL-3243: Added CSG mods. Fixed field names.
Removed old test files
Added Parse_url and parse_query() functions
Fix unit test

This closes #607


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4a82bc13
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4a82bc13
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4a82bc13

Branch: refs/heads/master
Commit: 4a82bc137865393f6f88b31805ad7c62cc9ca20b
Parents: 46c0f2a
Author: cgivre <cgivre@gmail.com>
Authored: Wed Oct 5 23:29:26 2016 -0400
Committer: Parth Chandra <parthc@apache.org>
Committed: Tue Nov 1 10:42:22 2016 -0700

----------------------------------------------------------------------
 .../exec/expr/fn/impl/ParseQueryFunction.java   |  87 ++++
 .../exec/expr/fn/impl/ParseUrlFunction.java     | 153 ++++++
 .../exec/store/httpd/HttpdFormatPlugin.java     | 487 -------------------
 .../exec/store/httpd/HttpdLogFormatPlugin.java  |   3 +-
 .../drill/exec/store/httpd/HttpdLogRecord.java  |   2 +-
 .../drill/exec/store/httpd/HttpdParser.java     | 123 ++++-
 .../drill/exec/store/httpd/HttpdParserTest.java |  48 ++
 .../dfs/TestFormatPluginOptionExtractor.java    |   3 +
 .../drill/exec/store/httpd/HttpdParserTest.java |  48 --
 .../store/httpd/TestHttpdLogFormatPlugin.java   |  97 ----
 .../drill/exec/store/httpd/TestHttpdPlugin.java |  31 --
 11 files changed, 411 insertions(+), 671 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseQueryFunction.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseQueryFunction.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseQueryFunction.java
new file mode 100644
index 0000000..7dce1fc
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseQueryFunction.java
@@ -0,0 +1,87 @@
+package org.apache.drill.exec.expr.fn.impl;
+
+//*
+
+import io.netty.buffer.DrillBuf;
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter;
+
+import javax.inject.Inject;
+
+/* Copyright 2001-2004 The Apache Software Foundation.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+@FunctionTemplate(
+        name="parse_query",
+        scope= FunctionTemplate.FunctionScope.SIMPLE,
+        nulls = FunctionTemplate.NullHandling.NULL_IF_NULL
+)
+
+public class ParseQueryFunction implements DrillSimpleFunc {
+
+    @Param
+    NullableVarCharHolder input;
+
+    @Output
+    BaseWriter.ComplexWriter outWriter;
+
+    @Inject
+    DrillBuf outBuffer;
+
+    public void setup() {
+    }
+
+    public void eval() {
+
+        org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter queryMapWriter = outWriter.rootAsMap();
+
+        String queryString = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(input.start, input.end, input.buffer);
+
+        if( queryString.isEmpty() || queryString.equals("null")){
+            queryString = "";
+        }
+
+        String firstLetter = queryString.substring(0, 1);
+
+        //If the first character is a &, it doesn't split properly.  This checks to see if the first character is an & and if so, removes it.
+        if(firstLetter.equals("&")){
+            queryString = queryString.substring(1);
+        }
+
+        String[] arguments = queryString.split("&");
+
+        for (int i = 0; i < arguments.length; i++) {
+            String[] queryParts = arguments[i].split("=");
+
+            org.apache.drill.exec.expr.holders.VarCharHolder rowHolder = new org.apache.drill.exec.expr.holders.VarCharHolder();
+
+            byte[] rowStringBytes = queryParts[1].getBytes();
+
+            outBuffer.reallocIfNeeded(rowStringBytes.length);
+            outBuffer.setBytes(0, rowStringBytes);
+
+            rowHolder.start = 0;
+            rowHolder.end = rowStringBytes.length;
+            rowHolder.buffer = outBuffer;
+
+            queryMapWriter.varChar(queryParts[0]).write(rowHolder);
+
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseUrlFunction.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseUrlFunction.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseUrlFunction.java
new file mode 100644
index 0000000..fa339d4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseUrlFunction.java
@@ -0,0 +1,153 @@
+package org.apache.drill.exec.expr.fn.impl;
+
+/*
+ * Copyright 2001-2004 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import io.netty.buffer.DrillBuf;
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter;
+
+import javax.inject.Inject;
+
+@FunctionTemplate(
+        name="parse_url",
+        scope= FunctionTemplate.FunctionScope.SIMPLE,
+        nulls = FunctionTemplate.NullHandling.NULL_IF_NULL
+)
+
+public class ParseUrlFunction implements DrillSimpleFunc {
+
+    @Param VarCharHolder input;
+
+    @Output BaseWriter.ComplexWriter outWriter;
+
+    @Inject DrillBuf outBuffer;
+
+    public void setup() {}
+
+    public void eval() {
+
+        org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter urlMapWriter = outWriter.rootAsMap();
+
+        String urlString = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(input.start, input.end, input.buffer);
+
+        try {
+            java.net.URL aURL = new java.net.URL(urlString);
+
+            String protocol = aURL.getProtocol();
+            String authority = aURL.getAuthority();
+            String host = aURL.getHost();
+            java.lang.Integer port = aURL.getPort();
+            String path = aURL.getPath();
+            String query = aURL.getQuery();
+            String filename = aURL.getFile();
+            String ref = aURL.getRef();
+
+            org.apache.drill.exec.expr.holders.VarCharHolder rowHolder = new org.apache.drill.exec.expr.holders.VarCharHolder();
+
+            byte[] rowStringBytes = protocol.getBytes();
+
+            outBuffer.reallocIfNeeded(rowStringBytes.length);
+            outBuffer.setBytes(0, rowStringBytes);
+
+            rowHolder.start = 0;
+            rowHolder.end = rowStringBytes.length;
+            rowHolder.buffer = outBuffer;
+
+            urlMapWriter.varChar("protocol").write(rowHolder);
+
+
+            byte[] authRowStringBytes = authority.getBytes();
+
+            outBuffer.reallocIfNeeded(authRowStringBytes.length);
+            outBuffer.setBytes(0, authRowStringBytes);
+
+            rowHolder.start = 0;
+            rowHolder.end = authRowStringBytes.length;
+            rowHolder.buffer = outBuffer;
+
+            urlMapWriter.varChar("authority").write(rowHolder);
+
+
+            byte[] hostRowStringBytes = host.getBytes();
+
+            outBuffer.reallocIfNeeded(hostRowStringBytes.length);
+            outBuffer.setBytes(0, hostRowStringBytes);
+
+            rowHolder.start = 0;
+            rowHolder.end = hostRowStringBytes.length;
+            rowHolder.buffer = outBuffer;
+
+            urlMapWriter.varChar("host").write(rowHolder);
+
+
+            byte[] pathRowStringBytes = path.getBytes();
+
+            outBuffer.reallocIfNeeded(pathRowStringBytes.length);
+            outBuffer.setBytes(0, pathRowStringBytes);
+
+            rowHolder.start = 0;
+            rowHolder.end = pathRowStringBytes.length;
+            rowHolder.buffer = outBuffer;
+
+            urlMapWriter.varChar("path").write(rowHolder);
+
+
+            byte[] queryRowStringBytes = query.getBytes();
+
+            outBuffer.reallocIfNeeded(queryRowStringBytes.length);
+            outBuffer.setBytes(0, queryRowStringBytes);
+
+            rowHolder.start = 0;
+            rowHolder.end = queryRowStringBytes.length;
+            rowHolder.buffer = outBuffer;
+
+            urlMapWriter.varChar("query").write(rowHolder);
+
+
+            byte[] filenameRowStringBytes = filename.getBytes();
+
+            outBuffer.reallocIfNeeded(filenameRowStringBytes.length);
+            outBuffer.setBytes(0, filenameRowStringBytes);
+
+            rowHolder.start = 0;
+            rowHolder.end = filenameRowStringBytes.length;
+            rowHolder.buffer = outBuffer;
+
+            urlMapWriter.varChar("filename").write(rowHolder);
+
+
+            byte[] refRowStringBytes = ref.getBytes();
+
+            outBuffer.reallocIfNeeded(refRowStringBytes.length);
+            outBuffer.setBytes(0, refRowStringBytes);
+
+            rowHolder.start = 0;
+            rowHolder.end = refRowStringBytes.length;
+            rowHolder.buffer = outBuffer;
+
+            urlMapWriter.varChar("ref").write(rowHolder);
+
+            org.apache.drill.exec.expr.holders.IntHolder intHolder = new org.apache.drill.exec.expr.holders.IntHolder();
+            intHolder.value = port;
+            urlMapWriter.integer("port").write(intHolder);
+        }
+        catch (Exception e ) {}
+    }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdFormatPlugin.java
deleted file mode 100644
index 7b8dc0e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdFormatPlugin.java
+++ /dev/null
@@ -1,487 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.httpd;
-
-import io.netty.buffer.DrillBuf;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import nl.basjes.parse.core.Parser;
-import nl.basjes.parse.core.exceptions.DissectionFailure;
-import nl.basjes.parse.core.exceptions.InvalidDissectorException;
-import nl.basjes.parse.core.exceptions.MissingDissectorsException;
-import nl.basjes.parse.httpdlog.ApacheHttpdLogFormatDissector;
-import nl.basjes.parse.httpdlog.dissectors.HttpFirstLineDissector;
-
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.FormatPluginConfig;
-import org.apache.drill.common.logical.StoragePluginConfig;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.ops.OperatorContext;
-import org.apache.drill.exec.physical.impl.OutputMutator;
-import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.AbstractRecordReader;
-import org.apache.drill.exec.store.RecordWriter;
-import org.apache.drill.exec.store.dfs.DrillFileSystem;
-import org.apache.drill.exec.store.dfs.easy.EasyFormatPlugin;
-import org.apache.drill.exec.store.dfs.easy.EasyWriter;
-import org.apache.drill.exec.store.dfs.easy.FileWork;
-import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
-import org.apache.drill.exec.vector.complex.writer.BigIntWriter;
-import org.apache.drill.exec.vector.complex.writer.Float8Writer;
-import org.apache.drill.exec.vector.complex.writer.VarCharWriter;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.LineRecordReader;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.TextInputFormat;
-
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.base.Charsets;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-public class HttpdFormatPlugin extends EasyFormatPlugin<HttpdFormatPlugin.HttpdLogFormatConfig> {
-
-  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HttpdFormatPlugin.class);
-
-  private static final String DEFAULT_EXTENSION = "httpd";
-
-  public HttpdFormatPlugin(
-      String name,
-      DrillbitContext context,
-      Configuration fsConf,
-      StoragePluginConfig storageConfig,
-      HttpdLogFormatConfig formatConfig) {
-    super(name, context, fsConf, storageConfig, formatConfig, true, false, true, true,
-        Lists.newArrayList(DEFAULT_EXTENSION), DEFAULT_EXTENSION);
-  }
-
-  @JsonTypeName("httpd")
-  public static class HttpdLogFormatConfig implements FormatPluginConfig {
-    public String format;
-  }
-
-  private class RecordReader extends AbstractRecordReader {
-
-    private final DrillFileSystem fs;
-    private final FileWork work;
-    private final FragmentContext fragmentContext;
-
-    private ComplexWriter writer;
-    private Parser<ComplexWriterFacade> parser;
-    private LineRecordReader lineReader;
-    private LongWritable lineNumber;
-    private ComplexWriterFacade record;
-    private DrillBuf managedBuffer;
-
-    public RecordReader(FragmentContext context, DrillFileSystem fs, FileWork work) {
-      this.fs = fs;
-      this.work = work;
-      fragmentContext = context;
-      managedBuffer = context.getManagedBuffer();
-    }
-
-    @Override
-    public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException {
-
-      try {
-
-        parser = new PartiallyDissectedParser<ComplexWriterFacade>(
-            ComplexWriterFacade.class,
-            HttpdFormatPlugin.this.getConfig().format);
-        writer = new VectorContainerWriter(output);
-        record = new ComplexWriterFacade(writer);
-        record.addAsParseTarget(parser);
-
-        final Path path = fs.makeQualified(new Path(work.getPath()));
-        FileSplit split = new FileSplit(path, work.getStart(), work.getLength(), new String[] { "" });
-        TextInputFormat inputFormat = new TextInputFormat();
-        JobConf job = new JobConf(fs.getConf());
-        job.setInt("io.file.buffer.size", fragmentContext.getConfig()
-            .getInt(ExecConstants.TEXT_LINE_READER_BUFFER_SIZE));
-        job.setInputFormat(inputFormat.getClass());
-        lineReader = (LineRecordReader) inputFormat.getRecordReader(split, job, Reporter.NULL);
-        lineNumber = lineReader.createKey();
-      } catch (Exception e) {
-        throw handleAndGenerate("Failure in creating record reader", e);
-      }
-
-
-    }
-
-    private DrillBuf buf(int size) {
-      if (managedBuffer.capacity() < size) {
-        managedBuffer = managedBuffer.reallocIfNeeded(size);
-      }
-      return managedBuffer;
-    }
-
-    protected RuntimeException handleAndGenerate(String s, Exception e) {
-      throw UserException.dataReadError(e)
-          .message(s + "\n%s", e.getMessage())
-          .addContext("Path", work.getPath())
-          .addContext("Split Start", work.getStart())
-          .addContext("Split Length", work.getLength())
-          .addContext("Local Line Number", lineNumber.get())
-          .build(logger);
-    }
-
-    @Override
-    public int next() {
-      try {
-        final Text currentText = lineReader.createValue();
-
-        writer.allocate();
-        writer.reset();
-        int recordCount = 0;
-
-        for (; recordCount < 4095 && lineReader.next(lineNumber, currentText); recordCount++) {
-          writer.setPosition(recordCount);
-          parser.parse(record, currentText.toString());
-        }
-
-        writer.setValueCount(recordCount);
-        return recordCount;
-      } catch (DissectionFailure | InvalidDissectorException | MissingDissectorsException | IOException e) {
-        throw handleAndGenerate("Failure while reading httpd log record.", e);
-      }
-    }
-
-    @Override
-    public void cleanup() {
-
-      try {
-        if (lineReader != null) {
-          lineReader.close();
-        }
-      } catch (IOException e) {
-        logger.warn("Failure while closing Httpd reader.", e);
-      }
-    }
-
-    /**
-     * Maps Httpd Log Libraries calls to Drills ComplexWriter interface.
-     */
-    public class ComplexWriterFacade {
-      private final ComplexWriter writer;
-      private final Map<String, VarCharWriter> stringWriters = Maps.newHashMap();
-      private final Map<String, BigIntWriter> longWriters = Maps.newHashMap();
-      private final Map<String, Float8Writer> doubleWriters = Maps.newHashMap();
-
-      private ComplexWriterFacade(ComplexWriter writer) {
-        this.writer = writer;
-      }
-
-      @SuppressWarnings("unused")
-      public void set(final String name, final String value) {
-        if (value != null) {
-          final byte[] stringBytes = value.getBytes(Charsets.UTF_8);
-          final DrillBuf stringBuffer = buf(stringBytes.length);
-          stringBuffer.clear();
-          stringBuffer.writeBytes(stringBytes);
-          final VarCharWriter writer = stringWriters.get(name);
-          if (writer != null) {
-            writer.writeVarChar(0, stringBytes.length, stringBuffer);
-          } else {
-            logger.warn("Dropped string.  Name: {}, Value: {}", name, value);
-          }
-        }
-      }
-
-      @SuppressWarnings("unused")
-      public void set(String name, Long value) {
-        if (value != null) {
-          longWriters.get(name).writeBigInt(value);
-        }
-      }
-
-      @SuppressWarnings("unused")
-      public void set(String name, Double value) {
-        if (value != null) {
-          doubleWriters.get(name).writeFloat8(value);
-        }
-      }
-
-      private void add(Parser<ComplexWriterFacade> parser, String path, VarCharWriter writer)
-          throws NoSuchMethodException,
-          SecurityException {
-        stringWriters.put(path, writer);
-        parser.addParseTarget(
-            ComplexWriterFacade.class.getMethod("set", new Class[] { String.class, String.class }),
-            path);
-      }
-
-      @SuppressWarnings("unused")
-      private void add(Parser<ComplexWriterFacade> parser, String path, Float8Writer writer)
-          throws NoSuchMethodException,
-          SecurityException {
-        doubleWriters.put(path, writer);
-        parser.addParseTarget(
-            ComplexWriterFacade.class.getMethod("set", new Class[] { String.class, Double.class }),
-            path);
-      }
-
-      private void add(Parser<ComplexWriterFacade> parser, String path, BigIntWriter writer)
-          throws NoSuchMethodException,
-          SecurityException {
-        longWriters.put(path, writer);
-        parser.addParseTarget(
-            ComplexWriterFacade.class.getMethod("set", new Class[] { String.class, Long.class }),
-            path);
-      }
-
-      public void addAsParseTarget(Parser<ComplexWriterFacade> parser) {
-        try {
-
-          for (final String path : parser.getPossiblePaths()) {
-            switch (path) {
-            case "IP:connection.client.ip":
-              add(parser, path, writer.rootAsMap().map("client").varChar("ip"));
-              break;
-            case "IP:connection.client.peerip":
-              add(parser, path, writer.rootAsMap().map("client").varChar("peer_ip"));
-              break;
-            case "IP:connection.server.ip":
-              add(parser, path, writer.rootAsMap().map("server").varChar("ip"));
-              break;
-            case "BYTES:response.body.bytes":
-              add(parser, path, writer.rootAsMap().map("response").bigInt("bytes"));
-              break;
-            case "BYTES:response.body.bytesclf":
-              add(parser, path, writer.rootAsMap().map("response").bigInt("bytes"));
-              break;
-            case "HTTP.COOKIE:request.cookies.":
-              add(parser, path, writer.rootAsMap().map("request").varChar("cookies"));
-              break;
-            case "MICROSECONDS:server.process.time":
-              add(parser, path, writer.rootAsMap().map("response").bigInt("process_time"));
-              break;
-            case "FILENAME:server.filename":
-              add(parser, path, writer.rootAsMap().map("response").varChar("filename"));
-              break;
-            case "IP:connection.client.host":
-              add(parser, path, writer.rootAsMap().map("client").varChar("host"));
-              break;
-            case "PROTOCOL:request.protocol":
-              add(parser, path, writer.rootAsMap().map("request").varChar("protocol"));
-              break;
-            case "HTTP.HEADER:request.header.":
-              add(parser, path, writer.rootAsMap().map("request").varChar("header"));
-              break;
-            case "NUMBER:connection.keepalivecount":
-              add(parser, path, writer.rootAsMap().map("client").bigInt("keepalivecount"));
-              break;
-            case "NUMBER:connection.client.logname":
-              add(parser, path, writer.rootAsMap().map("request").bigInt("logname"));
-              break;
-            case "STRING:request.errorlogid":
-              add(parser, path, writer.rootAsMap().map("request").varChar("errorlogid"));
-              break;
-            case "HTTP.METHOD:request.method":
-              add(parser, path, writer.rootAsMap().map("request").varChar("method"));
-              break;
-            case "PORT:request.server.port.canonical":
-              add(parser, path, writer.rootAsMap().map("server").bigInt("canonical_port"));
-              break;
-            case "PORT:connection.server.port.canonical":
-              add(parser, path, writer.rootAsMap().map("server").bigInt("canonical_port"));
-              break;
-            case "PORT:connection.client.port":
-              add(parser, path, writer.rootAsMap().map("client").bigInt("port"));
-              break;
-            case "NUBMER:connection.server.child.processid":
-              add(parser, path, writer.rootAsMap().map("server").bigInt("process_id"));
-              break;
-            case "NUMBER:connection.server.child.threadid":
-              add(parser, path, writer.rootAsMap().map("server").bigInt("thread_id"));
-              break;
-            case "STRING:connection.server.child.hexthreadid":
-              add(parser, path, writer.rootAsMap().map("connection").varChar("hex_thread_id"));
-              break;
-            case "HTTP.QUERYSTRING:request.querystring":
-              add(parser, path, writer.rootAsMap().map("").varChar(""));
-              break;
-            case "HTTP.FIRSTLINE:request.firstline":
-              add(parser, path, writer.rootAsMap().map("").varChar(""));
-              break;
-            case "STRING:request.handler":
-              add(parser, path, writer.rootAsMap().map("request").varChar("handler"));
-              break;
-            case "STRING:request.status.original":
-              add(parser, path, writer.rootAsMap().map("request").varChar("status_original"));
-              break;
-            case "STRING:request.status.last":
-              add(parser, path, writer.rootAsMap().map("request").varChar("status_last"));
-              break;
-            case "TIME.STAMP:request.receive.time":
-              add(parser, path, writer.rootAsMap().map("request").varChar("timestamp"));
-              break;
-            case "TIME.EPOCH:request.receive.time.begin.msec":
-              add(parser, path, writer.rootAsMap().map("request").bigInt("begin_msec"));
-              break;
-            case "TIME.EPOCH:request.receive.time.end.msec":
-              add(parser, path, writer.rootAsMap().map("request").bigInt("end_msec"));
-              break;
-            case "TIME.EPOCH.USEC:request.receive.time.begin.usec":
-              add(parser, path, writer.rootAsMap().map("request").bigInt("begin_usec"));
-              break;
-            case "TIME.EPOCH.USEC:request.receive.time.end.usec":
-              add(parser, path, writer.rootAsMap().map("request").bigInt("end_usec"));
-              break;
-            case "TIME.EPOCH:request.receive.time.begin.msec_frac":
-              add(parser, path, writer.rootAsMap().map("request").bigInt("begin_msec_frac"));
-              break;
-            case "TIME.EPOCH:request.receive.time.end.msec_frac":
-              add(parser, path, writer.rootAsMap().map("request").varChar("end_msec_frac"));
-              break;
-            case "TIME.EPOCH.USEC_FRAC:request.receive.time.begin.usec_frac":
-              add(parser, path, writer.rootAsMap().map("request").varChar("begin_usec_frac"));
-              break;
-            case "TIME.EPOCH.USEC_FRAC:request.receive.time.end.usec_frac":
-              add(parser, path, writer.rootAsMap().map("request").varChar("end_usec_frac"));
-              break;
-            case "SECONDS:response.server.processing.time":
-              add(parser, path, writer.rootAsMap().map("response").varChar("processing_time"));
-              break;
-            case "STRING:connection.client.user":
-              add(parser, path, writer.rootAsMap().map("client").varChar("user"));
-              break;
-            case "URI:request.urlpath":
-              add(parser, path, writer.rootAsMap().map("request").varChar("url"));
-              break;
-            case "STRING:connection.server.name.canonical":
-              add(parser, path, writer.rootAsMap().map("server").varChar("canonical_name"));
-              break;
-            case "STRING:connection.server.name":
-              add(parser, path, writer.rootAsMap().map("server").varChar("name"));
-              break;
-            case "HTTP.CONNECTSTATUS:response.connection.status":
-              add(parser, path, writer.rootAsMap().map("response").varChar("connection_status"));
-              break;
-            case "BYTES:request.bytes":
-              add(parser, path, writer.rootAsMap().map("request").varChar("bytes"));
-              break;
-            case "BYTES:response.bytes":
-              add(parser, path, writer.rootAsMap().map("response").bigInt("bytes"));
-              break;
-            case "HTTP.COOKIES:request.cookies":
-              add(parser, path, writer.rootAsMap().map("request").varChar("cookies"));
-              break;
-            case "HTTP.SETCOOKIES:response.cookies":
-              add(parser, path, writer.rootAsMap().map("response").varChar("cookies"));
-              break;
-            case "HTTP.USERAGENT:request.user-agent":
-              add(parser, path, writer.rootAsMap().map("request").varChar("useragent"));
-              break;
-            case "HTTP.URI:request.referer":
-              add(parser, path, writer.rootAsMap().map("request").varChar("referer"));
-              break;
-            case "HTTP.METHOD:method":
-              add(parser, path, writer.rootAsMap().map("request").varChar("method"));
-              break;
-            case "HTTP.URI:uri":
-              add(parser, path, writer.rootAsMap().map("request").varChar("uri"));
-              break;
-            case "HTTP.PROTOCOL:protocol":
-              add(parser, path, writer.rootAsMap().map("request").varChar("protocol"));
-              break;
-            case "HTTP.PROTOCOL.VERSION:protocol.version":
-              add(parser, path, writer.rootAsMap().map("request").varChar("protocol_version"));
-              break;
-            case "HTTP.METHOD:request.firstline.method":
-              add(parser, path, writer.rootAsMap().map("request").varChar("method"));
-              break;
-            case "HTTP.URI:request.firstline.uri":
-              add(parser, path, writer.rootAsMap().map("request").varChar("uri"));
-              break;
-            case "HTTP.PROTOCOL:request.firstline.protocol":
-              add(parser, path, writer.rootAsMap().map("request").varChar("protocol"));
-              break;
-            case "HTTP.PROTOCOL.VERSION:request.firstline.protocol.version":
-              add(parser, path, writer.rootAsMap().map("request").varChar("protocol_version"));
-              break;
-            default:
-
-              // if we don't know what to do, just write the raw value.
-              parser.addParseTarget(
-                  ComplexWriterFacade.class.getMethod("set", new Class[] { String.class, String.class }),
-                  path);
-              final String noPeriodPath = path.replace(".", "_");
-              stringWriters.put(path, writer.rootAsMap().varChar(noPeriodPath));
-              break;
-
-            }
-          }
-
-
-        } catch (MissingDissectorsException | SecurityException | NoSuchMethodException | InvalidDissectorException e) {
-          throw handleAndGenerate("Failure while setting up log mappings.", e);
-        }
-      }
-    }
-  }
-
-  @Override
-  public boolean supportsPushDown() {
-    return true;
-  }
-
-
-  @Override
-  public RecordReader getRecordReader(FragmentContext context, DrillFileSystem dfs,
-      FileWork fileWork, List<SchemaPath> columns) throws ExecutionSetupException {
-    return new RecordReader(context, dfs, fileWork);
-  }
-
-  @Override
-  public RecordWriter getRecordWriter(FragmentContext context, EasyWriter writer) throws IOException {
-    throw new UnsupportedOperationException("Drill doesn't currently support writing to HTTPD logs.");
-  }
-
-  @Override
-  public int getReaderOperatorType() {
-    return -1;
-  }
-
-  @Override
-  public int getWriterOperatorType() {
-    return -1;
-  }
-
-  private class PartiallyDissectedParser<RECORD> extends Parser<RECORD> {
-    public PartiallyDissectedParser(Class<RECORD> clazz, final String logformat) {
-      super(clazz);
-
-      addDissector(new ApacheHttpdLogFormatDissector(logformat));
-      addDissector(new HttpFirstLineDissector());
-      setRootType(ApacheHttpdLogFormatDissector.INPUT_TYPE);
-    }
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
index 7a83d45..a6dc7d9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
@@ -1,3 +1,4 @@
+
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
  * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
@@ -243,4 +244,4 @@ public class HttpdLogFormatPlugin extends EasyFormatPlugin<HttpdLogFormatPlugin.
   public int getWriterOperatorType() {
     return -1;
   }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
index 03f70c1..2775285 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
@@ -296,4 +296,4 @@ public class HttpdLogRecord {
       strings.put(parserFieldName, mapWriter.varChar(drillFieldName));
     }
   }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java
index 114a7f4..a896638 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java
@@ -18,9 +18,6 @@ package org.apache.drill.exec.store.httpd;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
 import io.netty.buffer.DrillBuf;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
 import nl.basjes.parse.core.Casts;
 import nl.basjes.parse.core.Parser;
 import nl.basjes.parse.core.exceptions.DissectionFailure;
@@ -31,6 +28,11 @@ import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 public class HttpdParser {
 
   private static final Logger LOG = LoggerFactory.getLogger(HttpdParser.class);
@@ -41,6 +43,105 @@ public class HttpdParser {
   private final Parser<HttpdLogRecord> parser;
   private final HttpdLogRecord record;
 
+    public static final HashMap<String, String> LOGFIELDS = new HashMap<String, String>();
+    static
+    {
+        LOGFIELDS.put("request_receive_time_weekyear__utc", "TIME_YEAR:request_receive_time_weekyear__utc");
+        LOGFIELDS.put("request_referer_ref", "HTTP_REF:request_referer_ref");
+        LOGFIELDS.put("request_referer_protocol", "HTTP_PROTOCOL:request_referer_protocol");
+        LOGFIELDS.put("request_receive_time_timezone", "TIME_ZONE:request_receive_time_timezone");
+        LOGFIELDS.put("connection_client_host", "IP:connection_client_host");
+        LOGFIELDS.put("connection_client_ip", "IP:connection_client_ip");
+        LOGFIELDS.put("connection_client_peerip", "IP:connection_client_peerip");
+        LOGFIELDS.put("connection_server_ip", "IP:connection_server_ip");
+        LOGFIELDS.put("request_receive_time_day", "TIME_DAY:request_receive_time_day");
+        LOGFIELDS.put("request_receive_time_minute__utc", "TIME_MINUTE:request_receive_time_minute__utc");
+        LOGFIELDS.put("request_referer_query_$", "STRING:request_referer_query_$");
+        LOGFIELDS.put("request_receive_time_millisecond__utc", "TIME_MILLISECOND:request_receive_time_millisecond__utc");
+        LOGFIELDS.put("request_firstline_uri_port", "HTTP_PORT:request_firstline_uri_port");
+        LOGFIELDS.put("request_referer_userinfo", "HTTP_USERINFO:request_referer_userinfo");
+        LOGFIELDS.put("request_receive_time_second__utc", "TIME_SECOND:request_receive_time_second__utc");
+        LOGFIELDS.put("request_firstline_uri_protocol", "HTTP_PROTOCOL:request_firstline_uri_protocol");
+        LOGFIELDS.put("request_receive_time_month", "TIME_MONTH:request_receive_time_month");
+        LOGFIELDS.put("request_firstline_uri_query", "HTTP_QUERYSTRING:request_firstline_uri_query");
+        LOGFIELDS.put("request_firstline_uri_path", "HTTP_PATH:request_firstline_uri_path");
+        LOGFIELDS.put("request_receive_time_hour__utc", "TIME_HOUR:request_receive_time_hour__utc");
+        LOGFIELDS.put("request_receive_time_monthname", "TIME_MONTHNAME:request_receive_time_monthname");
+        LOGFIELDS.put("request_receive_time_year__utc", "TIME_YEAR:request_receive_time_year__utc");
+        LOGFIELDS.put("request_receive_time_second", "TIME_SECOND:request_receive_time_second");
+        LOGFIELDS.put("request_referer", "HTTP_URI:request_referer");
+        LOGFIELDS.put("request_receive_time_monthname__utc", "TIME_MONTHNAME:request_receive_time_monthname__utc");
+        LOGFIELDS.put("request_referer_path", "HTTP_PATH:request_referer_path");
+        LOGFIELDS.put("request_receive_time_weekyear", "TIME_YEAR:request_receive_time_weekyear");
+        LOGFIELDS.put("request_firstline_protocol", "HTTP_PROTOCOL:request_firstline_protocol");
+        LOGFIELDS.put("request_referer_port", "HTTP_PORT:request_referer_port");
+        LOGFIELDS.put("request_receive_time_minute", "TIME_MINUTE:request_receive_time_minute");
+        LOGFIELDS.put("request_status_last", "STRING:request_status_last");
+        LOGFIELDS.put("request_receive_time_hour", "TIME_HOUR:request_receive_time_hour");
+        LOGFIELDS.put("request_firstline_protocol_version", "HTTP_PROTOCOL_VERSION:request_firstline_protocol_version");
+        LOGFIELDS.put("request_receive_time", "TIME_STAMP:request_receive_time");
+        LOGFIELDS.put("request_firstline_method", "HTTP_METHOD:request_firstline_method");
+        LOGFIELDS.put("request_receive_time_epoch", "TIME_EPOCH:request_receive_time_epoch");
+        LOGFIELDS.put("request_receive_time_weekofweekyear", "TIME_WEEK:request_receive_time_weekofweekyear");
+        LOGFIELDS.put("request_firstline_uri_host", "HTTP_HOST:request_firstline_uri_host");
+        LOGFIELDS.put("request_referer_query", "HTTP_QUERYSTRING:request_referer_query");
+        LOGFIELDS.put("request_firstline_uri_userinfo", "HTTP_USERINFO:request_firstline_uri_userinfo");
+        LOGFIELDS.put("response_body_bytes", "BYTES:response_body_bytes");
+        LOGFIELDS.put("response_body_bytesclf", "BYTES:response_body_bytesclf");
+        LOGFIELDS.put("request_referer_host", "HTTP_HOST:request_referer_host");
+        LOGFIELDS.put("request_receive_time_weekofweekyear__utc", "TIME_WEEK:request_receive_time_weekofweekyear__utc");
+        LOGFIELDS.put("request_firstline_uri", "HTTP_URI:request_firstline_uri");
+        LOGFIELDS.put("request_firstline_uri_ref", "HTTP_REF:request_firstline_uri_ref");
+        LOGFIELDS.put("request_receive_time_year", "TIME_YEAR:request_receive_time_year");
+        LOGFIELDS.put("request_firstline", "HTTP_FIRSTLINE:request_firstline");
+        LOGFIELDS.put("request_user-agent", "HTTP_USERAGENT:request_user-agent");
+        LOGFIELDS.put("request_cookies", "HTTP_COOKIE:request_cookies");
+        LOGFIELDS.put("server_process_time", "MICROSECONDS:server_process_time");
+        LOGFIELDS.put("request_cookies_$", "HTTP_COOKIE:request_cookies_$");
+        LOGFIELDS.put("server_environment_$", "VARIABLE:server_environment_$");
+        LOGFIELDS.put("server_filename", "FILENAME:server_filename");
+        LOGFIELDS.put("request_protocol", "PROTOCOL:request_protocol");
+        LOGFIELDS.put("request_header_", "HTTP_HEADER:request_header_");
+        LOGFIELDS.put("connection_keepalivecount", "NUMBER:connection_keepalivecount");
+        LOGFIELDS.put("connection_client_logname", "NUMBER:connection_client_logname");
+        LOGFIELDS.put("request_errorlogid", "STRING:request_errorlogid");
+        LOGFIELDS.put("request_method", "HTTP_METHOD:request_method");
+        LOGFIELDS.put("server_module_note_$", "STRING:server_module_note_$");
+        LOGFIELDS.put("response_header_$", "HTTP_HEADER:response_header_$");
+        LOGFIELDS.put("request_server_port_canonical", "PORT:request_server_port_canonical");
+        LOGFIELDS.put("connection_server_port_canonical", "PORT:connection_server_port_canonical");
+        LOGFIELDS.put("connection_server_port", "PORT:connection_server_port");
+        LOGFIELDS.put("connection_client_port", "PORT:connection_client_port");
+        LOGFIELDS.put("connection_server_child_processid", "NUMBER:connection_server_child_processid");
+        LOGFIELDS.put("connection_server_child_threadid", "NUMBER:connection_server_child_threadid");
+        LOGFIELDS.put("connection_server_child_hexthreadid", "NUMBER:connection_server_child_hexthreadid");
+        LOGFIELDS.put("request_querystring", "HTTP_QUERYSTRING:request_querystring");
+        LOGFIELDS.put("request_handler", "STRING:request_handler");
+        LOGFIELDS.put("request_status_original", "STRING:request_status_original");
+        LOGFIELDS.put("request_status_last", "STRING:request_status_last");
+        LOGFIELDS.put("request_receive_time_begin_msec", "TIME_EPOCH:request_receive_time_begin_msec");
+        LOGFIELDS.put("request_receive_time_end_msec", "TIME_EPOCH:request_receive_time_end_msec");
+        LOGFIELDS.put("request_receive_time_begin_usec", "TIME_EPOCH_USEC:request_receive_time_begin_usec");
+        LOGFIELDS.put("request_receive_time_begin_usec", "TIME_EPOCH_USEC:request_receive_time_begin_usec");
+        LOGFIELDS.put("request_receive_time_end_usec", "TIME_EPOCH_USEC:request_receive_time_end_usec");
+        LOGFIELDS.put("request_receive_time_begin_msec_frac", "TIME_EPOCH:request_receive_time_begin_msec_frac");
+        LOGFIELDS.put("request_receive_time_begin_msec_frac", "TIME_EPOCH:request_receive_time_begin_msec_frac");
+        LOGFIELDS.put("request_receive_time_end_msec_frac", "TIME_EPOCH:request_receive_time_end_msec_frac");
+        LOGFIELDS.put("request_receive_time_begin_usec_frac", "TIME_EPOCH_USEC_FRAC:request_receive_time_begin_usec_frac");
+        LOGFIELDS.put("request_receive_time_begin_usec_frac", "TIME_EPOCH_USEC_FRAC:request.receive.time.begin.usec_frac");
+        LOGFIELDS.put("request_receive_time_end_usec_frac", "TIME_EPOCH_USEC_FRAC:request_receive_time_end_usec_frac");
+        LOGFIELDS.put("response_server_processing_time", "SECONDS:response_server_processing_time");
+        LOGFIELDS.put("connection_client_user", "STRING:connection_client_user");
+        LOGFIELDS.put("request_urlpath", "URI:request_urlpath");
+        LOGFIELDS.put("connection_server_name_canonical", "STRING:connection_server_name_canonical");
+        LOGFIELDS.put("connection_server_name", "STRING:connection_server_name");
+        LOGFIELDS.put("response_connection_status", "HTTP_CONNECTSTATUS:response_connection_status");
+        LOGFIELDS.put("request_bytes", "BYTES:request_bytes");
+        LOGFIELDS.put("response_bytes", "BYTES:response_bytes");
+    }
+
+    //Map map = Collections.synchronizedMap(LOGFIELDS);
+
   public HttpdParser(final MapWriter mapWriter, final DrillBuf managedBuffer, final String logFormat,
       final String timestampFormat, final Map<String, String> fieldMapping)
       throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {
@@ -96,7 +197,10 @@ public class HttpdParser {
    * @return
    */
   public static String parserFormattedFieldName(final String drillFieldName) {
-    return drillFieldName.replace(SAFE_WILDCARD, PARSER_WILDCARD).replaceAll(SAFE_SEPARATOR, ".").replaceAll("\\.\\.", "_");
+      String tempFieldName;
+      tempFieldName = LOGFIELDS.get(drillFieldName);
+      return tempFieldName.replace(SAFE_WILDCARD, PARSER_WILDCARD).replaceAll(SAFE_SEPARATOR, ".").replaceAll("\\.\\.", "_");
+
   }
 
   /**
@@ -108,7 +212,14 @@ public class HttpdParser {
    * @return
    */
   public static String drillFormattedFieldName(final String parserFieldName) {
-    return parserFieldName.replaceAll("_", "__").replace(PARSER_WILDCARD, SAFE_WILDCARD).replaceAll("\\.", SAFE_SEPARATOR);
+
+      if (parserFieldName.contains(":") ) {
+        String[] fieldPart= parserFieldName.split(":");
+        return fieldPart[1].replaceAll("_", "__").replace(PARSER_WILDCARD, SAFE_WILDCARD).replaceAll("\\.", SAFE_SEPARATOR);
+        }
+    else{
+      return parserFieldName.replaceAll("_", "__").replace(PARSER_WILDCARD, SAFE_WILDCARD).replaceAll("\\.", SAFE_SEPARATOR);
+    }
   }
 
   private void setupParser(final MapWriter mapWriter, final String logFormat, final Map<String, String> fieldMapping)
@@ -168,4 +279,4 @@ public class HttpdParser {
       record.addField(parser, mapWriter, casts, entry.getValue(), entry.getKey());
     }
   }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
new file mode 100644
index 0000000..b82b1ee
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2015 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.httpd;
+
+import io.netty.buffer.DrillBuf;
+import java.util.Map;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HttpdParserTest {
+
+  private static final Logger LOG = LoggerFactory.getLogger(HttpdParserTest.class);
+
+  private void runTest(String logFormat, String logLine) throws Exception {
+    MapWriter mapWriter = null;
+    DrillBuf managedBuffer = null;
+    Map<String, String> configuredFields = null;
+    HttpdParser parser = new HttpdParser(mapWriter, managedBuffer, logFormat, null, configuredFields);
+    parser.parse(logLine);
+  }
+
+//  @Test
+  public void testFirstPattern() throws Exception {
+    LOG.info("testFirstPattern");
+//    final String format = "common";
+//    final String format = "%h %l %u %t \"%r\" %>s %b";
+    final String format = "%h %t \"%r\" %>s %b \"%{Referer}i\"";
+    final String line = "127.0.0.1 [31/Dec/2012:23:49:41 +0100] "
+        + "\"GET /foo HTTP/1.1\" 200 "
+        + "1213 \"http://localhost/index.php?mies=wim\"";
+    runTest(format, line);
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java
index c341295..2cbc09a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java
@@ -61,6 +61,9 @@ public class TestFormatPluginOptionExtractor {
         case "avro":
           assertEquals(d.typeName, "(type: String)", d.presentParams());
           break;
+        case "httpd":
+          assertEquals("(type: String, logFormat: String, timestampFormat: String)", d.presentParams());
+          break;
         default:
           fail("add validation for format plugin type " + d.typeName);
       }

http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
deleted file mode 100644
index 961d9a6..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2015 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.httpd;
-
-import io.netty.buffer.DrillBuf;
-import java.util.Map;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HttpdParserTest {
-
-  private static final Logger LOG = LoggerFactory.getLogger(HttpdParserTest.class);
-
-  private void runTest(String logFormat, String logLine) throws Exception {
-    MapWriter mapWriter = null;
-    DrillBuf managedBuffer = null;
-    Map<String, String> configuredFields = null;
-    HttpdParser parser = new HttpdParser(mapWriter, managedBuffer, logFormat, null, configuredFields);
-    parser.parse(logLine);
-  }
-
-//  @Test
-  public void testFirstPattern() throws Exception {
-    LOG.info("testFirstPattern");
-//    final String format = "common";
-//    final String format = "%h %l %u %t \"%r\" %>s %b";
-    final String format = "%h %t \"%r\" %>s %b \"%{Referer}i\"";
-    final String line = "127.0.0.1 [31/Dec/2012:23:49:41 +0100] "
-        + "\"GET /foo HTTP/1.1\" 200 "
-        + "1213 \"http://localhost/index.php?mies=wim\"";
-    runTest(format, line);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdLogFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdLogFormatPlugin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdLogFormatPlugin.java
deleted file mode 100644
index 6c2eaf8..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdLogFormatPlugin.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
- * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
- * License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
- * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations under the License.
- */
-package org.apache.drill.exec.store.httpd;
-
-import java.util.List;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.exec.rpc.user.QueryDataBatch;
-import static org.junit.Assert.*;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestHttpdLogFormatPlugin extends BaseTestQuery {
-
-  private static final Logger LOG = LoggerFactory.getLogger(TestHttpdLogFormatPlugin.class);
-
-  /**
-   * This test covers the test bootstrap-storage-plugins.json section of httpd.
-   *
-   * Indirectly this validates the HttpdLogFormatPlugin.HttpdLogFormatConfig deserializing properly.
-   *
-   * @throws Exception
-   */
-  @Test
-  public void testDfsTestBootstrap_star() throws Exception {
-    test("select * from dfs_test.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-test-bootstrap-test.httpd`");
-  }
-
-  /**
-   * This test covers the test bootstrap-storage-plugins.json section of httpd.
-   *
-   * Indirectly this validates the HttpdLogFormatPlugin.HttpdLogFormatConfig deserializing properly.
-   *
-   * @throws Exception
-   */
-  @Test
-  public void testDfsTestBootstrap_notstar() throws Exception {
-    test("select `TIME_STAMP:request_receive_time`, `HTTP_METHOD:request_firstline_method`, `STRING:request_status_last`, `BYTES:response_body_bytesclf` \n"
-        + "from dfs_test.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-test-bootstrap-test.httpd`");
-  }
-
-  /**
-   * This test covers the main bootstrap-storage-plugins.json section of httpd.
-   *
-   * @throws Exception
-   */
-  @Test
-  public void testDfsBootstrap_star() throws Exception {
-    test("select * from dfs.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-bootstrap.httpd`");
-  }
-
-  /**
-   * This test covers the main bootstrap-storage-plugins.json section of httpd.
-   *
-   * @throws Exception
-   */
-  @Test
-  public void testDfsBootstrap_wildcard() throws Exception {
-    test("select `STRING:request_referer_query_$` from dfs.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-bootstrap.httpd`");
-  }
-
-  /**
-   * This test covers the main bootstrap-storage-plugins.json section of httpd.
-   *
-   * @throws Exception
-   */
-  @Test
-  public void testDfsBootstrap_underscore() throws Exception {
-    test("select `TIME_DAY:request_receive_time_day__utc` from dfs.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-bootstrap.httpd`");
-  }
-
-  @Test
-  public void testGroupBy_1() throws Exception {
-    final List<QueryDataBatch> actualResults = testSqlWithResults(
-        "select `HTTP_METHOD:request_firstline_method` as http_method, `STRING:request_status_last` as status_code, sum(`BYTES:response_body_bytesclf`) as total_bytes \n"
-        + "from dfs_test.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-test-bootstrap-test.httpd`\n"
-        + "group by `HTTP_METHOD:request_firstline_method`, `STRING:request_status_last`"
-    );
-
-    final TestResultSet expectedResultSet = new TestResultSet();
-    expectedResultSet.addRow("GET", "200", "46551");
-    expectedResultSet.addRow("POST", "302", "18186");
-
-    TestResultSet actualResultSet = new TestResultSet(actualResults);
-    assertTrue(expectedResultSet.equals(actualResultSet));
-  }
-}

http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdPlugin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdPlugin.java
deleted file mode 100644
index ce1f685..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdPlugin.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.httpd;
-
-import org.apache.drill.BaseTestQuery;
-import org.junit.Test;
-
-public class TestHttpdPlugin extends BaseTestQuery {
-  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestHttpdPlugin.class);
-
-  @Test
-  public void tryBasicQuery() throws Exception {
-    // test("select * from cp.`store/httpd/example1.httpd`");
-    test("select * from dfs.`${WORKING_PATH}/src/test/resources/store/httpd/example1.httpd`");
-  }
-}


Mime
View raw message