ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [13/17] ambari git commit: AMBARI-21144 Create .md files to describe Log Search input configurations (mgergely)
Date Wed, 31 May 2017 14:37:27 GMT
AMBARI-21144 Create .md files to describe Log Search input configurations (mgergely)

Change-Id: I632de5e66607119548ba083fdfed31d39a4e085f


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f0d970b5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f0d970b5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f0d970b5

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: f0d970b5b93a40c2e0830d5eaf3039c81ada2290
Parents: 5845d54
Author: Miklos Gergely <mgergely@hortonworks.com>
Authored: Wed May 31 09:10:00 2017 +0200
Committer: Miklos Gergely <mgergely@hortonworks.com>
Committed: Wed May 31 09:10:00 2017 +0200

----------------------------------------------------------------------
 .../api/model/inputconfig/InputDescriptor.java  |   2 -
 .../inputconfig/impl/InputDescriptorImpl.java   |  12 -
 .../inputconfig/impl/MapDateDescriptorImpl.java |   2 +-
 .../ambari-logsearch-logfeeder/README.md        |  17 +-
 .../ambari-logsearch-logfeeder/docs/filter.md   |  59 ++++
 .../ambari-logsearch-logfeeder/docs/input.md    |  63 ++++
 .../docs/inputConfig.md                         |  29 ++
 .../docs/postMapValues.md                       |  61 ++++
 .../logfeeder/input/AbstractInputFile.java      |   9 +-
 .../ambari/logfeeder/mapper/MapperDate.java     |   2 +-
 .../ambari/logfeeder/input/InputFileTest.java   |   1 -
 .../resources/samples/config/config_audit.json  | 339 +++++++++----------
 .../samples/config/config_service.json          |  79 +++--
 .../logsearch/model/common/LSServerInput.java   |   8 -
 .../logsearch/model/common/LSServerMapDate.java |   2 +-
 .../logfeeder/shipper-conf/global.config.json   |   5 +-
 .../package/templates/global.config.json.j2     |   3 +-
 17 files changed, 442 insertions(+), 251 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
index c41da93..82e9504 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
@@ -38,8 +38,6 @@ public interface InputDescriptor {
 
   Boolean isUseEventMd5AsId();
 
-  String getStartPosition();
-
   Boolean isCacheEnabled();
 
   String getCacheKeyField();

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
index 94dcc2a..54b4b9b 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
@@ -53,10 +53,6 @@ public abstract class InputDescriptorImpl implements InputDescriptor {
   @Expose
   @SerializedName("use_event_md5_as_id")
   private Boolean useEventMd5AsId;
-  
-  @Expose
-  @SerializedName("start_position")
-  private String startPosition;
 
   @Expose
   @SerializedName("cache_enabled")
@@ -146,14 +142,6 @@ public abstract class InputDescriptorImpl implements InputDescriptor
{
     this.useEventMd5AsId = useEventMd5AsId;
   }
 
-  public String getStartPosition() {
-    return startPosition;
-  }
-
-  public void setStartPosition(String startPosition) {
-    this.startPosition = startPosition;
-  }
-
   public Boolean isCacheEnabled() {
     return cacheEnabled;
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
index 9daad2b..2e54e7a 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
@@ -31,7 +31,7 @@ public class MapDateDescriptorImpl implements MapDateDescriptor {
   }
 
   @Expose
-  @SerializedName("source_date_pattern")
+  @SerializedName("src_date_pattern")
   private String sourceDatePattern;
 
   @Expose

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/README.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/README.md b/ambari-logsearch/ambari-logsearch-logfeeder/README.md
index d00eb57..c5c6848 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/README.md
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/README.md
@@ -17,9 +17,10 @@ limitations under the License.
 {% endcomment %}
 -->
 
-#LogSearch - LogFeeder:
+#Log Search - Log Feeder:
 
-Logfeeder is a tool that reads log, parses it and stores it in Apache Solr for analyising
purpose.
+Log Feeder is a component of the Log Search service that reads logs, parses them and stores
them in Apache Solr for the purpose
+of later analysis.
 
 #Compilation
 mvn clean compile package
@@ -34,3 +35,15 @@ mkdir /opt/logfeeder
 cd /opt/logfeeder
 tar xfz ~/logsearch-logfeeder.tar.gz 
 ```
+
+#Input Configuration
+
+The configuration for the log feeder contains
+* description of the log files
+* description of the filters that parse the data of the log entries
+* description of the mappers that modify the parsed fields
+
+The element description can be found [here](docs/inputConfig.md)
+
+All these data are stored in json files, which should be named in the directory /etc/ambari-logsearch-logfeeder/conf,
and the
+name of the files should be input.config-<service\_name>.json
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md
new file mode 100644
index 0000000..42c3271
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md
@@ -0,0 +1,59 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+#Filter
+
+The filter element in the [input configuration](inputConfig.md) contains a list of filter
descriptions, each describing one filter
+applied on an input.
+
+The general elements in the json are the following:
+
+| Field                 | Description                                                   
                                 | Default     |
+|-----------------------|-------------------------------------------------------------------------------------------------|-------------|
+| filter                | The type of the filter, currently grok, keyvalue and json are supported
                        | -           |
+| conditions            | The conditions of which input to filter                       
                                 | -           |
+| sort\_order           | Describes the order in which the filters should be applied    
                                 | -           |
+| source\_field         | The source of the filter, must be set for keyvalue filters    
                                 | log_message |
+| remove\_source\_field | Remove the source field after the filter is applied           
                                 | false       |
+| post\_map\_values     | Mappings done after the filtering provided it's result, see [post
map values](postMapValues.md) | -           |
+| is\_enabled           | A flag to show if the filter should be used                   
                                 | true        |
+
+
+##Grok Filter
+
+Grok filters have the following additional parameters:
+
+| Field              | Description                                                      
                                         | Default |
+|--------------------|------------------------------------------------------------------------------------------------------------|---------|
+| log4j\_format      | The log4j pattern of the log, not used, it is only there for documentation
                                | -       |
+| multiline\_pattern | The grok pattern that shows that the line is not a log line on it's
own but the part of a multi line entry | -       |
+| message\_pattern   | The grok pattern to use to parse the log entry                   
                                         | -       |
+
+
+##Key-value Filter
+
+Key-value filters have the following additional parameters:
+
+| Field          | Description                                                          
                    | Default |
+|----------------|-------------------------------------------------------------------------------------------|---------|
+| field\_split   | The string that splits the key-value pairs                           
                    | "\t"    |
+| value\_split   | The string that separates keys from values                           
                    | "="     |
+| value\_borders | The borders around the value, must be 2 characters long, first before
it, second after it | -       |
+
+If value\_borders is only used if it is specified, and value\_split is not.

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md
new file mode 100644
index 0000000..e95401a
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md
@@ -0,0 +1,63 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+#Input
+
+The input element in the [input configuration](inputConfig.md) contains a list of input descriptions,
each describing one source
+of input.
+
+The general elements in the json are the following:
+
+| Field                       | Description                                             
                                             | Default      |
+|-----------------------------|-------------------------------------------------------------------------------------------------------|--------------|
+| type                        | The type of the input source, currently file and s3_file
are supported                                | -            |
+| rowtype                     | The type of the row, can be service / audit             
                                             | -            |
+| path                        | The path of the source, may contain '*' characters too  
                                             | -            |
+| add\_fields                 | The element contains field\_name: field\_value pairs which
will be added to each rows data            | -            |
+| tail                        | The input should check for only the latest file matching
the pattern, not all of them                 | true         |
+| gen\_event\_md5             | Generate an event\_md5 field for each row by creating a hash
of the row data                          | true         |
+| use\_event\_md5\_as\_id     | Generate an id for each row by creating a hash of the row
data                                        | false        |
+| start\_position             | Should the parsing start from the beginning             
                                             | beginning    |
+| cache\_enabled              | Allows the input to use a cache to filter out duplications
                                           | true         |
+| cache\_key\_field           | Specifies the field for which to use the cache to find duplications
of                                | log\_message |
+| cache\_last\_dedup\_enabled | Allow to filter out entries which are same as the most recent
one irrelevant of it's time             | false        |
+| cache\_size                 | The number of entries to store in the cache             
                                             | 100          |
+| cache\_dedup\_interval      | The maximum interval in ms which may pass between two identical
log messages to filter the latter out | 1000         |
+| is\_enabled                 | A flag to show if the input should be used              
                                             | true         |
+
+
+##File Input
+
+File inputs have the following parameters too:
+
+| Field                    | Description                                                
       | Default |
+|--------------------------|--------------------------------------------------------------------|---------|
+| checkpoint\_interval\_ms | The time interval in ms when the checkpoint file should be updated
| 5000    |
+| process\_file            | Should the file be processed                               
       | true    |
+| copy\_file               | Should the file be copied \(only if not processed\)        
       | false   |
+
+
+##S3 File Input
+
+S3 file inputs have the following parameters in addition to the general file parameters:
+
+| Field           | Description                             | Default |
+|-----------------|-----------------------------------------|---------|
+| s3\_access\_key | The access key used for AWS credentials | -       |
+| s3\_secret\_key | The secret key used for AWS credentials | -       |

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/docs/inputConfig.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/inputConfig.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/inputConfig.md
new file mode 100644
index 0000000..ae7ac05
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/inputConfig.md
@@ -0,0 +1,29 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+#Input Configuration
+
+The input configurations are stored in json files. Each of them are describing the processing
of the log files of a service.
+
+The json contains two elements:
+
+| Field  | Description                                |
+|--------|--------------------------------------------|
+| input  | A list of [input descriptions](input.md)   |
+| filter | A list of [filter descriptions](filter.md) |
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md
new file mode 100644
index 0000000..aa381cd
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md
@@ -0,0 +1,61 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+#Post Map Values
+
+The Post Map Values element in the [filter](filter.md) field names as keys, the values are
lists of sets of post map values, each
+describing one mapping done on a field named before obtained after filtering.
+
+Currently there are four kind of mappings are supported:
+
+##Map Date
+
+The name of the mapping element should be map\_date. The value json element may contain the
following parameters:
+
+| Field                 | Description                                                   
                                        |
+|-----------------------|--------------------------------------------------------------------------------------------------------|
+| src\_date\_pattern    | If it is specified than the mapper converts from this format to
the target, and also adds missing year |
+| target\_date\_pattern | If 'epoch' then the field is parsed as seconds from 1970, otherwise
the content used as pattern        |
+
+
+##Map Copy
+
+The name of the mapping element should be map\_copy. The value json element should contain
the following parameter:
+
+| Field      | Description                   |
+|------------|-------------------------------|
+| copy\_name | The name of the copied field  |
+
+
+##Map Field Name
+
+The name of the mapping element should be map\_fieldname. The value json element should contain
the following parameter:
+
+| Field            | Description                   |
+|------------------|-------------------------------|
+| new\_field\_name | The name of the renamed field |
+
+##Map Field Value
+
+The name of the mapping element should be map\_fieldvalue. The value json element should
contain the following parameter:
+
+| Field       | Description                                                        |
+|-------------|--------------------------------------------------------------------|
+| pre\_value  | The value that the field must match \(ignoring case\) to be mapped |
+| post\_value | The value to which the field is modified to                        |

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
index cfa1903..2bc4e68 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
@@ -45,7 +45,6 @@ public abstract class AbstractInputFile extends Input {
   protected String base64FileKey;
 
   protected boolean isReady;
-  private boolean isStartFromBegining = true;
 
   private String checkPointExtension;
   private File checkPointFile;
@@ -82,12 +81,6 @@ public abstract class AbstractInputFile extends Input {
       return;
     }
 
-    String startPosition = inputDescriptor.getStartPosition();
-    if (StringUtils.isEmpty(startPosition) || startPosition.equalsIgnoreCase("beginning")
||
-        startPosition.equalsIgnoreCase("begining") || !tail) {
-      isStartFromBegining = true;
-    }
-
     setFilePath(logPath);
     boolean isFileReady = isReady();
 
@@ -109,7 +102,7 @@ public abstract class AbstractInputFile extends Input {
       
       br = openLogFile(logPathFile);
 
-      boolean resume = isStartFromBegining;
+      boolean resume = true;
       int resumeFromLineNumber = getResumeFromLineNumber();
       if (resumeFromLineNumber > 0) {
         resume = false;

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
index 5d34c06..305688b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
@@ -38,7 +38,7 @@ public class MapperDate extends Mapper {
 
   private SimpleDateFormat targetDateFormatter = null;
   private boolean isEpoch = false;
-  private SimpleDateFormat srcDateFormatter=null;
+  private SimpleDateFormat srcDateFormatter = null;
 
   @Override
   public boolean init(String inputDesc, String fieldName, String mapClassCode, MapFieldDescriptor
mapFieldDescriptor) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
index 3a5f31e..efebc08 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
@@ -81,7 +81,6 @@ public class InputFileTest {
     inputFileDescriptor.setSource("file");
     inputFileDescriptor.setTail(true);
     inputFileDescriptor.setGenEventMd5(true);
-    inputFileDescriptor.setStartPosition("beginning");
     inputFileDescriptor.setType("hdfs_datanode");
     inputFileDescriptor.setRowtype("service");
     inputFileDescriptor.setPath(path);

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json
b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json
index b7d0a6e..05f3bff 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json
@@ -1,173 +1,172 @@
 {
-	"global": {
-		"add_fields": {
-			"cluster": "audit"
-		},
-		"source": "file",
-		"tail": "true",
-		"gen_event_md5": "true",
-		"start_position": "beginning"
-	},
-	"input": [{
-		"type": "hdfs_audit",
-		"rowtype": "hdfs_audit",
-		"path": "{path}/src/test/resources/samples/jsonlogs/audit_log.json"
-	}],
-	"filter": [{
-			"filter": "json",
-			"conditions": {
-				"fields": {
-					"type": [
-						"hdfs_audit"
-					]
-
-				}
-			}
-		}, {
-			"filter": "keyvalue",
-			"sort_order": 1,
-			"conditions": {
-				"fields": {
-					"type": [
-						"hdfs_audit"
-					]
-
-				}
-
-			},
-			"source_field": "log_message",
-			"value_split": "=",
-			"field_split": "\t",
-			"post_map_values": {
-				"src": {
-					"map_fieldname": {
-						"new_fieldname": "resource"
-					}
-
-				},
-				"ip": {
-					"map_fieldname": {
-						"new_fieldname": "cliIP"
-					}
-
-				},
-				"allowed": [{
-						"map_fieldvalue": {
-							"pre_value": "true",
-							"post_value": "1"
-						}
-
-					}, {
-						"map_fieldvalue": {
-							"pre_value": "false",
-							"post_value": "0"
-						}
-
-					}, {
-						"map_fieldname": {
-							"new_fieldname": "result"
-						}
-
-					}
-
-				],
-				"cmd": {
-					"map_fieldname": {
-						"new_fieldname": "action"
-					}
-
-				},
-				"proto": {
-					"map_fieldname": {
-						"new_fieldname": "cliType"
-					}
-
-				},
-				"callerContext": {
-					"map_fieldname": {
-						"new_fieldname": "req_caller_id"
-					}
-
-				}
-
-			}
-
-		}, {
-			"filter": "grok",
-			"sort_order": 2,
-			"source_field": "ugi",
-			"remove_source_field": "false",
-			"conditions": {
-				"fields": {
-					"type": [
-						"hdfs_audit"
-					]
-
-				}
-
-			},
-			"message_pattern": "%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}",
-			"post_map_values": {
-				"user": {
-					"map_fieldname": {
-						"new_fieldname": "reqUser"
-					}
-
-				},
-				"x_user": {
-					"map_fieldname": {
-						"new_fieldname": "reqUser"
-					}
-
-				},
-				"p_user": {
-					"map_fieldname": {
-						"new_fieldname": "reqUser"
-					}
-
-				},
-				"k_user": {
-					"map_fieldname": {
-						"new_fieldname": "proxyUsers"
-					}
-
-				},
-				"p_authType": {
-					"map_fieldname": {
-						"new_fieldname": "authType"
-					}
-
-				},
-				"k_authType": {
-					"map_fieldname": {
-						"new_fieldname": "proxyAuthType"
-					}
-
-				}
-
-			}
-
-		}
-
-	],
-
-	"output": [{
-			"is_enabled": "true",
-			"comment": "Output to file for audit logs",
-			"destination": "solr",
-			"url": "http://localhost:8983/solr/audit_logs",
-			"collection": "audit_logs",
-			"number_of_shards": "1",
-			"splits_interval_mins": "100000",
-			"conditions": {
-				"fields": {
-					"rowtype": [
-						"hdfs_audit"
-					]
-				}
-			}
-		}
-
-	]
+  "global": {
+    "add_fields": {
+      "cluster": "audit"
+    },
+    "source": "file",
+    "tail": "true",
+    "gen_event_md5": "true"
+  },
+  "input": [{
+    "type": "hdfs_audit",
+    "rowtype": "hdfs_audit",
+    "path": "{path}/src/test/resources/samples/jsonlogs/audit_log.json"
+  }],
+  "filter": [{
+      "filter": "json",
+      "conditions": {
+        "fields": {
+          "type": [
+            "hdfs_audit"
+          ]
+
+        }
+      }
+    }, {
+      "filter": "keyvalue",
+      "sort_order": 1,
+      "conditions": {
+        "fields": {
+          "type": [
+            "hdfs_audit"
+          ]
+
+        }
+
+      },
+      "source_field": "log_message",
+      "value_split": "=",
+      "field_split": "\t",
+      "post_map_values": {
+        "src": {
+          "map_fieldname": {
+            "new_fieldname": "resource"
+          }
+
+        },
+        "ip": {
+          "map_fieldname": {
+            "new_fieldname": "cliIP"
+          }
+
+        },
+        "allowed": [{
+            "map_fieldvalue": {
+              "pre_value": "true",
+              "post_value": "1"
+            }
+
+          }, {
+            "map_fieldvalue": {
+              "pre_value": "false",
+              "post_value": "0"
+            }
+
+          }, {
+            "map_fieldname": {
+              "new_fieldname": "result"
+            }
+
+          }
+
+        ],
+        "cmd": {
+          "map_fieldname": {
+            "new_fieldname": "action"
+          }
+
+        },
+        "proto": {
+          "map_fieldname": {
+            "new_fieldname": "cliType"
+          }
+
+        },
+        "callerContext": {
+          "map_fieldname": {
+            "new_fieldname": "req_caller_id"
+          }
+
+        }
+
+      }
+
+    }, {
+      "filter": "grok",
+      "sort_order": 2,
+      "source_field": "ugi",
+      "remove_source_field": "false",
+      "conditions": {
+        "fields": {
+          "type": [
+            "hdfs_audit"
+          ]
+
+        }
+
+      },
+      "message_pattern": "%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}",
+      "post_map_values": {
+        "user": {
+          "map_fieldname": {
+            "new_fieldname": "reqUser"
+          }
+
+        },
+        "x_user": {
+          "map_fieldname": {
+            "new_fieldname": "reqUser"
+          }
+
+        },
+        "p_user": {
+          "map_fieldname": {
+            "new_fieldname": "reqUser"
+          }
+
+        },
+        "k_user": {
+          "map_fieldname": {
+            "new_fieldname": "proxyUsers"
+          }
+
+        },
+        "p_authType": {
+          "map_fieldname": {
+            "new_fieldname": "authType"
+          }
+
+        },
+        "k_authType": {
+          "map_fieldname": {
+            "new_fieldname": "proxyAuthType"
+          }
+
+        }
+
+      }
+
+    }
+
+  ],
+
+  "output": [{
+      "is_enabled": "true",
+      "comment": "Output to file for audit logs",
+      "destination": "solr",
+      "url": "http://localhost:8983/solr/audit_logs",
+      "collection": "audit_logs",
+      "number_of_shards": "1",
+      "splits_interval_mins": "100000",
+      "conditions": {
+        "fields": {
+          "rowtype": [
+            "hdfs_audit"
+          ]
+        }
+      }
+    }
+
+  ]
 
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json
b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json
index 2e96fd7..0fff050 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json
@@ -1,43 +1,42 @@
 {
-	"global": {
-		"add_fields": {
-			"cluster": "cluster_name"
-		},
-		"source": "file",
-		"tail": "true",
-		"gen_event_md5": "true",
-		"start_position": "beginning"
-	},
-	"input": [{
-		"type": "logsearch",
-		"rowtype": "service",
-		"path": "{path}/src/test/resources/samples/jsonlogs/service_log.json"
-	}],
-	"filter": [{
-		"filter": "json",
-		"conditions": {
-			"fields": {
-				"type": [
-					"logsearch"
-				]
-			}
-		}
-	}],
-	"output": [{
-		"comment": "Output to solr for service records",
-		"is_enabled": "true",
-		"destination": "solr",
-		"url": "http://localhost:8983/solr/hadoop_logs",
-		"collection": "hadoop_logs",
-		"number_of_shards": "1",
-		"splits_interval_mins": "100000",
-		"conditions": {
-			"fields": {
-				"rowtype": [
-					"service"
-				]
-			}
-		}
-	}]
+  "global": {
+    "add_fields": {
+      "cluster": "cluster_name"
+    },
+    "source": "file",
+    "tail": "true",
+    "gen_event_md5": "true"
+  },
+  "input": [{
+    "type": "logsearch",
+    "rowtype": "service",
+    "path": "{path}/src/test/resources/samples/jsonlogs/service_log.json"
+  }],
+  "filter": [{
+    "filter": "json",
+    "conditions": {
+      "fields": {
+        "type": [
+          "logsearch"
+        ]
+      }
+    }
+  }],
+  "output": [{
+    "comment": "Output to solr for service records",
+    "is_enabled": "true",
+    "destination": "solr",
+    "url": "http://localhost:8983/solr/hadoop_logs",
+    "collection": "hadoop_logs",
+    "number_of_shards": "1",
+    "splits_interval_mins": "100000",
+    "conditions": {
+      "fields": {
+        "rowtype": [
+          "service"
+        ]
+      }
+    }
+  }]
 
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java
b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java
index e0f7564..6ef3d3f 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java
@@ -53,9 +53,6 @@ public abstract class LSServerInput {
   @JsonProperty("use_event_md5_as_id")
   private Boolean useEventMd5AsId;
   
-  @JsonProperty("start_position")
-  private String startPosition;
-  
   @JsonProperty("cache_enabled")
   private Boolean cacheEnabled;
   
@@ -85,7 +82,6 @@ public abstract class LSServerInput {
     this.tail = inputDescriptor.isTail();
     this.genEventMd5 = inputDescriptor.isGenEventMd5();
     this.useEventMd5AsId = inputDescriptor.isUseEventMd5AsId();
-    this.startPosition = inputDescriptor.getStartPosition();
     this.cacheEnabled = inputDescriptor.isCacheEnabled();
     this.cacheKeyField = inputDescriptor.getCacheKeyField();
     this.cacheLastDedupEnabled = inputDescriptor.getCacheLastDedupEnabled();
@@ -126,10 +122,6 @@ public abstract class LSServerInput {
     return useEventMd5AsId;
   }
 
-  public String getStartPosition() {
-    return startPosition;
-  }
-
   public Boolean getCacheEnabled() {
     return cacheEnabled;
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java
b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java
index 3147402..dcb0393 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java
@@ -34,7 +34,7 @@ public class LSServerMapDate extends LSServerMapField {
     return "map_date";
   }
 
-  @JsonProperty("source_date_pattern")
+  @JsonProperty("src_date_pattern")
   private String sourceDatePattern;
 
   @NotNull

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json
b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json
index 7ac29dd..65da726 100644
--- a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json
+++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json
@@ -5,7 +5,6 @@
     },
     "source":"file",
     "tail":"true",
-    "gen_event_md5":"true",
-    "start_position":"beginning"
+    "gen_event_md5":"true"
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2
index f337527..1b63e2a 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2
@@ -22,7 +22,6 @@
     },
     "source":"file",
     "tail":"true",
-    "gen_event_md5":"true",
-    "start_position":"beginning"
+    "gen_event_md5":"true"
   }
 }
\ No newline at end of file


Mime
View raw message