ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mgerg...@apache.org
Subject ambari git commit: AMBARI-21423 Add REST end point for the documentation of the Log Feeder shipper properties (mgergely)
Date Mon, 10 Jul 2017 12:51:33 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk c0882898d -> 15dd999ff


AMBARI-21423 Add REST end point for the documentation of the Log Feeder shipper properties (mgergely)

Change-Id: If6d1b66c3a1f74b118ae60a7edc26624d49fb7e6


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/15dd999f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/15dd999f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/15dd999f

Branch: refs/heads/trunk
Commit: 15dd999fff99fb80bc65ddfc94513e890a6efdef
Parents: c088289
Author: Miklos Gergely <mgergely@hortonworks.com>
Authored: Mon Jul 10 14:51:23 2017 +0200
Committer: Miklos Gergely <mgergely@hortonworks.com>
Committed: Mon Jul 10 14:51:23 2017 +0200

----------------------------------------------------------------------
 .../api/ShipperConfigElementDescription.java    |  59 +++++++++++
 .../api/ShipperConfigTypeDescription.java       |  44 ++++++++
 .../model/inputconfig/impl/ConditionsImpl.java  |  13 +++
 .../model/inputconfig/impl/FieldsImpl.java      |  14 +++
 .../inputconfig/impl/FilterDescriptorImpl.java  |  51 ++++++++++
 .../impl/FilterGrokDescriptorImpl.java          |  24 +++++
 .../impl/FilterKeyValueDescriptorImpl.java      |  28 +++++
 .../model/inputconfig/impl/InputConfigImpl.java |  18 ++++
 .../inputconfig/impl/InputDescriptorImpl.java   | 101 +++++++++++++++++++
 .../impl/InputFileBaseDescriptorImpl.java       |  27 +++++
 .../impl/InputS3FileDescriptorImpl.java         |  16 +++
 .../impl/MapAnonymizeDescriptorImpl.java        |  21 +++-
 .../inputconfig/impl/MapDateDescriptorImpl.java |  20 +++-
 .../impl/MapFieldCopyDescriptorImpl.java        |  14 ++-
 .../impl/MapFieldDescriptorImpl.java            |  33 ++++++
 .../impl/MapFieldNameDescriptorImpl.java        |  14 ++-
 .../impl/MapFieldValueDescriptorImpl.java       |  20 +++-
 .../inputconfig/impl/PostMapValuesAdapter.java  |   2 +-
 .../ambari-logsearch-logfeeder/docs/filter.md   |   4 +-
 .../ambari-logsearch-logfeeder/docs/input.md    |  10 +-
 .../docs/postMapValues.md                       |   2 +-
 .../ambari/logfeeder/filter/FilterJSONTest.java |  12 ++-
 .../common/ShipperConfigDescriptionStorage.java |  67 ++++++++++++
 .../ambari/logsearch/doc/DocConstants.java      |   1 +
 .../ambari/logsearch/manager/InfoManager.java   |   9 ++
 .../response/ShipperConfigDescriptionData.java  |  52 ++++++++++
 .../ambari/logsearch/rest/InfoResource.java     |  10 ++
 27 files changed, 667 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigElementDescription.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigElementDescription.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigElementDescription.java
new file mode 100644
index 0000000..d65bf8e
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigElementDescription.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.api;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker for the shipper configuration properties.
+ * Can be used to generate documentation about the shipper configs.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.FIELD})
+public @interface ShipperConfigElementDescription {
+
+  /**
+   * The path of the json element.
+   */
+  String path();
+
+  /**
+   * The type of the json element.
+   */
+  String type();
+
+  /**
+   * Describe what the json element is used for.
+   */
+  String description();
+
+  /**
+   * An example value for the element, if applicable.
+   */
+  String[] examples() default {};
+
+  /**
+   * Default value of the json element, if applicable.
+   */
+  String defaultValue() default "";
+
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigTypeDescription.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigTypeDescription.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigTypeDescription.java
new file mode 100644
index 0000000..1c112d8
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigTypeDescription.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.api;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker for the shipper configuration types.
+ * Can be used to generate documentation about the shipper configs.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE})
+public @interface ShipperConfigTypeDescription {
+
+  /**
+   * The name of the element type.
+   */
+  String name();
+
+  /**
+   * The description of the json element.
+   */
+  String description();
+
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java
index 8bbff8f..2ba472c 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java
@@ -19,11 +19,24 @@
 
 package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
 
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.Conditions;
 
 import com.google.gson.annotations.Expose;
 
+@ShipperConfigTypeDescription(
+  name = "Conditions",
+  description = "Describes the conditions that should be met in order to match a filter to an input element.\n" +
+                "\n" +
+                "It has the following attributes:"
+)
 public class ConditionsImpl implements Conditions {
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/conditions/fields",
+    type = "json object",
+    description = "The fields in the input element of which's value should be met."
+  )
   @Expose
   private FieldsImpl fields;
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java
index 68cd0e2..32a0348 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java
@@ -21,11 +21,25 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
 
 import java.util.Set;
 
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.Fields;
 
 import com.google.gson.annotations.Expose;
 
+@ShipperConfigTypeDescription(
+    name = "Fields",
+    description = "Describes a the fields which's value should be met in order to match a filter to an input element.\n" +
+                  "\n" +
+                  "It has the following attributes:"
+  )
 public class FieldsImpl implements Fields {
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/conditions/fields/type",
+    type = "list of strings",
+    description = "The acceptable values for the type field in the input element.",
+    examples = {"ambari_server", "\"spark_jobhistory_server\", \"spark_thriftserver\", \"livy_server\""}
+  )
   @Expose
   private Set<String> type;
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java
index 4e11715..eb9d38c 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java
@@ -22,35 +22,86 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.PostMapValues;
 
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
+@ShipperConfigTypeDescription(
+  name = "Filter",
+  description = "The filter element in the [input configuration](inputConfig.md) contains a list of filter descriptions, each describing one filter applied on an input.\n" +
+                "\n" +
+                "The general elements in the json are the following:"
+)
 public abstract class FilterDescriptorImpl implements FilterDescriptor {
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/filter",
+    type = "string",
+    description = "The type of the filter.",
+    examples = {"grok", "keyvalue", "json"}
+  )
   @Expose
   private String filter;
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/conditions",
+    type = "json object",
+    description = "The conditions of which input to filter."
+  )
   @Expose
   private ConditionsImpl conditions;
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/sort_order",
+    type = "integer",
+    description = "Describes the order in which the filters should be applied.",
+    examples = {"1", "3"}
+  )
   @Expose
   @SerializedName("sort_order")
   private Integer sortOrder;
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/source_field",
+    type = "integer",
+    description = "The source of the filter, must be set for keyvalue filters.",
+    examples = {"field_further_to_filter"},
+    defaultValue = "log_message"
+  )
   @Expose
   @SerializedName("source_field")
   private String sourceField;
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/remove_source_field",
+    type = "boolean",
+    description = "Remove the source field after the filter is applied.",
+    examples = {"true", "false"},
+    defaultValue = "false"
+  )
   @Expose
   @SerializedName("remove_source_field")
   private Boolean removeSourceField;
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/post_map_values",
+    type = "dictionary string to list of json objects",
+    description = "Mappings done after the filtering provided it's result."
+  )
   @Expose
   @SerializedName("post_map_values")
   private Map<String, List<PostMapValuesImpl>> postMapValues;
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/is_enabled",
+    type = "boolean",
+    description = "A flag to show if the filter should be used.",
+    examples = {"true", "false"},
+    defaultValue = "true"
+  )
   @Expose
   @SerializedName("is_enabled")
   private Boolean isEnabled;

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java
index 995f76b..e140df0 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java
@@ -19,20 +19,44 @@
 
 package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
 
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterGrokDescriptor;
 
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
+@ShipperConfigTypeDescription(
+  name = "Grok Filter",
+  description = "Grok filters have the following additional parameters:"
+)
 public class FilterGrokDescriptorImpl extends FilterDescriptorImpl implements FilterGrokDescriptor {
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/log4j_format",
+    type = "string",
+    description = "The log4j pattern of the log, not used, it is only there for documentation.",
+    examples = {"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n"}
+  )
   @Expose
   @SerializedName("log4j_format")
   private String log4jFormat;
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/multiline_pattern",
+    type = "string",
+    description = "The grok pattern that shows that the line is not a log line on it's own but the part of a multi line entry.",
+    examples = {"^(%{TIMESTAMP_ISO8601:logtime})"}
+  )
   @Expose
   @SerializedName("multiline_pattern")
   private String multilinePattern;
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/message_pattern",
+    type = "string",
+    description = "The grok pattern to use to parse the log entry.",
+    examples = {"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}"}
+  )
   @Expose
   @SerializedName("message_pattern")
   private String messagePattern;

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java
index 8e89990..1c782c5 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java
@@ -19,20 +19,48 @@
 
 package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
 
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterKeyValueDescriptor;
 
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
+@ShipperConfigTypeDescription(
+    name = "Key-value Filter",
+    description = "value_borders is only used if it is specified, and value_split is not.\n" +
+                  "\n" +
+                  "Key-value filters have the following additional parameters:"
+)
 public class FilterKeyValueDescriptorImpl extends FilterDescriptorImpl implements FilterKeyValueDescriptor {
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/field_split",
+    type = "string",
+    description = "The string that splits the key-value pairs.",
+    examples = {" ", ","},
+    defaultValue = "\\t"
+  )
   @Expose
   @SerializedName("field_split")
   private String fieldSplit;
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/value_split",
+    type = "string",
+    description = "The string that separates keys from values.",
+    examples = {":", "->"},
+    defaultValue = "="
+  )
   @Expose
   @SerializedName("value_split")
   private String valueSplit;
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/value_borders",
+    type = "string",
+    description = "The borders around the value, must be 2 characters long, first before it, second after it.",
+    examples = {"()", "[]", "{}"}
+  )
   @Expose
   @SerializedName("value_borders")
   private String valueBorders;

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java
index a4eba8e..6ce634f 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java
@@ -21,16 +21,34 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
 
 import java.util.List;
 
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor;
 
 import com.google.gson.annotations.Expose;
 
+@ShipperConfigTypeDescription(
+  name = "Input Config",
+  description = "The input configurations are stored in json files. Each of them are describing the processing of the log files of a service.\n" +
+                "\n" +
+                "The json contains two elements:"
+)
 public class InputConfigImpl implements InputConfig {
+  @ShipperConfigElementDescription(
+    path = "/input",
+    type = "list of json objects",
+    description = "A list of input descriptions"
+  )
   @Expose
   private List<InputDescriptorImpl> input;
 
+  @ShipperConfigElementDescription(
+    path = "/filter",
+    type = "list of json objects",
+    description = "A list of filter descriptions"
+  )
   @Expose
   private List<FilterDescriptorImpl> filter;
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
index 54b4b9b..cec16c8 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
@@ -21,59 +21,160 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
 
 import java.util.Map;
 
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor;
 
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
+@ShipperConfigTypeDescription(
+  name = "Input",
+  description = "The input element in the input configuration contains a list of input descriptions, each describing one source of input.\n" +
+                "\n" +
+                "The general elements in the json are the following:"
+)
 public abstract class InputDescriptorImpl implements InputDescriptor {
+  @ShipperConfigElementDescription(
+    path = "/input/[]/type",
+    type = "string",
+    description = "The log id for this source.",
+    examples = {"zookeeper", "ambari_server"}
+  )
   @Expose
   private String type;
 
+  @ShipperConfigElementDescription(
+    path = "/input/[]/rowtype",
+    type = "string",
+    description = "The type of the row.",
+    examples = {"service", "audit"}
+  )
   @Expose
   private String rowtype;
 
+  @ShipperConfigElementDescription(
+    path = "/input/[]/path",
+    type = "string",
+    description = "The path of the source, may contain '*' characters too.",
+    examples = {"/var/log/ambari-logsearch-logfeeder/logsearch-logfeeder.json", "/var/log/zookeeper/zookeeper*.log"}
+  )
   @Expose
   private String path;
 
+  @ShipperConfigElementDescription(
+    path = "/input/[]/add_fields",
+    type = "dictionary",
+    description = "The element contains field_name: field_value pairs which will be added to each rows data.",
+    examples = {"\"cluster\":\"cluster_name\""}
+  )
   @Expose
   @SerializedName("add_fields")
   private Map<String, String> addFields;
   
+  @ShipperConfigElementDescription(
+    path = "/input/[]/source",
+    type = "dictionary",
+    description = "The type of the input source.",
+    examples = {"file", "s3_file"}
+  )
   @Expose
   private String source;
   
+  @ShipperConfigElementDescription(
+    path = "/input/[]/tail",
+    type = "boolean",
+    description = "The input should check for only the latest file matching the pattern, not all of them.",
+    examples = {"true", "false"},
+    defaultValue = "true"
+  )
   @Expose
   private Boolean tail;
   
+  @ShipperConfigElementDescription(
+    path = "/input/[]/gen_event_md5",
+    type = "boolean",
+    description = "Generate an event_md5 field for each row by creating a hash of the row data.",
+    examples = {"true", "false"},
+    defaultValue = "true"
+  )
   @Expose
   @SerializedName("gen_event_md5")
   private Boolean genEventMd5;
   
+  @ShipperConfigElementDescription(
+    path = "/input/[]/use_event_md5_as_id",
+    type = "boolean",
+    description = "Generate an id for each row by creating a hash of the row data.",
+    examples = {"true", "false"},
+    defaultValue = "false"
+  )
   @Expose
   @SerializedName("use_event_md5_as_id")
   private Boolean useEventMd5AsId;
 
+  @ShipperConfigElementDescription(
+    path = "/input/[]/cache_enabled",
+    type = "boolean",
+    description = "Allows the input to use a cache to filter out duplications.",
+    examples = {"true", "false"},
+    defaultValue = "false"
+  )
   @Expose
   @SerializedName("cache_enabled")
   private Boolean cacheEnabled;
 
+  @ShipperConfigElementDescription(
+    path = "/input/[]/cache_key_field",
+    type = "string",
+    description = "Specifies the field for which to use the cache to find duplications of.",
+    examples = {"some_field_prone_to_repeating_value"},
+    defaultValue = "log_message"
+  )
   @Expose
   @SerializedName("cache_key_field")
   private String cacheKeyField;
 
+  @ShipperConfigElementDescription(
+    path = "/input/[]/cache_last_dedup_enabled",
+    type = "boolean",
+    description = "Allow to filter out entries which are same as the most recent one irrelevant of it's time.",
+    examples = {"true", "false"},
+    defaultValue = "false"
+  )
   @Expose
   @SerializedName("cache_last_dedup_enabled")
   private Boolean cacheLastDedupEnabled;
 
+  @ShipperConfigElementDescription(
+    path = "/input/[]/cache_size",
+    type = "integer",
+    description = "The number of entries to store in the cache.",
+    examples = {"50"},
+    defaultValue = "100"
+  )
   @Expose
   @SerializedName("cache_size")
   private Integer cacheSize;
 
+  @ShipperConfigElementDescription(
+    path = "/input/[]/cache_dedup_interval",
+    type = "integer",
+    description = "The maximum interval in ms which may pass between two identical log messages to filter the latter out.",
+    examples = {"500"},
+    defaultValue = "1000"
+  )
   @Expose
   @SerializedName("cache_dedup_interval")
   private Long cacheDedupInterval;
 
+  @ShipperConfigElementDescription(
+    path = "/input/[]/is_enabled",
+    type = "boolean",
+    description = "A flag to show if the input should be used.",
+    examples = {"true", "false"},
+    defaultValue = "true"
+  )
   @Expose
   @SerializedName("is_enabled")
   private Boolean isEnabled;

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java
index 51c7ec8..8281daa 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java
@@ -19,20 +19,47 @@
 
 package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
 
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.InputFileBaseDescriptor;
 
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
+@ShipperConfigTypeDescription(
+  name = "File Input",
+  description = "File inputs have some additional parameters:"
+)
 public class InputFileBaseDescriptorImpl extends InputDescriptorImpl implements InputFileBaseDescriptor {
+  @ShipperConfigElementDescription(
+    path = "/input/[]/checkpoint_interval_ms",
+    type = "integer",
+    description = "The time interval in ms when the checkpoint file should be updated.",
+    examples = {"10000"},
+    defaultValue = "5000"
+  )
   @Expose
   @SerializedName("checkpoint_interval_ms")
   private Integer checkpointIntervalMs;
 
+  @ShipperConfigElementDescription(
+    path = "/input/[]/process_file",
+    type = "boolean",
+    description = "Should the file be processed.",
+    examples = {"true", "false"},
+    defaultValue = "true"
+  )
   @Expose
   @SerializedName("process_file")
   private Boolean processFile;
 
+  @ShipperConfigElementDescription(
+    path = "/input/[]/copy_file",
+    type = "boolean",
+    description = "Should the file be copied (only if not processed).",
+    examples = {"true", "false"},
+    defaultValue = "false"
+  )
   @Expose
   @SerializedName("copy_file")
   private Boolean copyFile;

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java
index 277a57c..19f52d3 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java
@@ -19,16 +19,32 @@
 
 package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
 
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.InputS3FileDescriptor;
 
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
+@ShipperConfigTypeDescription(
+  name = "S3 File Input",
+  description = "S3 file inputs have the following parameters in addition to the general file parameters:"
+)
 public class InputS3FileDescriptorImpl extends InputFileBaseDescriptorImpl implements InputS3FileDescriptor {
+  @ShipperConfigElementDescription(
+    path = "/input/[]/s3_access_key",
+    type = "string",
+    description = "The access key used for AWS credentials."
+  )
   @Expose
   @SerializedName("s3_access_key")
   private String s3AccessKey;
 
+  @ShipperConfigElementDescription(
+    path = "/input/[]/s3_secret_key",
+    type = "string",
+    description = "The secret key used for AWS credentials."
+  )
   @Expose
   @SerializedName("s3_secret_key")
   private String s3SecretKey;

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java
index 5fdbbab..8c128de 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java
@@ -19,20 +19,39 @@
 
 package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
 
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.MapAnonymizeDescriptor;
 
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
-public class MapAnonymizeDescriptorImpl implements MapAnonymizeDescriptor {
+@ShipperConfigTypeDescription(
+    name = "Map Anonymize",
+    description = "The name of the mapping element should be map_anonymize. The value json element should contain the following parameter:"
+)
+public class MapAnonymizeDescriptorImpl extends MapFieldDescriptorImpl implements MapAnonymizeDescriptor {
   @Override
   public String getJsonName() {
     return "map_anonymize";
   }
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/post_map_values/{field_name}/[]/map_anonymize/pattern",
+    type = "string",
+    description = "The pattern to use to identify parts to anonymize. The parts to hide should be marked with the \"<hide>\" string.",
+    examples = {"Some secret is here: <hide>, and another one is here: <hide>"}
+  )
   @Expose
   private String pattern;
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/post_map_values/{field_name}/[]/map_anonymize/hide_char",
+    type = "string",
+    description = "The character to hide with",
+    defaultValue = "*",
+    examples = {"X", "-"}
+  )
   @Expose
   @SerializedName("hide_char")
   private Character hideChar;

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
index 2e54e7a..feec4b6 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
@@ -19,21 +19,39 @@
 
 package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
 
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.MapDateDescriptor;
 
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
-public class MapDateDescriptorImpl implements MapDateDescriptor {
+@ShipperConfigTypeDescription(
+    name = "Map Date",
+    description = "The name of the mapping element should be map_date. The value json element may contain the following parameters:"
+)
+public class MapDateDescriptorImpl extends MapFieldDescriptorImpl implements MapDateDescriptor {
   @Override
   public String getJsonName() {
     return "map_date";
   }
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/post_map_values/{field_name}/[]/map_date/src_date_pattern",
+    type = "string",
+    description = "If it is specified than the mapper converts from this format to the target, and also adds missing year",
+    examples = {"MMM dd HH:mm:ss"}
+  )
   @Expose
   @SerializedName("src_date_pattern")
   private String sourceDatePattern;
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/post_map_values/{field_name}/[]/map_date/target_date_pattern",
+    type = "string",
+    description = "If 'epoch' then the field is parsed as seconds from 1970, otherwise the content used as pattern",
+    examples = {"yyyy-MM-dd HH:mm:ss,SSS", "epoch"}
+  )
   @Expose
   @SerializedName("target_date_pattern")
   private String targetDatePattern;

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java
index 4a8d746..e7b8fdf 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java
@@ -19,17 +19,29 @@
 
 package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
 
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.MapFieldCopyDescriptor;
 
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
-public class MapFieldCopyDescriptorImpl implements MapFieldCopyDescriptor {
+@ShipperConfigTypeDescription(
+    name = "Map Copy",
+    description = "The name of the mapping element should be map_copy. The value json element should contain the following parameter:"
+)
+public class MapFieldCopyDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldCopyDescriptor {
   @Override
   public String getJsonName() {
     return "map_fieldcopy";
   }
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/post_map_values/{field_name}/[]/map_copy/copy_name",
+    type = "string",
+    description = "The name of the copied field",
+    examples = {"new_name"}
+  )
   @Expose
   @SerializedName("copy_name")
   private String copyName;

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java
new file mode 100644
index 0000000..101e0d4
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
+import org.apache.ambari.logsearch.config.api.model.inputconfig.MapFieldDescriptor;
+
+@ShipperConfigTypeDescription(
+    name = "Post Map Values",
+    description = "The Post Map Values element in the [filter](filter.md) field names as keys, the values are lists of sets of " +
+                  "post map values, each describing one mapping done on a field named before obtained after filtering.\n" +
+                  "\n" +
+                  "Currently there are four kind of mappings are supported:"
+  )
+public abstract class MapFieldDescriptorImpl implements MapFieldDescriptor {
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java
index bd32018..e1b71e6 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java
@@ -19,17 +19,29 @@
 
 package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
 
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.MapFieldNameDescriptor;
 
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
-public class MapFieldNameDescriptorImpl implements MapFieldNameDescriptor {
+@ShipperConfigTypeDescription(
+    name = "Map Field Name",
+    description = "The name of the mapping element should be map_fieldname. The value json element should contain the following parameter:"
+)
+public class MapFieldNameDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldNameDescriptor {
   @Override
   public String getJsonName() {
     return "map_fieldname";
   }
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldname/new_field_name",
+    type = "string",
+    description = "The name of the renamed field",
+    examples = {"new_name"}
+  )
   @Expose
   @SerializedName("new_field_name")
   private String newFieldName;

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java
index 599e152..a80a994 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java
@@ -19,21 +19,39 @@
 
 package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
 
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
 import org.apache.ambari.logsearch.config.api.model.inputconfig.MapFieldValueDescriptor;
 
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
-public class MapFieldValueDescriptorImpl implements MapFieldValueDescriptor {
+@ShipperConfigTypeDescription(
+    name = "Map Field Value",
+    description = "The name of the mapping element should be map_fieldvalue. The value json element should contain the following parameter:"
+)
+public class MapFieldValueDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldValueDescriptor {
   @Override
   public String getJsonName() {
     return "map_fieldvalue";
   }
 
+  @ShipperConfigElementDescription(
+    path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldvalue/pre_value",
+    type = "string",
+    description = "The value that the field must match (ignoring case) to be mapped",
+    examples = {"old_value"}
+  )
   @Expose
   @SerializedName("pre_value")
   private String preValue;
 
+  @ShipperConfigElementDescription(
+      path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldvalue/post_value",
+      type = "string",
+      description = "The value to which the field is modified to",
+      examples = {"new_value"}
+    )
   @Expose
   @SerializedName("post_value")
   private String postValue;

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java
index 3c21fd8..e3f9886 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java
@@ -95,7 +95,7 @@ public class PostMapValuesAdapter implements JsonDeserializer<List<PostMapValues
   private JsonElement createMapperObject(PostMapValuesImpl postMapValues, JsonSerializationContext context) {
     JsonObject jsonObject = new JsonObject();
     for (MapFieldDescriptor m : postMapValues.getMappers()) {
-      jsonObject.add(((MapFieldDescriptor)m).getJsonName(), context.serialize(m));
+      jsonObject.add(((MapFieldDescriptorImpl)m).getJsonName(), context.serialize(m));
     }
     return jsonObject;
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md
index 129279b..d825290 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md
@@ -48,6 +48,8 @@ Grok filters have the following additional parameters:
 
 ## Key-value Filter
 
+value\_borders is only used if it is specified, and value\_split is not.
+
 Key-value filters have the following additional parameters:
 
 | Field          | Description                                                                               | Default |
@@ -56,4 +58,4 @@ Key-value filters have the following additional parameters:
 | value\_split   | The string that separates keys from values                                                | "="     |
 | value\_borders | The borders around the value, must be 2 characters long, first before it, second after it | -       |
 
-If value\_borders is only used if it is specified, and value\_split is not.
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md
index 661eeb8..1a9ce8d 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md
@@ -20,20 +20,18 @@ limitations under the License.
 # Input
 
 The input element in the [input configuration](inputConfig.md) contains a list of input descriptions, each describing one source
-of input.
-
-The general elements in the json are the following:
+of input. The general elements in the json are the following:
 
 | Field                       | Description                                                                                           | Default      |
 |-----------------------------|-------------------------------------------------------------------------------------------------------|--------------|
-| type                        | The type of the input source, currently file and s3_file are supported                                | -            |
+| type                        | The log id for this source                                                                            | -            |
 | rowtype                     | The type of the row, can be service / audit                                                           | -            |
 | path                        | The path of the source, may contain '*' characters too                                                | -            |
 | add\_fields                 | The element contains field\_name: field\_value pairs which will be added to each rows data            | -            |
+| source                      | The type of the input source, currently file and s3_file are supported                                | -            |
 | tail                        | The input should check for only the latest file matching the pattern, not all of them                 | true         |
 | gen\_event\_md5             | Generate an event\_md5 field for each row by creating a hash of the row data                          | true         |
 | use\_event\_md5\_as\_id     | Generate an id for each row by creating a hash of the row data                                        | false        |
-| start\_position             | Should the parsing start from the beginning                                                           | beginning    |
 | cache\_enabled              | Allows the input to use a cache to filter out duplications                                            | true         |
 | cache\_key\_field           | Specifies the field for which to use the cache to find duplications of                                | log\_message |
 | cache\_last\_dedup\_enabled | Allow to filter out entries which are same as the most recent one irrelevant of it's time             | false        |
@@ -44,7 +42,7 @@ The general elements in the json are the following:
 
 ## File Input
 
-File inputs have the following parameters too:
+File inputs have some additional parameters:
 
 | Field                    | Description                                                        | Default |
 |--------------------------|--------------------------------------------------------------------|---------|

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md
index 7ec439a..bc219df 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md
@@ -67,4 +67,4 @@ The name of the mapping element should be map\_anonymize. The value json element
 | Field      | Description                                                                                                     |
 |------------|-----------------------------------------------------------------------------------------------------------------|
 | pattern    | The pattern to use to identify parts to anonymize. The parts to hide should be marked with the "<hide>" string. |
-| hide\_char | The character to hide with, if it is not specified then the default is 'X'                                      |
+| hide\_char | The character to hide with, if it is not specified then the default is '*'                                      |

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
index 7abf177..acc3d4d 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
@@ -131,13 +131,15 @@ public class FilterJSONTest {
   @Test
   public void testJSONFilterCode_invalidJson() throws Exception {
     LOG.info("testJSONFilterCode_invalidJson()");
+    
     init(new FilterJsonDescriptorImpl());
-    String inputStr="invalid json";
+    
+    String inputStr = "invalid json";
     try{
-    filterJson.apply(inputStr,new InputMarker(null, null, 0));
-    fail("Expected LogFeederException was not occured");
-    }catch(LogFeederException logFeederException){
-      assertEquals("Json parsing failed for inputstr = "+inputStr, logFeederException.getLocalizedMessage());
+      filterJson.apply(inputStr,new InputMarker(null, null, 0));
+      fail("Expected LogFeederException was not occured");
+    } catch(LogFeederException logFeederException) {
+      assertEquals("Json parsing failed for inputstr = " + inputStr, logFeederException.getLocalizedMessage());
     }
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java
new file mode 100644
index 0000000..7d4bc2c
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.common;
+
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.model.response.ShipperConfigDescriptionData;
+import org.reflections.Reflections;
+import org.reflections.scanners.FieldAnnotationsScanner;
+
+import javax.annotation.PostConstruct;
+import javax.inject.Named;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+@Named
+public class ShipperConfigDescriptionStorage {
+
+  private static final String SHIPPER_CONFIG_PACKAGE = "org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl";
+  
+  private final List<ShipperConfigDescriptionData> shipperConfigDescription = new ArrayList<>();
+
+  @PostConstruct
+  public void postConstruct() {
+    Thread loadShipperConfigDescriptionThread = new Thread("load_shipper_config_description") {
+      @Override
+      public void run() {
+        fillShipperConfigDescriptions();
+      }
+    };
+    loadShipperConfigDescriptionThread.setDaemon(true);
+    loadShipperConfigDescriptionThread.start();
+  }
+
+  public List<ShipperConfigDescriptionData> getShipperConfigDescription() {
+    return shipperConfigDescription;
+  }
+
+  private void fillShipperConfigDescriptions() {
+    Reflections reflections = new Reflections(SHIPPER_CONFIG_PACKAGE, new FieldAnnotationsScanner());
+    Set<Field> fields = reflections.getFieldsAnnotatedWith(ShipperConfigElementDescription.class);
+    for (Field field : fields) {
+      ShipperConfigElementDescription description = field.getAnnotation(ShipperConfigElementDescription.class);
+      shipperConfigDescription.add(new ShipperConfigDescriptionData(description.path(), description.description(),
+          description.examples(), description.defaultValue()));
+    }
+    
+    shipperConfigDescription.sort((o1, o2) -> o1.getPath().compareTo(o2.getPath()));
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
index 6d1382d..da0a8bb 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
@@ -104,6 +104,7 @@ public class DocConstants {
     public static final String GET_AUTH_DETAILS_OD = "Get authentication details.";
     public static final String GET_ALL_PROPERTIES_INFO_OD = "List all available properties for Log Search and Log Feeder";
     public static final String GET_LOGSEARCH_PROPERTIES_INFO_OD = "List all available properties for Log Search property file (e.g: logsearch.properties/logfeeder.properties)";
+    public static final String GET_ALL_SHIPPER_CONFIG_INFO_OD = "List all available shipper configuration element";
   }
 
   public class EventHistoryDescriptions {

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java
index f6d0449..2f63492 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java
@@ -25,7 +25,9 @@ import java.util.Map;
 
 import org.apache.ambari.logsearch.conf.AuthPropsConfig;
 import org.apache.ambari.logsearch.common.PropertyDescriptionStorage;
+import org.apache.ambari.logsearch.common.ShipperConfigDescriptionStorage;
 import org.apache.ambari.logsearch.model.response.PropertyDescriptionData;
+import org.apache.ambari.logsearch.model.response.ShipperConfigDescriptionData;
 
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -39,6 +41,9 @@ public class InfoManager extends JsonManagerBase {
   @Inject
   private PropertyDescriptionStorage propertyDescriptionStore;
 
+  @Inject
+  private ShipperConfigDescriptionStorage shipperConfigDescriptionStore;
+
   public Map<String, Boolean> getAuthMap() {
     Map<String, Boolean> authMap = new HashMap<>();
     authMap.put("external", authPropsConfig.isAuthExternalEnabled());
@@ -56,4 +61,8 @@ public class InfoManager extends JsonManagerBase {
   public List<PropertyDescriptionData> getLogSearchPropertyDescriptions(String propertiesFile) {
     return getPropertyDescriptions().get(propertiesFile);
   }
+  
+  public List<ShipperConfigDescriptionData> getLogSearchShipperConfigDescription() {
+    return shipperConfigDescriptionStore.getShipperConfigDescription();
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ShipperConfigDescriptionData.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ShipperConfigDescriptionData.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ShipperConfigDescriptionData.java
new file mode 100644
index 0000000..91f7420
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ShipperConfigDescriptionData.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.model.response;
+
+public class ShipperConfigDescriptionData {
+  private final String path;
+
+  private final String description;
+
+  private final String[] examples;
+
+  private final String defaultValue;
+
+  public ShipperConfigDescriptionData(String path, String description, String[] examples, String defaultValue) {
+    this.path = path;
+    this.description = description;
+    this.examples = examples;
+    this.defaultValue = defaultValue;
+  }
+
+  public String getPath() {
+    return path;
+  }
+
+  public String getDescription() {
+    return description;
+  }
+
+  public String[] getExamples() {
+    return examples;
+  }
+
+  public String getDefaultValue() {
+    return defaultValue;
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java
index 6ea0bab..e49be90 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java
@@ -29,12 +29,14 @@ import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import org.apache.ambari.logsearch.manager.InfoManager;
 import org.apache.ambari.logsearch.model.response.PropertyDescriptionData;
+import org.apache.ambari.logsearch.model.response.ShipperConfigDescriptionData;
 import org.springframework.context.annotation.Scope;
 
 import java.util.List;
 import java.util.Map;
 
 import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_ALL_PROPERTIES_INFO_OD;
+import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_ALL_SHIPPER_CONFIG_INFO_OD;
 import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_LOGSEARCH_PROPERTIES_INFO_OD;
 import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_AUTH_DETAILS_OD;
 
@@ -70,4 +72,12 @@ public class InfoResource {
   public List<PropertyDescriptionData> getPropertyFileDescription(@PathParam("propertyFile") String propertyFile) {
     return infoManager.getLogSearchPropertyDescriptions(propertyFile);
   }
+
+  @GET
+  @Path("/shipperconfig")
+  @Produces({"application/json"})
+  @ApiOperation(GET_ALL_SHIPPER_CONFIG_INFO_OD)
+  public List<ShipperConfigDescriptionData> getShipperConfigDescription() {
+    return infoManager.getLogSearchShipperConfigDescription();
+  }
 }


Mime
View raw message