flink-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ches...@apache.org
Subject [1/2] flink git commit: [FLINK-5781][docs] Generate HTML from ConfigOptions
Date Tue, 16 May 2017 07:35:11 GMT
Repository: flink
Updated Branches:
  refs/heads/master 4651a1690 -> 5ba5cd5ac


[FLINK-5781][docs] Generate HTML from ConfigOptions

This closes #3495.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b54a72e4
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b54a72e4
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b54a72e4

Branch: refs/heads/master
Commit: b54a72e4507b0d968c8209f2c447729b51d5d3bd
Parents: 4651a16
Author: Dawid Wysakowicz <dawid@getindata.com>
Authored: Wed Mar 8 15:57:33 2017 +0100
Committer: zentol <chesnay@apache.org>
Committed: Tue May 16 08:53:01 2017 +0200

----------------------------------------------------------------------
 docs/README.md                                  |  15 +
 flink-core/pom.xml                              |  34 +++
 .../apache/flink/configuration/ConfigGroup.java |  34 +++
 .../flink/configuration/ConfigGroups.java       |  37 +++
 .../flink/configuration/ConfigOption.java       |  32 ++-
 .../ConfigOptionsDocGenerator.java              | 280 +++++++++++++++++++
 .../configuration/DelegatingConfiguration.java  |   5 +-
 .../ConfigOptionsDocGeneratorTest.java          | 160 +++++++++++
 flink-libraries/flink-python/pom.xml            |  34 +++
 flink-yarn/pom.xml                              |  34 +++
 pom.xml                                         |  31 ++
 11 files changed, 692 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/b54a72e4/docs/README.md
----------------------------------------------------------------------
diff --git a/docs/README.md b/docs/README.md
index d243b04..b904799 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -49,6 +49,21 @@ to files that are modified. Note that if you are making changes that affect
 the sidebar navigation, you'll have to build the entire site to see
 those changes reflected on every page.
 
+## Generate configuration tables
+
+Configuration descriptions are auto generated from code. To trigger the generation you need
to run:
+
+```
+mvn -Pgenerate-config-docs install
+```
+
+The resulting html files will be written to `_include/generated`. Tables are regenerated
each time the command is invoked.
+These tables can be directly included into the documentation:
+
+```
+{% include generated/file_name.html %}
+```
+
 # Contribute
 
 ## Markdown

http://git-wip-us.apache.org/repos/asf/flink/blob/b54a72e4/flink-core/pom.xml
----------------------------------------------------------------------
diff --git a/flink-core/pom.xml b/flink-core/pom.xml
index 6b0d778..5b42220 100644
--- a/flink-core/pom.xml
+++ b/flink-core/pom.xml
@@ -123,6 +123,40 @@ under the License.
 		</dependency>
     </dependencies>
 
+	<profiles>
+		<profile>
+			<id>generate-config-docs</id>
+
+			<build>
+				<plugins>
+					<plugin>
+						<artifactId>maven-antrun-plugin</artifactId>
+						<version>1.7</version>
+						<executions>
+							<execution>
+								<phase>package</phase>
+								<goals>
+									<goal>run</goal>
+								</goals>
+							</execution>
+						</executions>
+						<configuration>
+							<target>
+								<mkdir dir="${rootDir}/${generated.docs.dir}"/>
+								<java classname="org.apache.flink.configuration.ConfigOptionsDocGenerator" fork="true">
+									<classpath refid="maven.compile.classpath" />
+									<arg value="${rootDir}/${generated.docs.dir}/" />
+									<!--package with configuration classes-->
+									<arg value="org.apache.flink.configuration" />
+								</java>
+							</target>
+						</configuration>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+	</profiles>
+
 	<build>
 		<plugins>
 

http://git-wip-us.apache.org/repos/asf/flink/blob/b54a72e4/flink-core/src/main/java/org/apache/flink/configuration/ConfigGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigGroup.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigGroup.java
new file mode 100644
index 0000000..a0ad000
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigGroup.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.configuration;
+
+import java.lang.annotation.Target;
+import org.apache.flink.annotation.Internal;
+
+/**
+ * A class that specifies a group of {@link ConfigOption}. The name of the group will be
used as the basis for the
+ * filename of the generated html file, as defined in {@link ConfigOptionsDocGenerator}.
+ * 
+ * @see ConfigGroups
+ */
+@Target({})
+@Internal
+public @interface ConfigGroup {
+	String name();
+	String keyPrefix();
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/b54a72e4/flink-core/src/main/java/org/apache/flink/configuration/ConfigGroups.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigGroups.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigGroups.java
new file mode 100644
index 0000000..2c1f871
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigGroups.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.configuration;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Annotation used on classes containing {@link ConfigOption}s that enables the separation
of options into different
+ * tables based on key prefixes. A {@link ConfigOption} is assigned to a {@link ConfigGroup}
if the option key matches
+ * the group prefix. If a key matches multiple prefixes the longest matching prefix takes
priority. An option is never
+ * assigned to multiple groups. Options that don't match any group are implicitly added to
a default group.
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+@Internal
+public @interface ConfigGroups {
+	ConfigGroup[] groups() default {};
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/b54a72e4/flink-core/src/main/java/org/apache/flink/configuration/ConfigOption.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigOption.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigOption.java
index 3531f6d..22d2cc4 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigOption.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigOption.java
@@ -51,6 +51,9 @@ public class ConfigOption<T> {
 	/** The default value for this option */
 	private final T defaultValue;
 
+	/** The description for this option */
+	private final String description;
+
 	// ------------------------------------------------------------------------
 
 	/**
@@ -61,6 +64,7 @@ public class ConfigOption<T> {
 	 */
 	ConfigOption(String key, T defaultValue) {
 		this.key = checkNotNull(key);
+		this.description = "";
 		this.defaultValue = defaultValue;
 		this.deprecatedKeys = EMPTY;
 	}
@@ -72,8 +76,9 @@ public class ConfigOption<T> {
 	 * @param defaultValue    The default value for this option
 	 * @param deprecatedKeys  The list of deprecated keys, in the order to be checked
 	 */
-	ConfigOption(String key, T defaultValue, String... deprecatedKeys) {
+	ConfigOption(String key, String description, T defaultValue, String... deprecatedKeys) {
 		this.key = checkNotNull(key);
+		this.description = description;
 		this.defaultValue = defaultValue;
 		this.deprecatedKeys = deprecatedKeys == null || deprecatedKeys.length == 0 ? EMPTY : deprecatedKeys;
 	}
@@ -92,7 +97,20 @@ public class ConfigOption<T> {
 	 * @return A new config options, with the given deprecated keys.
 	 */
 	public ConfigOption<T> withDeprecatedKeys(String... deprecatedKeys) {
-		return new ConfigOption<>(key, defaultValue, deprecatedKeys);
+		return new ConfigOption<>(key, description, defaultValue, deprecatedKeys);
+	}
+
+	/**
+	 * Creates a new config option, using this option's key and default value, and
+	 * adding the given description. The given description is used when generation the configuration
documention.
+	 *
+	 * <p><b>NOTE:</b> You can use html to format the output of the generated
cell.
+	 *
+	 * @param description The description for this option.
+	 * @return A new config option, with given description.
+	 */
+	public ConfigOption<T> withDescription(final String description) {
+		return new ConfigOption<>(key, description, defaultValue, deprecatedKeys);
 	}
 
 	// ------------------------------------------------------------------------
@@ -137,6 +155,14 @@ public class ConfigOption<T> {
 		return deprecatedKeys == EMPTY ? Collections.<String>emptyList() : Arrays.asList(deprecatedKeys);
 	}
 
+	/**
+	 * Returns the description of this option.
+	 * @return The option's description.
+	 */
+	public String description() {
+		return description;
+	}
+
 	// ------------------------------------------------------------------------
 
 	@Override
@@ -168,4 +194,4 @@ public class ConfigOption<T> {
 		return String.format("Key: '%s' , default: %s (deprecated keys: %s)",
 				key, defaultValue, Arrays.toString(deprecatedKeys));
 	}
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/b54a72e4/flink-core/src/main/java/org/apache/flink/configuration/ConfigOptionsDocGenerator.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigOptionsDocGenerator.java
b/flink-core/src/main/java/org/apache/flink/configuration/ConfigOptionsDocGenerator.java
new file mode 100644
index 0000000..b112843
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigOptionsDocGenerator.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.configuration;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.java.tuple.Tuple2;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Class used for generating code based documentation of configuration parameters.
+ */
+public class ConfigOptionsDocGenerator {
+
+	/**
+	 * This method generates html tables from set of classes containing {@link ConfigOption
ConfigOptions}.
+	 *
+	 * <p>For each class 1 or more html tables will be generated and placed into a separate
file, depending on whether
+	 * the class is annotated with {@link ConfigGroups}. The tables contain the key, default
value and description for
+	 * every {@link ConfigOption}.
+	 *
+	 * @param args first argument is output path for the generated files, second argument is
full package name containing
+	 *             classes with {@link ConfigOption}
+	 */
+	public static void main(String[] args) throws IOException, ClassNotFoundException {
+		String outputPath = args[0];
+		String packageName = args[1];
+
+		Path configDir = Paths.get("../src/main/java", packageName.replaceAll("\\.", "/"));
+
+		Pattern p = Pattern.compile("(([a-zA-Z]*)(Options))\\.java");
+		try (DirectoryStream<Path> stream = Files.newDirectoryStream(configDir, "*Options.java"))
{
+			for (Path entry : stream) {
+				String fileName = entry.getFileName().toString();
+				Matcher matcher = p.matcher(fileName);
+				if (!fileName.equals("ConfigOptions.java") && matcher.matches()) {
+					Class<?> optionsClass = Class.forName(packageName + "." + matcher.group(1));
+					List<Tuple2<ConfigGroup, String>> tables = generateTablesForClass(optionsClass);
+					if (tables.size() > 0) {
+						for (Tuple2<ConfigGroup, String> group : tables) {
+
+							String name = group.f0 == null
+								? matcher.group(2).replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase()
+								: group.f0.name().replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase();
+
+							String outputFile = name + "_configuration.html";
+							Files.write(Paths.get(outputPath, outputFile), group.f1.getBytes(StandardCharsets.UTF_8));
+						}
+					}
+				}
+			}
+		}
+	}
+
+	@VisibleForTesting
+	static List<Tuple2<ConfigGroup, String>> generateTablesForClass(Class<?>
optionsClass) {
+		ConfigGroups configGroups = optionsClass.getAnnotation(ConfigGroups.class);
+		List<Tuple2<ConfigGroup, String>> tables = new ArrayList<>();
+		List<ConfigOption> allOptions = extractConfigOptions(optionsClass);
+
+		if (configGroups != null) {
+			Tree tree = new Tree(configGroups.groups(), allOptions);
+
+			for (ConfigGroup group : configGroups.groups()) {
+				List<ConfigOption> configOptions = tree.findConfigOptions(group);
+				sortOptions(configOptions);
+				tables.add(Tuple2.of(group, toHtmlTable(configOptions)));
+			}
+			List<ConfigOption> configOptions = tree.getDefaultOptions();
+			sortOptions(configOptions);
+			tables.add(Tuple2.<ConfigGroup, String>of(null, toHtmlTable(configOptions)));
+		} else {
+			sortOptions(allOptions);
+			tables.add(Tuple2.<ConfigGroup, String>of(null, toHtmlTable(allOptions)));
+		}
+		return tables;
+	}
+
+	private static List<ConfigOption> extractConfigOptions(Class<?> clazz) {
+		try {
+			List<ConfigOption> configOptions = new ArrayList<>();
+			Field[] fields = clazz.getFields();
+			for (Field field : fields) {
+				if (field.getType().equals(ConfigOption.class) && field.getAnnotation(Deprecated.class)
== null) {
+					configOptions.add((ConfigOption) field.get(null));
+				}
+			}
+
+			return configOptions;
+		} catch (Exception e) {
+			throw new RuntimeException("Failed to extract config options from class " + clazz + ".",
e);
+		}
+	}
+
+
+	/**
+	 * Transforms this configuration group into HTML formatted table.
+	 * Options are sorted alphabetically by key.
+	 *
+	 * @param options list of options to include in this group
+	 * @return string containing HTML formatted table
+	 */
+	private static String toHtmlTable(final List<ConfigOption> options) {
+		StringBuilder htmlTable = new StringBuilder(
+			"<table class=\"table table-bordered\"><thead><tr><th class=\"text-left\"
style=\"width: 20%\">Key</th>" +
+			"<th class=\"text-left\" style=\"width: 15%\">Default Value</th><th class=\"text-left\"
" +
+			"style=\"width: 65%\">Description</th></tr></thead><tbody>");
+
+		for (ConfigOption option : options) {
+			htmlTable.append(toHtmlString(option));
+		}
+
+		htmlTable.append("</tbody></table>");
+
+		return htmlTable.toString();
+	}
+
+	/**
+	 * Transforms option to table row.
+	 *
+	 * @param option option to transform
+	 * @return row with the option description
+	 */
+	private static String toHtmlString(final ConfigOption<?> option) {
+		Object defaultValue = option.defaultValue();
+		// This is a temporary hack that should be removed once FLINK-6490 is resolved.
+		// These options use System.getProperty("java.io.tmpdir") as the default.
+		// As a result the generated table contains an actual path as the default, which is simply
wrong.
+		if (option == JobManagerOptions.WEB_TMP_DIR || option.key().equals("python.dc.tmp.dir"))
{
+			defaultValue = null;
+		}
+		return "<tr>" +
+			"<td><h5>" + escapeCharacters(option.key()) + "</h5></td>" +
+			"<td>" + escapeCharacters(defaultValueToHtml(defaultValue)) + "</td>" +
+			"<td>" + escapeCharacters(option.description()) + "</td>" +
+			"</tr>";
+	}
+
+	private static String defaultValueToHtml(Object value) {
+		if (value instanceof String) {
+			if (((String) value).isEmpty()) {
+				return "(none)";
+			}
+			return "\"" + value + "\"";
+		}
+
+		return value == null ? "(none)" : value.toString();
+	}
+
+	private static String escapeCharacters(String value) {
+		return value
+			.replaceAll("<", "&#60;")
+			.replaceAll(">", "&#62;");
+	}
+
+	private static void sortOptions(List<ConfigOption> configOptions) {
+		Collections.sort(configOptions, new Comparator<ConfigOption>() {
+			@Override
+			public int compare(ConfigOption o1, ConfigOption o2) {
+				return o1.key().compareTo(o2.key());
+			}
+		});
+	}
+
+	/**
+	 * Data structure used to assign {@link ConfigOption ConfigOptions} to the {@link ConfigGroup}
with the longest
+	 * matching prefix.
+	 */
+	private static class Tree {
+		private final Node root = new Node();
+
+		Tree(ConfigGroup[] groups, Collection<ConfigOption> options) {
+			// generate a tree based on all key prefixes
+			for (ConfigGroup group : groups) {
+				String[] keyComponents = group.keyPrefix().split("\\.");
+				Node currentNode = root;
+				for (String keyComponent : keyComponents) {
+					currentNode = currentNode.addChild(keyComponent);
+				}
+				currentNode.markAsGroupRoot();
+			}
+
+			// assign options to their corresponding group, i.e. the last group root node encountered
when traversing
+			// the tree based on the option key
+			for (ConfigOption<?> option : options) {
+				findGroupRoot(option.key()).assignOption(option);
+			}
+		}
+
+		List<ConfigOption> findConfigOptions(ConfigGroup configGroup) {
+			Node groupRoot = findGroupRoot(configGroup.keyPrefix());
+			return groupRoot.getConfigOptions();
+		}
+
+		List<ConfigOption> getDefaultOptions() {
+			return root.getConfigOptions();
+		}
+
+		private Node findGroupRoot(String key) {
+			String[] keyComponents = key.split("\\.");
+			Node currentNode = root;
+			for (String keyComponent : keyComponents) {
+				currentNode = currentNode.findChild(keyComponent);
+			}
+			return currentNode.isGroupRoot() ? currentNode : root;
+		}
+
+		private static class Node {
+			private final List<ConfigOption> configOptions = new ArrayList<>();
+			private final Map<String, Node> children = new HashMap<>();
+			private boolean isGroupRoot = false;
+
+			private Node addChild(String keyComponent) {
+				Node child = children.get(keyComponent);
+				if (child == null) {
+					child = new Node();
+					children.put(keyComponent, child);
+				}
+				return child;
+			}
+
+			private Node findChild(String keyComponent) {
+				Node child = children.get(keyComponent);
+				if (child == null) {
+					return this;
+				}
+				return child;
+			}
+
+			private void assignOption(ConfigOption option) {
+				configOptions.add(option);
+			}
+
+			private boolean isGroupRoot() {
+				return isGroupRoot;
+			}
+
+			private void markAsGroupRoot() {
+				this.isGroupRoot = true;
+			}
+
+			private List<ConfigOption> getConfigOptions() {
+				return configOptions;
+			}
+		}
+	}
+
+	private ConfigOptionsDocGenerator() {
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/b54a72e4/flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java
b/flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java
index 1b14e9e..29f76b5 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/DelegatingConfiguration.java
@@ -342,6 +342,9 @@ public final class DelegatingConfiguration extends Configuration {
 		}
 
 		String[] deprecated = deprecatedKeys.toArray(new String[deprecatedKeys.size()]);
-		return new ConfigOption<T>(key, option.defaultValue(), deprecated);
+		return new ConfigOption<T>(key,
+			option.description(),
+			option.defaultValue(),
+			deprecated);
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/b54a72e4/flink-core/src/test/java/org/apache/flink/configuration/ConfigOptionsDocGeneratorTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/configuration/ConfigOptionsDocGeneratorTest.java
b/flink-core/src/test/java/org/apache/flink/configuration/ConfigOptionsDocGeneratorTest.java
new file mode 100644
index 0000000..04facf0
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/configuration/ConfigOptionsDocGeneratorTest.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.configuration;
+
+import java.util.HashMap;
+import java.util.List;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class ConfigOptionsDocGeneratorTest {
+
+	static class TestConfigGroup {
+		public static ConfigOption<Integer> firstOption = ConfigOptions
+			.key("first.option.a")
+			.defaultValue(2)
+			.withDescription("This is example description for the first option.");
+
+		public static ConfigOption<String> secondOption = ConfigOptions
+			.key("second.option.a")
+			.noDefaultValue()
+			.withDescription("This is long example description for the second option.");
+	}
+
+	@Test
+	public void testCreatingDescription() throws Exception {
+		final String expectedTable = "<table class=\"table table-bordered\">" +
+			"<thead>" +
+			"<tr>" +
+			"<th class=\"text-left\" style=\"width: 20%\">Key</th>" +
+			"<th class=\"text-left\" style=\"width: 15%\">Default Value</th>" +
+			"<th class=\"text-left\" style=\"width: 65%\">Description</th>" +
+			"</tr>" +
+			"</thead>" +
+			"<tbody>" +
+			"<tr>" +
+			"<td><h5>first.option.a</h5></td>" +
+			"<td>2</td>" +
+			"<td>This is example description for the first option.</td>" +
+			"</tr>" +
+			"<tr>" +
+			"<td><h5>second.option.a</h5></td>" +
+			"<td>(none)</td>" +
+			"<td>This is long example description for the second option.</td>" +
+			"</tr>" +
+			"</tbody>" +
+			"</table>";
+		final String htmlTable = ConfigOptionsDocGenerator.generateTablesForClass(TestConfigGroup.class).get(0).f1;
+
+		assertEquals(expectedTable, htmlTable);
+	}
+
+	@ConfigGroups(groups = {
+			@ConfigGroup(name = "firstGroup", keyPrefix = "first"),
+			@ConfigGroup(name = "secondGroup", keyPrefix = "second")})
+	static class TestConfigMultipleSubGroup {
+		public static ConfigOption<Integer> firstOption = ConfigOptions
+			.key("first.option.a")
+			.defaultValue(2)
+			.withDescription("This is example description for the first option.");
+
+		public static ConfigOption<String> secondOption = ConfigOptions
+			.key("second.option.a")
+			.noDefaultValue()
+			.withDescription("This is long example description for the second option.");
+
+		public static ConfigOption<Integer> thirdOption = ConfigOptions
+			.key("third.option.a")
+			.defaultValue(2)
+			.withDescription("This is example description for the third option.");
+
+		public static ConfigOption<String> fourthOption = ConfigOptions
+			.key("fourth.option.a")
+			.noDefaultValue()
+			.withDescription("This is long example description for the fourth option.");
+	}
+
+	@Test
+	public void testCreatingMultipleGroups() throws Exception {
+		final List<Tuple2<ConfigGroup, String>> tables = ConfigOptionsDocGenerator.generateTablesForClass(
+			TestConfigMultipleSubGroup.class);
+
+		assertEquals(tables.size(), 3);
+		final HashMap<String, String> tablesConverted = new HashMap<>();
+		for (Tuple2<ConfigGroup, String> table : tables) {
+			tablesConverted.put(table.f0 != null ? table.f0.name() : "default", table.f1);
+		}
+
+		assertEquals("<table class=\"table table-bordered\">" +
+		             "<thead>" +
+		             "<tr>" +
+		             "<th class=\"text-left\" style=\"width: 20%\">Key</th>" +
+		             "<th class=\"text-left\" style=\"width: 15%\">Default Value</th>"
+
+		             "<th class=\"text-left\" style=\"width: 65%\">Description</th>"
+
+		             "</tr>" +
+		             "</thead>" +
+		             "<tbody>" +
+		             "<tr>" +
+		             "<td><h5>first.option.a</h5></td>" +
+		             "<td>2</td>" +
+		             "<td>This is example description for the first option.</td>" +
+		             "</tr>" +
+		             "</tbody>" +
+		             "</table>", tablesConverted.get("firstGroup"));
+		assertEquals("<table class=\"table table-bordered\">" +
+		             "<thead>" +
+		             "<tr>" +
+		             "<th class=\"text-left\" style=\"width: 20%\">Key</th>" +
+		             "<th class=\"text-left\" style=\"width: 15%\">Default Value</th>"
+
+		             "<th class=\"text-left\" style=\"width: 65%\">Description</th>"
+
+		             "</tr>" +
+		             "</thead>" +
+		             "<tbody>" +
+		             "<tr>" +
+		             "<td><h5>second.option.a</h5></td>" +
+		             "<td>(none)</td>" +
+		             "<td>This is long example description for the second option.</td>"
+
+		             "</tr>" +
+		             "</tbody>" +
+		             "</table>", tablesConverted.get("secondGroup"));
+		assertEquals("<table class=\"table table-bordered\">" +
+		             "<thead>" +
+		             "<tr>" +
+		             "<th class=\"text-left\" style=\"width: 20%\">Key</th>" +
+		             "<th class=\"text-left\" style=\"width: 15%\">Default Value</th>"
+
+		             "<th class=\"text-left\" style=\"width: 65%\">Description</th>"
+
+		             "</tr>" +
+		             "</thead>" +
+		             "<tbody>" +
+		             "<tr>" +
+		             "<td><h5>fourth.option.a</h5></td>" +
+		             "<td>(none)</td>" +
+		             "<td>This is long example description for the fourth option.</td>"
+
+		             "</tr>" +
+		             "<tr>" +
+		             "<td><h5>third.option.a</h5></td>" +
+		             "<td>2</td>" +
+		             "<td>This is example description for the third option.</td>" +
+		             "</tr>" +
+		             "</tbody>" +
+		             "</table>", tablesConverted.get("default"));
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/b54a72e4/flink-libraries/flink-python/pom.xml
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/pom.xml b/flink-libraries/flink-python/pom.xml
index 5b6405e..6a57c3e 100644
--- a/flink-libraries/flink-python/pom.xml
+++ b/flink-libraries/flink-python/pom.xml
@@ -51,6 +51,40 @@ under the License.
         </plugins>
     </build>
 
+	<profiles>
+		<profile>
+			<id>generate-config-docs</id>
+
+			<build>
+				<plugins>
+					<plugin>
+						<artifactId>maven-antrun-plugin</artifactId>
+						<version>1.7</version>
+						<executions>
+							<execution>
+								<phase>package</phase>
+								<goals>
+									<goal>run</goal>
+								</goals>
+							</execution>
+						</executions>
+						<configuration>
+							<target>
+								<mkdir dir="${rootDir}/${generated.docs.dir}"/>
+								<java classname="org.apache.flink.configuration.ConfigOptionsDocGenerator" fork="true">
+									<classpath refid="maven.compile.classpath" />
+									<arg value="${rootDir}/${generated.docs.dir}/" />
+									<!--package with configuration classes-->
+									<arg value="org.apache.flink.python.api" />
+								</java>
+							</target>
+						</configuration>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+	</profiles>
+
     <dependencies>
         
         <!-- core dependencies -->

http://git-wip-us.apache.org/repos/asf/flink/blob/b54a72e4/flink-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/flink-yarn/pom.xml b/flink-yarn/pom.xml
index e97fee5..ec0dd63 100644
--- a/flink-yarn/pom.xml
+++ b/flink-yarn/pom.xml
@@ -121,6 +121,40 @@ under the License.
 		</dependency>
 	</dependencies>
 
+	<profiles>
+		<profile>
+			<id>generate-config-docs</id>
+
+			<build>
+				<plugins>
+					<plugin>
+						<artifactId>maven-antrun-plugin</artifactId>
+						<version>1.7</version>
+						<executions>
+							<execution>
+								<phase>package</phase>
+								<goals>
+									<goal>run</goal>
+								</goals>
+							</execution>
+						</executions>
+						<configuration>
+							<target>
+								<mkdir dir="${rootDir}/${generated.docs.dir}"/>
+								<java classname="org.apache.flink.configuration.ConfigOptionsDocGenerator" fork="true">
+									<classpath refid="maven.compile.classpath" />
+									<arg value="${rootDir}/${generated.docs.dir}/" />
+									<!--package with configuration classes-->
+									<arg value="org.apache.flink.yarn.configuration" />
+								</java>
+							</target>
+						</configuration>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+	</profiles>
+
 	<build>
 		<plugins>
 			<!-- Scala Compiler -->

http://git-wip-us.apache.org/repos/asf/flink/blob/b54a72e4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a8377e1..71a6e29 100644
--- a/pom.xml
+++ b/pom.xml
@@ -114,6 +114,7 @@ under the License.
 			to revisit the impact at that time.
 		-->
 		<minikdc.version>2.7.2</minikdc.version>
+		<generated.docs.dir>./docs/_includes/generated</generated.docs.dir>
 	</properties>
 
 	<dependencies>
@@ -991,6 +992,7 @@ under the License.
 						<exclude>docs/content/**</exclude>
 						<exclude>**/scalastyle-output.xml</exclude>
 						<exclude>build-target/**</exclude>
+						<exclude>docs/_includes/generated/**</exclude>
 						<!-- Tools: watchdog -->
 						<exclude>tools/artifacts/**</exclude>
 						<exclude>tools/flink*/**</exclude>
@@ -1222,6 +1224,35 @@ under the License.
 					</execution>
 				</executions>
 			</plugin>
+
+			<!-- generate configuration docs -->
+			<plugin>
+				<groupId>org.commonjava.maven.plugins</groupId>
+				<artifactId>directory-maven-plugin</artifactId>
+				<version>0.1</version>
+				<executions>
+					<execution>
+						<id>directories</id>
+						<goals>
+							<goal>highest-basedir</goal>
+						</goals>
+						<phase>initialize</phase>
+						<configuration>
+							<property>rootDir</property>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+			<plugin>
+				<artifactId>maven-clean-plugin</artifactId>
+				<configuration>
+					<filesets>
+						<fileset>
+							<directory>${generated.docs.dir}</directory>
+						</fileset>
+					</filesets>
+				</configuration>
+			</plugin>
 		</plugins>
 
 		<!-- Plugin configurations for plugins activated in sub-projects --> 


Mime
View raw message