parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alexleven...@apache.org
Subject parquet-mr git commit: PARQUET-297: generate Version class using parquet-generator
Date Thu, 18 Jun 2015 23:58:48 GMT
Repository: parquet-mr
Updated Branches:
  refs/heads/master 1c1606858 -> ad4432103


PARQUET-297: generate Version class using parquet-generator

Author: Konstantin Shaposhnikov <Konstantin.Shaposhnikov@sc.com>
Author: Konstantin Shaposhnikov <k.shaposhnikov@gmail.com>

Closes #213 from kostya-sh/PARQUET-297_2 and squashes the following commits:

ddb469a [Konstantin Shaposhnikov] add comment about paddedByteCountFromBits coming from ByteUtils
6b47b04 [Konstantin Shaposhnikov] Change VersionGenerator to generate main() method
10d0b38 [Konstantin Shaposhnikov] PARQUET-297: generate Version class using parquet-generator
11d29bc [Konstantin Shaposhnikov] parquet-generator: remove dependency on parquet-common


Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/ad443210
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/ad443210
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/ad443210

Branch: refs/heads/master
Commit: ad443210312d2420efef6d03a0296d71e71feb22
Parents: 1c16068
Author: Konstantin Shaposhnikov <Konstantin.Shaposhnikov@sc.com>
Authored: Thu Jun 18 16:58:45 2015 -0700
Committer: Alex Levenson <alexlevenson@twitter.com>
Committed: Thu Jun 18 16:58:45 2015 -0700

----------------------------------------------------------------------
 parquet-common/pom.xml                          |  28 +++++
 .../main/java/org/apache/parquet/Version.java   | 103 -------------------
 parquet-generator/pom.xml                       |  11 +-
 .../ByteBasedBitPackingGenerator.java           |   9 +-
 .../org/apache/parquet/version/Generator.java   |  28 +++++
 .../parquet/version/VersionGenerator.java       |  83 +++++++++++++++
 .../main/resources/parquet-version.properties   |  20 ++++
 7 files changed, 171 insertions(+), 111 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/ad443210/parquet-common/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-common/pom.xml b/parquet-common/pom.xml
index b6468f5..eacf2ab 100644
--- a/parquet-common/pom.xml
+++ b/parquet-common/pom.xml
@@ -47,6 +47,34 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
       </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <version>1.2.1</version>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-generator</artifactId>
+            <version>${project.version}</version>
+          </dependency>
+        </dependencies>
+        <executions>
+          <execution>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>java</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <mainClass>org.apache.parquet.version.Generator</mainClass>
+          <includePluginDependencies>true</includePluginDependencies>
+          <arguments>
+            <argument>${basedir}/target/generated-src</argument>
+          </arguments>
+          <sourceRoot>${basedir}/target/generated-src</sourceRoot>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 </project>

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/ad443210/parquet-common/src/main/java/org/apache/parquet/Version.java
----------------------------------------------------------------------
diff --git a/parquet-common/src/main/java/org/apache/parquet/Version.java b/parquet-common/src/main/java/org/apache/parquet/Version.java
deleted file mode 100644
index 6580f33..0000000
--- a/parquet-common/src/main/java/org/apache/parquet/Version.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.parquet;
-
-import java.io.IOException;
-import java.net.URL;
-import java.util.Enumeration;
-import java.util.Properties;
-import java.util.jar.Manifest;
-
-/**
- * The version of the library
- *
- * parquet-mr version 1.0.0-SNAPSHOT (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55)
- *
- * @author Julien Le Dem
- *
- */
-public class Version {
-  private static final Log LOG = Log.getLog(Version.class);
-  
-  public static final String VERSION_NUMBER = readVersionNumber();
-  public static final String FULL_VERSION = readFullVersion();
-
-  private static String getJarPath() {
-    final URL versionClassBaseUrl = Version.class.getResource("");
-    if (versionClassBaseUrl.getProtocol().equals("jar")) {
-      String path = versionClassBaseUrl.getPath();
-      int jarEnd = path.indexOf("!");
-      if (jarEnd != -1) {
-        String jarPath = path.substring(0, jarEnd);
-        return jarPath;
-      }
-    }
-    return null;
-  }
-
-  private static URL getResourceFromJar(String jarPath, String path) throws IOException {
-    Enumeration<URL> resources = Version.class.getClassLoader().getResources(path);
-    while (resources.hasMoreElements()) {
-      URL url = resources.nextElement();
-      if (url.getProtocol().equals("jar") && url.getPath().startsWith(jarPath)) {
-        return url;
-      }
-    }
-    return null;
-  }
-  
-  private static String readVersionNumber() {
-    String version = null;
-    try {
-      String jarPath = getJarPath();
-      if (jarPath != null) {
-        URL pomPropertiesUrl = getResourceFromJar(jarPath, "META-INF/maven/com.twitter/parquet-column/pom.properties");
-        if (pomPropertiesUrl != null) {
-          Properties properties = new Properties();
-          properties.load(pomPropertiesUrl.openStream());
-          version = properties.getProperty("version");
-        }
-      }
-    } catch (Exception e) {
-      LOG.warn("can't read from META-INF", e);
-    }
-    return version;
-  }
-
-  private static String readFullVersion() {
-    String sha = null;
-    try {
-      String jarPath = getJarPath();
-      if (jarPath != null) {
-        URL manifestUrl = getResourceFromJar(jarPath, "META-INF/MANIFEST.MF");
-        if (manifestUrl != null) {
-          Manifest manifest = new Manifest(manifestUrl.openStream());
-          sha = manifest.getMainAttributes().getValue("git-SHA-1");
-        }
-      }
-    } catch (Exception e) {
-      LOG.warn("can't read from META-INF", e);
-    }
-    return "parquet-mr" + (VERSION_NUMBER != null ? " version " + VERSION_NUMBER : "") +
(sha != null ? " (build " + sha + ")" : "");
-  }
-
-  public static void main(String[] args) {
-    System.out.println(FULL_VERSION);
-  }
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/ad443210/parquet-generator/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-generator/pom.xml b/parquet-generator/pom.xml
index 5bc18d6..f73a3d7 100644
--- a/parquet-generator/pom.xml
+++ b/parquet-generator/pom.xml
@@ -36,14 +36,15 @@
   </properties>
 
   <dependencies>
-    <dependency>
-      <groupId>org.apache.parquet</groupId>
-      <artifactId>parquet-common</artifactId>
-      <version>${project.version}</version>
-    </dependency>
   </dependencies>
 
   <build>
+    <resources>
+      <resource>
+        <directory>src/main/resources</directory>
+        <filtering>true</filtering>
+      </resource>
+    </resources>
     <plugins>
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/ad443210/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
b/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
index 2249cbc..9a7c562 100644
--- a/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
+++ b/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
@@ -22,8 +22,6 @@ import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 
-import org.apache.parquet.bytes.BytesUtils;
-
 /**
  *
  * This class generates bit packers that pack the most significant bit first.
@@ -214,7 +212,7 @@ public class ByteBasedBitPackingGenerator {
         fw.append("      out[" + align(valueIndex, 2) + " + outPos] =\n");
 
         int startIndex = valueIndex * bitWidth / 8;
-        int endIndex = BytesUtils.paddedByteCountFromBits((valueIndex + 1) * bitWidth);
+        int endIndex = paddedByteCountFromBits((valueIndex + 1) * bitWidth);
 
         for (int byteIndex = startIndex; byteIndex < endIndex; byteIndex++) {
           if (byteIndex == startIndex) {
@@ -256,4 +254,9 @@ public class ByteBasedBitPackingGenerator {
     result.append(valueString);
     return result.toString();
   }
+
+  // duplicated from BytesUtils to avoid a circular dependency between parquet-common and
parquet-generator
+  private static int paddedByteCountFromBits(int bitLength) {
+    return (bitLength + 7) / 8;
+  }
 }

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/ad443210/parquet-generator/src/main/java/org/apache/parquet/version/Generator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/org/apache/parquet/version/Generator.java b/parquet-generator/src/main/java/org/apache/parquet/version/Generator.java
new file mode 100644
index 0000000..57de327
--- /dev/null
+++ b/parquet-generator/src/main/java/org/apache/parquet/version/Generator.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.version;
+
+/**
+ * main class for code generation hook in build for version generation
+ */
+public class Generator {
+  public static void main(String[] args) throws Exception {
+    VersionGenerator.main(args);
+  }
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/ad443210/parquet-generator/src/main/java/org/apache/parquet/version/VersionGenerator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/org/apache/parquet/version/VersionGenerator.java
b/parquet-generator/src/main/java/org/apache/parquet/version/VersionGenerator.java
new file mode 100644
index 0000000..e0c5e4d
--- /dev/null
+++ b/parquet-generator/src/main/java/org/apache/parquet/version/VersionGenerator.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.version;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+public class VersionGenerator {
+
+  public static void main(String[] args) throws IOException {
+    File srcFile = new File(args[0] + "/org/apache/parquet/Version.java");
+    srcFile = srcFile.getAbsoluteFile();
+    File parent = srcFile.getParentFile();
+    if (!parent.exists()) {
+      if (!parent.mkdirs()) {
+        throw new IOException("Couldn't mkdirs for " + parent);
+      }
+    }
+    new VersionGenerator(srcFile).run();
+  }
+
+  private final FileWriter writer;
+
+  public VersionGenerator(File file) throws IOException {
+    this.writer = new FileWriter(file);
+  }
+
+  public void run() throws IOException {
+    InputStream in = VersionGenerator.class.getResourceAsStream("/parquet-version.properties");
+    if (in == null) {
+    	throw new IOException("/parquet-version.properties not found");
+    }
+    Properties props = new Properties();
+    props.load(in);
+
+    add("package org.apache.parquet;\n" +
+        "\n" +
+        "/**\n" +
+        " * This class is auto-generated by {@link org.apache.parquet.version.VersionGenerator}\n"
+
+        " * Do not manually edit!\n" +
+        " */\n");
+
+    add("public class Version {\n");
+
+    add("  public static final String VERSION_NUMBER = \"");
+    add(props.getProperty("versionNumber"));
+    add("\";\n");
+
+    add("  public static final String FULL_VERSION = \"");
+    add(props.getProperty("fullVersion"));
+    add("\";\n\n");
+
+    add("  public static void main(String[] args) {\n");
+    add("    System.out.println(FULL_VERSION);\n");
+    add("  }\n");
+
+    add("}\n");
+    writer.close();
+  }
+
+  private void add(String s) throws IOException {
+    writer.write(s);
+  }
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/ad443210/parquet-generator/src/main/resources/parquet-version.properties
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/resources/parquet-version.properties b/parquet-generator/src/main/resources/parquet-version.properties
new file mode 100644
index 0000000..5617e42
--- /dev/null
+++ b/parquet-generator/src/main/resources/parquet-version.properties
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+versionNumber = ${project.version}
+fullVersion = parquet-mr version ${project.version} (build ${buildNumber})


Mime
View raw message