hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From heyongqi...@apache.org
Subject svn commit: r1209696 - /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java
Date Fri, 02 Dec 2011 21:27:37 GMT
Author: heyongqiang
Date: Fri Dec  2 21:27:37 2011
New Revision: 1209696

URL: http://svn.apache.org/viewvc?rev=1209696&view=rev
Log:
Hive-2607: Add caching to json_tuple (Kevin Wilfong via He Yongqiang)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java?rev=1209696&r1=1209695&r2=1209696&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java
Fri Dec  2 21:27:37 2011
@@ -19,6 +19,8 @@
 package org.apache.hadoop.hive.ql.udf.generic;
 
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -55,6 +57,28 @@ public class GenericUDTFJSONTuple extend
   boolean pathParsed = false;
   boolean seenErrors = false;
 
+  //An LRU cache using a linked hash map
+  static class HashCache<K, V> extends LinkedHashMap<K, V> {
+
+    private static final int CACHE_SIZE = 16;
+    private static final int INIT_SIZE = 32;
+    private static final float LOAD_FACTOR = 0.6f;
+
+    HashCache() {
+      super(INIT_SIZE, LOAD_FACTOR);
+    }
+
+    private static final long serialVersionUID = 1;
+
+    @Override
+    protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+      return size() > CACHE_SIZE;
+    }
+
+  }
+
+  static Map<String, JSONObject> jsonObjectCache = new HashCache<String, JSONObject>();
+
   @Override
   public void close() throws HiveException {
   }
@@ -124,7 +148,11 @@ public class GenericUDTFJSONTuple extend
       return;
     }
     try {
-      JSONObject jsonObj = new JSONObject(jsonStr);
+      JSONObject jsonObj = jsonObjectCache.get(jsonStr);
+      if (jsonObj == null) {
+        jsonObj = new JSONObject(jsonStr);
+        jsonObjectCache.put(jsonStr, jsonObj);
+      }
 
       for (int i = 0; i < numCols; ++i) {
         if (jsonObj.isNull(paths[i])) {



Mime
View raw message