Mailing-List: contact commits-help@atlas.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@atlas.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: madhan@apache.org
To: commits@atlas.incubator.apache.org
Message-Id: <558c9cb417c547b1a13a57af260d8efb@git.apache.org>
Subject: incubator-atlas git commit: ATLAS-1876: fix to handle large
 float/double values during ser-de, export/import
Date: Wed, 21 Jun 2017 04:27:15 +0000 (UTC)
archived-at: Wed, 21 Jun 2017 04:27:21 -0000

Repository: incubator-atlas
Updated Branches:
  refs/heads/master 242b5585c -> f053fd59b


ATLAS-1876: fix to handle large float/double values during ser-de, export/import

Signed-off-by: Madhan Neethiraj <madhan@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/f053fd59
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/f053fd59
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/f053fd59

Branch: refs/heads/master
Commit: f053fd59b5337429d96d441a07ad3d44eac65a5d
Parents: 242b558
Author: ashutoshm <amestry@hortonworks.com>
Authored: Tue Jun 20 20:09:39 2017 -0700
Committer: Madhan Neethiraj <madhan@apache.org>
Committed: Tue Jun 20 21:13:13 2017 -0700

----------------------------------------------------------------------
 .../apache/atlas/type/AtlasBuiltInTypes.java    |  14 ++++-
 .../java/org/apache/atlas/type/AtlasType.java   |   5 +-
 .../type/TestAtlasBuiltInTypesFloatDouble.java  |  63 +++++++++++++++++++
 .../atlas/repository/impexp/ZipSource.java      |  26 +-------
 .../atlas/repository/impexp/ZipSourceTest.java  |  22 +++++++
 repository/src/test/resources/stocks-float.zip  | Bin 0 -> 5986 bytes
 6 files changed, 103 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f053fd59/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java b/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
index f08a601..139d4ec 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
@@ -283,7 +283,12 @@ public class AtlasBuiltInTypes {
                     return ((Number) obj).floatValue();
                 } else {
                     try {
-                        return Float.valueOf(obj.toString());
+                        Float f = Float.valueOf(obj.toString());
+                        if(!Float.isInfinite(f)) {
+                            return f;
+                        } else {
+                            return null;
+                        }
                     } catch (NumberFormatException excp) {
                         // ignore
                     }
@@ -329,7 +334,12 @@ public class AtlasBuiltInTypes {
                     return ((Number) obj).doubleValue();
                 } else {
                     try {
-                        return Double.valueOf(obj.toString());
+                        Double d = Double.valueOf(obj.toString());
+                        if(!Double.isInfinite(d)) {
+                            return d;
+                        } else {
+                            return null;
+                        }
                     } catch (NumberFormatException excp) {
                         // ignore
                     }

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f053fd59/intg/src/main/java/org/apache/atlas/type/AtlasType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasType.java b/intg/src/main/java/org/apache/atlas/type/AtlasType.java
index f05cfd6..86072fe 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasType.java
@@ -21,9 +21,9 @@ package org.apache.atlas.type;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.TypeCategory;
 import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.codehaus.jackson.map.DeserializationConfig;
 import org.codehaus.jackson.map.ObjectMapper;
 
-
 import java.io.IOException;
 import java.util.List;
 
@@ -36,7 +36,8 @@ import java.util.List;
 
 public abstract class AtlasType {
 
-    private static final ObjectMapper mapper = new ObjectMapper();
+    private static final ObjectMapper mapper = new ObjectMapper()
+                                            .configure(DeserializationConfig.Feature.USE_BIG_DECIMAL_FOR_FLOATS, true);
 
     private final String       typeName;
     private final TypeCategory typeCategory;

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f053fd59/intg/src/test/java/org/apache/atlas/type/TestAtlasBuiltInTypesFloatDouble.java
----------------------------------------------------------------------
diff --git a/intg/src/test/java/org/apache/atlas/type/TestAtlasBuiltInTypesFloatDouble.java b/intg/src/test/java/org/apache/atlas/type/TestAtlasBuiltInTypesFloatDouble.java
new file mode 100644
index 0000000..573164f
--- /dev/null
+++ b/intg/src/test/java/org/apache/atlas/type/TestAtlasBuiltInTypesFloatDouble.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.type;
+
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+
+public class TestAtlasBuiltInTypesFloatDouble {
+
+    @Test
+    public void floatRangeCheck() {
+        assertFloatChecks(String.valueOf("-1.E-45"), true);
+        assertFloatChecks(String.valueOf(Float.MAX_VALUE), true);
+        assertFloatChecks("3.4028235E32", true);
+        assertFloatChecks("-3.4028235E32", true);
+        assertFloatChecks(String.valueOf(Float.MIN_VALUE), true);
+        assertFloatChecks("4028235e+555", false);
+        assertFloatChecks("-4028235e+555", false);
+    }
+
+    @Test
+    public void doubleRangeCheck() {
+        assertDoubleChecks(String.valueOf(Double.MAX_VALUE), true);
+        assertDoubleChecks("3.4028235E32", true);
+        assertDoubleChecks(String.valueOf(Double.MIN_VALUE), true);
+        assertDoubleChecks("4028235e+55555", false);
+        assertDoubleChecks("-4028235e+55555", false);
+    }
+
+    private void assertFloatChecks(String v, boolean notNull) {
+        assertNullNotNull(notNull, new AtlasBuiltInTypes.AtlasFloatType().getNormalizedValue(v));
+    }
+
+
+    private void assertDoubleChecks(String v, boolean notNull) {
+        assertNullNotNull(notNull, new AtlasBuiltInTypes.AtlasDoubleType().getNormalizedValue(v));
+    }
+
+    private void assertNullNotNull(boolean notNull, Object f) {
+        if(notNull) {
+            assertNotNull(f);
+        } else {
+            assertNull(f);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f053fd59/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
index aa1477f..edb816f 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
@@ -23,9 +23,8 @@ import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
 import org.apache.atlas.model.typedef.AtlasTypesDef;
 import org.apache.atlas.repository.store.graph.v1.EntityImportStream;
+import org.apache.atlas.type.AtlasType;
 import org.apache.commons.lang.StringUtils;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.type.TypeReference;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -39,8 +38,6 @@ import java.util.Map;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
-import static org.apache.atlas.AtlasErrorCode.JSON_ERROR_OBJECT_MAPPER_NULL_RETURNED;
-
 
 public class ZipSource implements EntityImportStream {
     private static final Logger LOG = LoggerFactory.getLogger(ZipSource.class);
@@ -90,7 +87,7 @@ public class ZipSource implements EntityImportStream {
 
         try {
             String s = getFromCache(fileName);
-            this.creationOrder = convertFromJson(new TypeReference<List<String>>(){}, s);
+            this.creationOrder = convertFromJson(List.class, s);
             this.iterator = this.creationOrder.iterator();
         } catch (AtlasBaseException e) {
             LOG.error(String.format("Error retrieving '%s' from zip.", fileName), e);
@@ -137,26 +134,9 @@ public class ZipSource implements EntityImportStream {
         return entityWithExtInfo;
     }
 
-    private <T> T convertFromJson(TypeReference clazz, String jsonData) throws AtlasBaseException {
-        try {
-            ObjectMapper mapper = new ObjectMapper();
-
-            T ret = mapper.readValue(jsonData, clazz);
-            if(ret == null) {
-                throw new AtlasBaseException(JSON_ERROR_OBJECT_MAPPER_NULL_RETURNED, clazz.toString());
-            }
-
-            return ret;
-        } catch (Exception e) {
-            throw new AtlasBaseException("Error converting file to JSON.", e);
-        }
-    }
-
     private <T> T convertFromJson(Class<T> clazz, String jsonData) throws AtlasBaseException {
         try {
-            ObjectMapper mapper = new ObjectMapper();
-
-            return mapper.readValue(jsonData, clazz);
+            return AtlasType.fromJson(jsonData, clazz);
 
         } catch (Exception e) {
             throw new AtlasBaseException("Error converting file to JSON.", e);

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f053fd59/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
index 8a57dcd..1c1c68f 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
@@ -44,6 +44,13 @@ public class ZipSourceTest {
         return new Object[][] {{ new ZipSource(fs) }};
     }
 
+    @DataProvider(name = "zipFileStocksFloat")
+    public static Object[][] getDataFromZipFileWithLongFloats() throws IOException {
+        FileInputStream fs = ZipFileResourceTestUtils.getFileInputStream("stocks-float.zip");
+
+        return new Object[][] {{ new ZipSource(fs) }};
+    }
+
     @DataProvider(name = "sales")
     public static Object[][] getDataFromQuickStart_v1_Sales(ITestContext context) throws IOException {
         return getZipSource("sales-v1-full.zip");
@@ -139,6 +146,21 @@ public class ZipSourceTest {
         assertTrue(zipSource.hasNext());
     }
 
+    @Test(dataProvider = "zipFileStocksFloat")
+    public void attemptToSerializeLongFloats(ZipSource zipSource) throws IOException, AtlasBaseException {
+        Assert.assertTrue(zipSource.hasNext());
+        assertTrue(zipSource.hasNext());
+        assertTrue(zipSource.hasNext());
+
+        AtlasEntity.AtlasEntityWithExtInfo e = zipSource.getNextEntityWithExtInfo();
+        assertNotNull(e);
+        assertTrue(e.getEntity().getClassifications().size() > 0);
+        assertNotNull(e.getEntity().getClassifications().get(0).getAttribute("fv"));
+        assertEquals(e.getEntity().getClassifications().get(0).getAttribute("fv").toString(), "3.4028235E+38");
+
+        assertTrue(zipSource.hasNext());
+    }
+
     @Test(dataProvider = "zipFileStocks")
     public void applyTransformation(ZipSource zipSource) throws IOException, AtlasBaseException {
         ImportTransforms transforms = getTransformForHiveDB();

http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f053fd59/repository/src/test/resources/stocks-float.zip
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/stocks-float.zip b/repository/src/test/resources/stocks-float.zip
new file mode 100644
index 0000000..ccef1e0
Binary files /dev/null and b/repository/src/test/resources/stocks-float.zip differ