usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From toddn...@apache.org
Subject [1/3] Refactored loader to group for batching, then call filters to perform the load, verification and result construction
Date Thu, 09 Oct 2014 21:50:17 GMT
Repository: incubator-usergrid
Updated Branches:
  refs/heads/collection_multiget f43e4f5c3 -> 454ef4ff1


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpEntityMapUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpEntityMapUtils.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpEntityMapUtils.java
new file mode 100644
index 0000000..feed396
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpEntityMapUtils.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.  For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+
+
+package org.apache.usergrid.corepersistence.util;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.Schema;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.field.ArrayField;
+import org.apache.usergrid.persistence.model.field.BooleanField;
+import org.apache.usergrid.persistence.model.field.ByteArrayField;
+import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.EntityObjectField;
+import org.apache.usergrid.persistence.model.field.Field;
+import org.apache.usergrid.persistence.model.field.FloatField;
+import org.apache.usergrid.persistence.model.field.IntegerField;
+import org.apache.usergrid.persistence.model.field.ListField;
+import org.apache.usergrid.persistence.model.field.LocationField;
+import org.apache.usergrid.persistence.model.field.LongField;
+import org.apache.usergrid.persistence.model.field.SetField;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.field.UUIDField;
+import org.apache.usergrid.persistence.model.field.value.EntityObject;
+import org.apache.usergrid.persistence.model.field.value.Location;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Utilities for converting entities to/from maps suitable for Core Persistence.
+ * Aware of unique properties via Schema.
+ */
+public class CpEntityMapUtils {
+    private static final Logger logger = LoggerFactory.getLogger( CpEntityMapUtils.class
);
+
+    public static ObjectMapper objectMapper = new ObjectMapper(  );
+
+    public static Entity fromMap( Map<String, Object> map, String entityType, boolean
topLevel ) {
+        return fromMap( null, map, entityType, topLevel );
+    }
+
+    public static Entity fromMap( Entity entity, Map<String, Object> map, String entityType,
boolean topLevel ) {
+
+        if ( entity == null ) {
+            entity = new Entity();
+        }
+
+        for ( String fieldName : map.keySet() ) {
+
+            Object value = map.get( fieldName );
+            boolean unique = Schema.getDefaultSchema().isPropertyUnique(entityType, fieldName);
+
+//            if ( unique ) {
+//                logger.debug("{} is a unique property", fieldName );
+//            }
+
+            if ( value instanceof String ) {
+                entity.setField( new StringField( fieldName, (String)value, unique &&
topLevel ));
+
+            } else if ( value instanceof Boolean ) {
+                entity.setField( new BooleanField( fieldName, (Boolean)value, unique &&
topLevel ));
+                        
+            } else if ( value instanceof Integer ) {
+                entity.setField( new IntegerField( fieldName, (Integer)value, unique &&
topLevel ));
+
+            } else if ( value instanceof Double ) {
+                entity.setField( new DoubleField( fieldName, (Double)value, unique &&
topLevel ));
+
+		    } else if ( value instanceof Float ) {
+                entity.setField( new FloatField( fieldName, (Float)value, unique &&
topLevel ));
+				
+            } else if ( value instanceof Long ) {
+                entity.setField( new LongField( fieldName, (Long)value, unique &&
topLevel ));
+
+            } else if ( value instanceof List) {
+                entity.setField( listToListField( fieldName, (List)value, entityType ));
 
+            
+            } else if ( value instanceof UUID) {
+                entity.setField( new UUIDField( fieldName, (UUID)value, unique &&
topLevel ));
+
+            } else if ( value instanceof Map ) {
+                processMapValue( value, fieldName, entity, entityType);
+
+            } else if ( value instanceof Enum ) {
+                entity.setField( new StringField( fieldName, value.toString(), unique &&
topLevel ));
+	
+			} else if ( value != null ) {
+                byte[] valueSerialized;
+                try {
+                    valueSerialized = objectMapper.writeValueAsBytes( value );
+                }
+                catch ( JsonProcessingException e ) {
+                    throw new RuntimeException( "Can't serialize object ",e );
+                }
+                catch ( IOException e ) {
+                    throw new RuntimeException( "Can't serialize object ",e );
+                }
+                ByteBuffer byteBuffer = ByteBuffer.wrap( valueSerialized );
+                ByteArrayField bf = new ByteArrayField( fieldName, byteBuffer.array(), value.getClass()
);
+                entity.setField( bf );
+            }
+        }
+
+        return entity;
+    }
+
+    private static void processMapValue(
+            Object value, String fieldName, Entity entity, String entityType) {
+
+        Field field = null;
+
+        // is the map really a location element?
+        Map<String, Object> m = (Map<String, Object>)value;
+        if ( m.size() == 2) {
+            Double lat = null;
+            Double lon = null;
+            try {
+                if ( m.get("latitude") != null && m.get("longitude") != null ) {
+                    lat = Double.parseDouble( m.get("latitude").toString() );
+                    lon = Double.parseDouble( m.get("longitude").toString() );
+                    
+                } else if ( m.get("lat") != null && m.get("lon") != null ) {
+                    lat = Double.parseDouble( m.get("lat").toString() );
+                    lon = Double.parseDouble( m.get("lon").toString() );
+                }
+            } catch ( NumberFormatException ignored ) {}
+            
+            if ( lat != null && lon != null ) {
+                field = new LocationField( fieldName, new Location( lat, lon ));
+            }
+        }
+        
+        if ( field == null ) {
+            
+            // not a location element, process it as map
+            entity.setField( new EntityObjectField( fieldName,
+                    fromMap( (Map<String, Object>)value, entityType, false ))); //
recursion
+            
+        } else {
+            entity.setField( field );
+        }
+    }
+
+    
+    private static ListField listToListField( String fieldName, List list, String entityType
) {
+
+        if (list.isEmpty()) {
+            return new ListField( fieldName );
+        }
+
+        Object sample = list.get(0);
+
+        if ( sample instanceof Map ) {
+            return new ListField<Entity>( fieldName, processListForField( list, entityType
));
+
+        } else if ( sample instanceof List ) {
+            return new ListField<List>( fieldName, processListForField( list, entityType
));
+            
+        } else if ( sample instanceof String ) {
+            return new ListField<String>( fieldName, (List<String>)list );
+                    
+        } else if ( sample instanceof Boolean ) {
+            return new ListField<Boolean>( fieldName, (List<Boolean>)list );
+                    
+        } else if ( sample instanceof Integer ) {
+            return new ListField<Integer>( fieldName, (List<Integer>)list );
+
+        } else if ( sample instanceof Double ) {
+            return new ListField<Double>( fieldName, (List<Double>)list );
+
+        } else if ( sample instanceof Long ) {
+            return new ListField<Long>( fieldName, (List<Long>)list );
+
+        } else {
+            throw new RuntimeException("Unknown type " + sample.getClass().getName());
+        }
+    }
+
+    
+    private static List processListForField( List list, String entityType ) {
+        if ( list.isEmpty() ) {
+            return list;
+        }
+        Object sample = list.get(0);
+
+        if ( sample instanceof Map ) {
+            List<Entity> newList = new ArrayList<Entity>();
+            for ( Map<String, Object> map : (List<Map<String, Object>>)list
) {
+                newList.add( fromMap( map, entityType, false ) );
+            }
+            return newList;
+
+        } else if ( sample instanceof List ) {
+            return processListForField( list, entityType ); // recursion
+            
+        } else { 
+            return list;
+        } 
+    }
+
+
+    /**
+     * Convert Entity to Map, adding version_ug_field and a {name}_ug_analyzed field for
each
+     * StringField.
+     */
+    public static Map toMap(EntityObject entity) {
+
+        Map<String, Object> entityMap = new TreeMap<>();
+
+        for (Object f : entity.getFields().toArray()) {
+            Field field = (Field) f;
+
+            if (f instanceof ListField || f instanceof ArrayField) {
+                List list = (List) field.getValue();
+                entityMap.put(field.getName(),
+                        new ArrayList( processCollectionForMap(list)));
+
+            } else if (f instanceof SetField) {
+                Set set = (Set) field.getValue();
+                entityMap.put(field.getName(),
+                        new ArrayList( processCollectionForMap(set)));
+
+            } else if (f instanceof EntityObjectField) {
+                EntityObject eo = (EntityObject) field.getValue();
+                entityMap.put( field.getName(), toMap(eo)); // recursion
+
+            } else if (f instanceof StringField) {
+                entityMap.put(field.getName(), ((String) field.getValue()));
+
+            } else if (f instanceof LocationField) {
+                LocationField locField = (LocationField) f;
+                Map<String, Object> locMap = new HashMap<String, Object>();
+
+                // field names lat and lon trigger ElasticSearch geo location 
+                locMap.put("lat", locField.getValue().getLatitude());
+                locMap.put("lon", locField.getValue().getLongitude());
+                 entityMap.put( field.getName(), field.getValue());
+
+            } else if (f instanceof ByteArrayField) {
+                    ByteArrayField bf = ( ByteArrayField ) f;
+
+                    byte[] serilizedObj =  bf.getValue();
+                    Object o;
+                    try {
+                        o = objectMapper.readValue( serilizedObj, bf.getClassinfo() );
+                    }
+                    catch ( IOException e ) {
+                        throw new RuntimeException( "Can't deserialize object ",e );
+                    }
+                    entityMap.put( bf.getName(), o );
+            }
+            else {
+                entityMap.put( field.getName(), field.getValue());
+            }
+        }
+
+        return entityMap;
+    }
+
+    
+    private static Collection processCollectionForMap(Collection c) {
+        if (c.isEmpty()) {
+            return c;
+        }
+        List processed = new ArrayList();
+        Object sample = c.iterator().next();
+
+        if (sample instanceof Entity) {
+            for (Object o : c.toArray()) {
+                Entity e = (Entity) o;
+                processed.add(toMap(e));
+            }
+
+        } else if (sample instanceof List) {
+            for (Object o : c.toArray()) {
+                List list = (List) o;
+                processed.add(processCollectionForMap(list)); // recursion;
+            }
+
+        } else if (sample instanceof Set) {
+            for (Object o : c.toArray()) {
+                Set set = (Set) o;
+                processed.add(processCollectionForMap(set)); // recursion;
+            }
+
+        } else {
+            for (Object o : c.toArray()) {
+                processed.add(o);
+            }
+        }
+        return processed;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
new file mode 100644
index 0000000..f2fce47
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -0,0 +1,106 @@
+package org.apache.usergrid.corepersistence.util;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.usergrid.persistence.Schema;
+
+
+/**
+ * Utilises for constructing standard naming conventions for collections and connections
+ */
+public class CpNamingUtils {
+
+    /**
+     * Edge types for all types
+     */
+    public static final String ALL_TYPES = "zzzalltypeszzz";
+
+    /**
+     * Edge types for collection suffix
+     */
+    public static final String EDGE_COLL_SUFFIX = "zzzcollzzz";
+
+    /**
+     * Edge types for connection suffix
+     */
+    public static final String EDGE_CONN_SUFFIX = "zzzconnzzz";
+
+
+    public static String getCollectionScopeNameFromEntityType( String type ) {
+        String csn = EDGE_COLL_SUFFIX + Schema.defaultCollectionName( type );
+        return csn.toLowerCase();
+    }
+
+
+    public static String getCollectionScopeNameFromCollectionName( String name ) {
+        String csn = EDGE_COLL_SUFFIX + name;
+        return csn.toLowerCase();
+    }
+
+
+    public static String getConnectionScopeName( String entityType, String connectionType
) {
+        String csn = EDGE_CONN_SUFFIX + connectionType + entityType;
+        return csn.toLowerCase();
+    }
+
+
+    public static boolean isCollectionEdgeType( String type ) {
+        return type.startsWith( EDGE_COLL_SUFFIX );
+    }
+
+
+    public  static boolean isConnectionEdgeType( String type ) {
+        return type.startsWith( EDGE_CONN_SUFFIX );
+    }
+
+
+    static public String getConnectionType( String edgeType ) {
+        String[] parts = edgeType.split( "\\|" );
+        return parts[1];
+    }
+
+
+    static public String getCollectionName( String edgeType ) {
+        String[] parts = edgeType.split( "\\|" );
+        return parts[1];
+    }
+
+
+    public static String getEdgeTypeFromConnectionType( String connectionType ) {
+
+        if ( connectionType != null ) {
+            String csn = EDGE_CONN_SUFFIX + "|" + connectionType;
+            return csn;
+        }
+
+        return null;
+    }
+
+
+    public static String getEdgeTypeFromCollectionName( String collectionName ) {
+
+        if ( collectionName != null ) {
+            String csn = EDGE_COLL_SUFFIX + "|" + collectionName;
+            return csn;
+        }
+
+
+        return null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityMapUtilsTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityMapUtilsTest.java
b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityMapUtilsTest.java
index 87f6b46..a75b9fd 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityMapUtilsTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityMapUtilsTest.java
@@ -25,6 +25,8 @@ import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+
+import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
 import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
 import org.apache.usergrid.persistence.model.field.ListField;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index c5d5782..5a1eca2 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -22,6 +22,7 @@ import java.util.List;
 import java.util.UUID;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.usergrid.AbstractCoreIT;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.persistence.Entity;
 import org.apache.usergrid.persistence.EntityManager;
 import org.apache.usergrid.persistence.EntityRef;


Mime
View raw message