usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mru...@apache.org
Subject [35/50] [abbrv] usergrid git commit: added apache header, cleaned up some things.
Date Mon, 01 Aug 2016 16:54:10 GMT
added apache header, cleaned up some things.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a57f8650
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a57f8650
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a57f8650

Branch: refs/heads/master
Commit: a57f8650e4573d6ede650b0066821f7fae2849fe
Parents: 180bccf
Author: Jeff West <jwest@apigee.com>
Authored: Tue Jul 26 15:52:46 2016 -0700
Committer: Jeff West <jwest@apigee.com>
Committed: Tue Jul 26 15:52:46 2016 -0700

----------------------------------------------------------------------
 .../es_tools/alias_mover.py                     | 107 +---
 .../es_tools/cluster_shard_allocation.py        |  23 +-
 .../es_tools/command_sender.py                  |  27 +-
 .../es_tools/es_index_iterator_reindexer.py     |  21 +-
 .../es_tools/es_searcher.py                     |  25 +-
 .../es_tools/index_deleter.py                   |  22 +-
 .../es_tools/index_iterator_size_checker.py     |  25 +-
 .../es_tools/index_prefix_checker.py            |  21 +-
 .../es_tools/index_replica_setter.py            |  57 ++-
 .../es_tools/index_shard_allocator.py           |  67 ++-
 .../es_tools/mapping_deleter.py                 |  19 +
 .../es_tools/mapping_retriever.py               |  19 +
 .../es_tools/monitor_tasks.py                   |  19 +
 .../index_test/document_creator.py              |  19 +
 .../index_test/index_test_mixed_batch.py        |  44 +-
 .../index_test/index_test_single_type_batch.py  |  41 +-
 .../activity_streams/activity_streams.py        |  19 +
 .../samples/beacon-event-example.py             |  28 +-
 .../samples/counter_test.py                     |  24 +-
 utils/usergrid-util-python/setup.py             |  19 +
 .../usergrid_tools/general/deleter.py           |  19 +
 .../general/duplicate_name_checker.py           |  19 +
 .../usergrid_tools/general/queue_monitor.py     |  19 +
 .../usergrid_tools/general/url_tester.py        |  20 +-
 .../general/user_confirm_activate.py            |  19 +
 .../usergrid_tools/general/user_creator.py      |  19 +
 .../usergrid_tools/groups/big_group_creater.py  |  31 +-
 .../usergrid_tools/indexing/batch_index_test.py |  19 +
 .../indexing/entity_index_test.py               |  19 +
 .../usergrid_tools/iterators/simple_iterator.py |  19 +
 .../iterators/usergrid_cross_region_iterator.py |  26 +-
 .../iterators/usergrid_iterator.py              | 504 +++++++++++++++++++
 .../usergrid_tools/library_check.py             |  19 +
 .../migration/usergrid_data_exporter.py         |  19 +
 .../migration/usergrid_data_migrator.py         |  23 +-
 .../parse_importer/parse_importer.py            |  19 +
 .../usergrid_tools/permissions/permissions.py   |  19 +
 .../queue/dlq-iterator-checker.py               |  19 +
 .../usergrid_tools/queue/dlq_requeue.py         |  19 +
 .../usergrid_tools/queue/queue_cleaner.py       |  19 +
 .../usergrid_tools/redis/redis_iterator.py      |  19 +
 .../usergrid_tools/redis/redisscan.py           |  19 +
 42 files changed, 1355 insertions(+), 218 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/alias_mover.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/alias_mover.py b/utils/usergrid-util-python/es_tools/alias_mover.py
index 2a8fe02..3b5caab 100644
--- a/utils/usergrid-util-python/es_tools/alias_mover.py
+++ b/utils/usergrid-util-python/es_tools/alias_mover.py
@@ -1,94 +1,35 @@
-import json
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
 
+import json
 import requests
 
-example_request = {
-    "actions": [
-        {
-            "remove": {
-                "index": "apigee-vfmplus",
-                "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_read_alias"
-            }
-        },
-        {
-            "remove": {
-                "index": "apigee-vfmplus",
-                "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_write_alias"
-            }
-        },
-        {
-            "remove": {
-                "index": "apigee-vfmplus",
-                "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_read_alias"
-            }
-        },
-        {
-            "remove": {
-                "index": "apigee-vfmplus",
-                "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_write_alias"
-            }
-        },
-        {
-            "remove": {
-                "index": "apigee-vfmplus",
-                "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_read_alias"
-            }
-        },
-        {
-            "remove": {
-                "index": "apigee-vfmplus",
-                "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_write_alias"
-            }
-        },
-        {
-            "add": {
-                "index": "apigee-vmplus-docvalues",
-                "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_read_alias"
-            }
-        },
-        {
-            "add": {
-                "index": "apigee-vmplus-docvalues",
-                "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_write_alias"
-            }
-        },
-        {
-            "add": {
-                "index": "apigee-vmplus-docvalues",
-                "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_read_alias"
-            }
-        },
-        {
-            "add": {
-                "index": "apigee-vmplus-docvalues",
-                "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_write_alias"
-            }
-        },
-        {
-            "add": {
-                "index": "apigee-vmplus-docvalues",
-                "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_read_alias"
-            }
-        },
-        {
-            "add": {
-                "index": "apigee-vmplus-docvalues",
-                "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_write_alias"
-            }
-        }
-    ]
-}
-
-cluster = 'rug000sr_euwi'
+cluster = 'cluster-1'
 
 work = {
     # 'remove': {
-    #     '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'rug000sr_euwi_applications_3',
-    #     '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'rug000sr_euwi_applications_3',
+    #     '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'applications_3',
+    #     '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'applications_3',
     # },
     'add': {
-        '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'apigee-vfmplus-1-no-doc-18',
-        '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'apigee-vfmplus-1-no-doc-18',
+        '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'my-index-1-no-doc-18',
+        '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'my-index-1-no-doc-18',
     }
 }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
index a462124..2e0fcbd 100644
--- a/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
+++ b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 import time
 import requests
@@ -10,8 +29,8 @@ __author__ = 'Jeff West @ ApigeeCorporation'
 SHUTDOWN_NODES = True
 
 nodes = [
-    # 'res206wo',
-    # 'res207wo',
+    'elasticsearch206west',
+    'elasticsearch207west',
 ]
 
 base_url = 'http://localhost:9200'

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/command_sender.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/command_sender.py b/utils/usergrid-util-python/es_tools/command_sender.py
index 92ecfff..8208e78 100644
--- a/utils/usergrid-util-python/es_tools/command_sender.py
+++ b/utils/usergrid-util-python/es_tools/command_sender.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 import requests
 
@@ -21,16 +40,16 @@ data = {
             "move": {
                 "index": "usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_target_final",
                 "shard": 14,
-                "from_node": "res018sy",
-                "to_node": "res021sy"
+                "from_node": "elasticsearch018",
+                "to_node": "elasticsearch021"
             }
         },
         {
             "move": {
                 "index": "usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_target_final",
                 "shard": 12,
-                "from_node": "res018sy",
-                "to_node": "res009sy"
+                "from_node": "elasticsearch018",
+                "to_node": "elasticsearch009"
             }
         },
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py b/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
index f151fcb..85872e2 100644
--- a/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
+++ b/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 import re
 from multiprocessing.pool import Pool
@@ -5,7 +24,7 @@ import requests
 
 # This script iterates an index and issues a PUT request for an empty string to force a reindex of the entity
 
-index_url_template = 'http://res013wo:9200/{index_name}/_search?size={size}&from={from_var}'
+index_url_template = 'http://elasticsearch013wo:9200/{index_name}/_search?size={size}&from={from_var}'
 
 index_names = [
     'es-index-name'

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/es_searcher.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/es_searcher.py b/utils/usergrid-util-python/es_tools/es_searcher.py
index 55e54ef..4c7a297 100644
--- a/utils/usergrid-util-python/es_tools/es_searcher.py
+++ b/utils/usergrid-util-python/es_tools/es_searcher.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 import requests
 
@@ -5,12 +24,14 @@ import requests
 
 __author__ = 'Jeff West @ ApigeeCorporation'
 
-url_template = 'http://localhost:9200/pea000ug_applications_2/_search'
+INDEX_NAME=''
+
+url_template = 'http://localhost:9200/%s/_search' % INDEX_NAME
 
 request = {
     "query": {
         "term": {
-            "entityId": "entityId(1a78d0a6-bffb-11e5-bc61-0af922a4f655,constratus)"
+            "entityId": "entityId(1a78d0a6-bffb-11e5-bc61-0af922a4f655,superbad)"
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_deleter.py b/utils/usergrid-util-python/es_tools/index_deleter.py
index a697cf8..9b60006 100644
--- a/utils/usergrid-util-python/es_tools/index_deleter.py
+++ b/utils/usergrid-util-python/es_tools/index_deleter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import requests
 import logging
 
@@ -23,8 +42,7 @@ print 'retrieved %s indices' % len(indices)
 NUMBER_VALUE = 0
 
 includes = [
-    'rug002sr_euwi',
-    # 'rug002mr',
+    'cluster1',
     # 'b6768a08-b5d5-11e3-a495-10ddb1de66c3',
     # 'b6768a08-b5d5-11e3-a495-11ddb1de66c9',
 ]

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py b/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
index 98d4373..03924b2 100644
--- a/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
+++ b/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 import re
 import traceback
@@ -92,7 +111,7 @@ def update_entity_fields(entity, fields):
             field_value = field.get('long')
 
         else:
-            print 'WTF! %s' % json.dumps(field)
+            print 'Unexpected field type! %s' % json.dumps(field)
             return entity_copy
 
         entity_copy = update_entity_field(entity_copy, field_name, field_value)
@@ -227,10 +246,6 @@ while keep_going:
             if len(app_id_find) > 0:
                 app_id = app_id_find[0]
 
-                if app_id != '5f20f423-f2a8-11e4-a478-12a5923b55dc':
-                    print 'SKIPP APP ID: ' + app_id
-                    continue
-
                 entity_id_tmp = source.get('entityId')
 
                 entity_id_find = re_ent_id.findall(entity_id_tmp)

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_prefix_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_prefix_checker.py b/utils/usergrid-util-python/es_tools/index_prefix_checker.py
index d72ff3d..0ccb245 100644
--- a/utils/usergrid-util-python/es_tools/index_prefix_checker.py
+++ b/utils/usergrid-util-python/es_tools/index_prefix_checker.py
@@ -1,7 +1,26 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 from collections import defaultdict
 import requests
-import logging
+
 
 __author__ = 'Jeff West @ ApigeeCorporation'
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_replica_setter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_replica_setter.py b/utils/usergrid-util-python/es_tools/index_replica_setter.py
index 7180fed..1214e48 100644
--- a/utils/usergrid-util-python/es_tools/index_replica_setter.py
+++ b/utils/usergrid-util-python/es_tools/index_replica_setter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 from multiprocessing import Pool
 import requests
 import time
@@ -32,22 +51,8 @@ payload = {
     "index.number_of_replicas": NUMBER_VALUE,
 }
 
-# indices = ['usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_manual']
-
 includes = [
     # '70be096e-c2e1-11e4-8a55-12b4f5e28868',
-    # 'b0c640af-bc6c-11e4-b078-12b4f5e28868',
-    # 'e62e465e-bccc-11e4-b078-12b4f5e28868',
-    # 'd82b6413-bccc-11e4-b078-12b4f5e28868',
-    # '45914256-c27f-11e4-8a55-12b4f5e28868',
-    # '2776a776-c27f-11e4-8a55-12b4f5e28868',
-    # 'a54f878c-bc6c-11e4-b044-0e4cd56e19cd',
-    # 'ed5b47ea-bccc-11e4-b078-12b4f5e28868',
-    # 'bd4874ab-bccc-11e4-b044-0e4cd56e19cd',
-    # '3d748996-c27f-11e4-8a55-12b4f5e28868',
-    # '1daab807-c27f-11e4-8a55-12b4f5e28868',
-    # 'd0c4f0da-d961-11e4-849d-12b4f5e28868',
-    # '93e756ac-bc4e-11e4-92ae-12b4f5e28868',
 ]
 
 excludes = [
@@ -87,18 +92,18 @@ def update_shards(index_name):
     if update:
         print index_name
 
-        # url = '%s/%s/_settings' % (url_base, index)
-        # print url
-        #
-        # response = requests.get('%s/%s/_settings' % (url_base, index))
-        # settings = response.json()
-        #
-        # index_settings = settings[index]['settings']['index']
-        #
-        # current_replicas = int(index_settings.get('number_of_replicas'))
-        #
-        # if current_replicas == NUMBER_VALUE:
-        #     continue
+        url = '%s/%s/_settings' % (url_base, index)
+        print url
+
+        response = requests.get('%s/%s/_settings' % (url_base, index))
+        settings = response.json()
+
+        index_settings = settings[index]['settings']['index']
+
+        current_replicas = int(index_settings.get('number_of_replicas'))
+
+        if current_replicas == NUMBER_VALUE:
+            return
 
         success = False
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_shard_allocator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_shard_allocator.py b/utils/usergrid-util-python/es_tools/index_shard_allocator.py
index ecee095..d411744 100644
--- a/utils/usergrid-util-python/es_tools/index_shard_allocator.py
+++ b/utils/usergrid-util-python/es_tools/index_shard_allocator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 from multiprocessing import Pool
 
@@ -11,33 +30,33 @@ __author__ = 'Jeff West @ ApigeeCorporation'
 
 
 nodes_c32xl = [
-    'res000eu',
-    'res001eu',
-    'res002eu',
-    'res003eu',
-    'res004eu',
-    'res005eu',
-    'res009eu',
-    'res010eu',
-    'res011eu',
-    'res012eu',
-    'res013eu',
-    'res014eu',
+    'elasticsearch000eu',
+    'elasticsearch001eu',
+    'elasticsearch002eu',
+    'elasticsearch003eu',
+    'elasticsearch004eu',
+    'elasticsearch005eu',
+    'elasticsearch009eu',
+    'elasticsearch010eu',
+    'elasticsearch011eu',
+    'elasticsearch012eu',
+    'elasticsearch013eu',
+    'elasticsearch014eu',
 ]
 
 nodes_c34xl = [
-    'res015eu',
-    'res018eu',
-    'res019eu',
-    'res020eu',
-    'res021eu',
-    'res022eu',
-    'res023eu',
-    'res024eu',
-    'res025eu',
-    'res026eu',
-    'res027eu',
-    'res028eu'
+    'elasticsearch015eu',
+    'elasticsearch018eu',
+    'elasticsearch019eu',
+    'elasticsearch020eu',
+    'elasticsearch021eu',
+    'elasticsearch022eu',
+    'elasticsearch023eu',
+    'elasticsearch024eu',
+    'elasticsearch025eu',
+    'elasticsearch026eu',
+    'elasticsearch027eu',
+    'elasticsearch028eu'
 ]
 
 nodes = nodes_c34xl

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/mapping_deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/mapping_deleter.py b/utils/usergrid-util-python/es_tools/mapping_deleter.py
index 74ad898..112cc20 100644
--- a/utils/usergrid-util-python/es_tools/mapping_deleter.py
+++ b/utils/usergrid-util-python/es_tools/mapping_deleter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 
 import requests

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/mapping_retriever.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/mapping_retriever.py b/utils/usergrid-util-python/es_tools/mapping_retriever.py
index 0da123b..29fbe11 100644
--- a/utils/usergrid-util-python/es_tools/mapping_retriever.py
+++ b/utils/usergrid-util-python/es_tools/mapping_retriever.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 import requests
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/monitor_tasks.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/monitor_tasks.py b/utils/usergrid-util-python/es_tools/monitor_tasks.py
index df23d49..b444322 100644
--- a/utils/usergrid-util-python/es_tools/monitor_tasks.py
+++ b/utils/usergrid-util-python/es_tools/monitor_tasks.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import datetime
 import requests
 import time

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/index_test/document_creator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/document_creator.py b/utils/usergrid-util-python/index_test/document_creator.py
index fd544c6..a43e965 100644
--- a/utils/usergrid-util-python/index_test/document_creator.py
+++ b/utils/usergrid-util-python/index_test/document_creator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 from __future__ import print_function
 from Queue import Empty
 import json

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/index_test_mixed_batch.py b/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
index d1dd40c..3e6552e 100644
--- a/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
+++ b/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 from multiprocessing import JoinableQueue, Process
 import random
@@ -6,34 +25,19 @@ import traceback
 import uuid
 import time
 import sys
-
 import argparse
 import loremipsum
 import requests
 from elasticsearch import Elasticsearch
 
-
 es_hosts = [
-    {'host': 'ees000wo', 'port': 9200},
-    {'host': 'ees001wo', 'port': 9200},
-    {'host': 'ees002wo', 'port': 9200},
-    {'host': 'ees003wo', 'port': 9200},
-    {'host': 'ees004wo', 'port': 9200},
-    {'host': 'ees005wo', 'port': 9200},
-    {'host': 'ees006wo', 'port': 9200},
-    {'host': 'ees007wo', 'port': 9200},
-    {'host': 'ees008wo', 'port': 9200},
-    {'host': 'ees009wo', 'port': 9200},
-    {'host': 'ees010wo', 'port': 9200},
-    {'host': 'ees011wo', 'port': 9200},
-    {'host': 'ees012wo', 'port': 9200},
-    {'host': 'ees013wo', 'port': 9200},
-    {'host': 'ees014wo', 'port': 9200},
-    {'host': 'ees015wo', 'port': 9200},
-    {'host': 'ees016wo', 'port': 9200},
-    {'host': 'ees017wo', 'port': 9200}
+    {'host': 'elasticsearch000west', 'port': 9200},
+    {'host': 'elasticsearch001west', 'port': 9200},
+    {'host': 'elasticsearch002west', 'port': 9200},
+    {'host': 'elasticsearch003west', 'port': 9200}
 ]
 
+
 def parse_args():
     parser = argparse.ArgumentParser(description='ElasticSearch Index Test 1')
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/index_test_single_type_batch.py b/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
index e3afdc3..cc25570 100644
--- a/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
+++ b/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 from multiprocessing import JoinableQueue, Process
 import random
@@ -13,24 +32,10 @@ import requests
 from elasticsearch import Elasticsearch
 
 es_hosts = [
-    {'host': 'ees000wo', 'port': 9200},
-    {'host': 'ees001wo', 'port': 9200},
-    {'host': 'ees002wo', 'port': 9200},
-    {'host': 'ees003wo', 'port': 9200},
-    {'host': 'ees004wo', 'port': 9200},
-    {'host': 'ees005wo', 'port': 9200},
-    {'host': 'ees006wo', 'port': 9200},
-    {'host': 'ees007wo', 'port': 9200},
-    {'host': 'ees008wo', 'port': 9200},
-    {'host': 'ees009wo', 'port': 9200},
-    {'host': 'ees010wo', 'port': 9200},
-    {'host': 'ees011wo', 'port': 9200},
-    {'host': 'ees012wo', 'port': 9200},
-    {'host': 'ees013wo', 'port': 9200},
-    {'host': 'ees014wo', 'port': 9200},
-    {'host': 'ees015wo', 'port': 9200},
-    {'host': 'ees016wo', 'port': 9200},
-    {'host': 'ees017wo', 'port': 9200}
+    {'host': 'elasticsearch000west', 'port': 9200},
+    {'host': 'elasticsearch001west', 'port': 9200},
+    {'host': 'elasticsearch002west', 'port': 9200},
+    {'host': 'elasticsearch003west', 'port': 9200}
 ]
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/activity_streams/activity_streams.py b/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
index ce38544..9d95bef 100644
--- a/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
+++ b/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 # docs page: http://docs.apigee.com/api-baas/content/creating-activity
 
 # create user 1

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/samples/beacon-event-example.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/beacon-event-example.py b/utils/usergrid-util-python/samples/beacon-event-example.py
index fc05cdc..8863b35 100644
--- a/utils/usergrid-util-python/samples/beacon-event-example.py
+++ b/utils/usergrid-util-python/samples/beacon-event-example.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 # URL Templates for Usergrid
 import json
 import random
@@ -13,16 +32,11 @@ connection_query_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb
 connection_create_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}/{target_uuid}"
 
 url_data = {
-    'api_url': 'https://amer-apibaas-prod.apigee.net/appservices',
-    'org': 'jwest-samples',
+    'api_url': 'https://usergridhost/basepath',
+    'org': 'samples',
     'app': 'event-example'
 }
 
-url_data = {
-    'api_url': 'http://usergrid_app.cfapps-01.haas-26.pez.pivotal.io',
-    'org': 'jwest',
-    'app': 'sandbox'
-}
 
 session = requests.Session()
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/samples/counter_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/counter_test.py b/utils/usergrid-util-python/samples/counter_test.py
index 7852b26..4e393b7 100644
--- a/utils/usergrid-util-python/samples/counter_test.py
+++ b/utils/usergrid-util-python/samples/counter_test.py
@@ -1,4 +1,22 @@
-import datetime
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import time
 import json
 
@@ -8,7 +26,7 @@ tstamp = time.gmtime() * 1000
 
 s = requests.Session()
 
-s.headers.update({'authorization': 'Bearer YWMt7AHANAKcEeaVR-EahuX8EgAAAVQ7Q56jxQjUsmhJn8rGLTth0XtRrBSIzDA'})
+s.headers.update({'authorization': 'Bearer TOKEN'})
 s.headers.update({'content-type': 'application/json'})
 
 url = 'https://host/appservices-new/usergrid/pushtest/events'
@@ -26,6 +44,6 @@ print r.status_code
 
 time.sleep(30)
 
-r = s.get('https://host/appservices-new/usergrid/pushtest//counters?counter=counters.jeff.west')
+r = s.get('https://host/appservices-new/usergrid/pushtest/counters?counter=counters.jeff.west')
 
 print r.text

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/setup.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/setup.py b/utils/usergrid-util-python/setup.py
index 1f19cb2..83ee6ae 100755
--- a/utils/usergrid-util-python/setup.py
+++ b/utils/usergrid-util-python/setup.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 from setuptools import setup, find_packages
 
 __author__ = 'Jeff West @ ApigeeCorporation'

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/deleter.py b/utils/usergrid-util-python/usergrid_tools/general/deleter.py
index 3c53cae..3e21fa4 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/deleter.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/deleter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 import traceback
 import requests

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
index 3682d18..a40e097 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 from usergrid import UsergridQueryIterator
 
 ### This iterates a collection using GRAPH and checks whether there are more than on entity with the same name

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py b/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
index 37594d1..2ad2950 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import argparse
 import json
 import datetime

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/url_tester.py b/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
index 62755df..9d999d6 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
@@ -1,6 +1,24 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import datetime
 import time
-
 import numpy
 import requests
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py b/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
index 9b1484e..3e94468 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 
 import requests

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/user_creator.py b/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
index d21183d..ace64ee 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 import requests
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py b/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
index 268b7f8..3e88dfe 100644
--- a/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
+++ b/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 import traceback
 from multiprocessing import Pool
@@ -12,22 +31,14 @@ users = 10000
 username_template = 'precisely-10k-%s'
 
 url_data = {
-    "api_url": "https://usergrid-e2e-prod.e2e.apigee.net/appservices-2-1/",
-    "org": "tempgrey",
+    "api_url": "https://usergrid.net",
+    "org": "org",
     "app": "sandbox",
     "client_id": "",
     "client_secret": "",
 
 }
 
-url_data = {
-    "api_url": "http://baas-ug002sr.apigee.net/",
-    "org": "apigee-vfmplus",
-    "app": "sandbox",
-    "client_id": "",
-    "client_secret": "",
-
-}
 
 collection_url_template = "{api_url}/{org}/{app}/{collection}"
 add_user_url_template = "{api_url}/{org}/{app}/groups/{group_name}/users/{uuid}"

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py b/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
index 6c910dd..1ad8fbc 100644
--- a/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
+++ b/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
@@ -1,4 +1,23 @@
 # -*- coding: utf-8 -*-
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 import logging
 import traceback

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py b/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
index 6f193a6..8ec73f6 100644
--- a/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
+++ b/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
@@ -1,4 +1,23 @@
 # -*- coding: utf-8 -*-
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 import logging
 from multiprocessing import Pool

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
index 60cc4a0..c0f2ebd 100644
--- a/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import logging
 import sys
 import uuid

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
index 35933a2..e5e860f 100644
--- a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
@@ -1,7 +1,23 @@
-from usergrid import UsergridQuery
-
-__author__ = 'Jeff West @ ApigeeCorporation'
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
 
+from usergrid import UsergridQuery
 from Queue import Empty
 import argparse
 import json
@@ -12,10 +28,10 @@ from multiprocessing import Process, JoinableQueue
 import datetime
 import requests
 import traceback
-from logging.handlers import RotatingFileHandler
-import urllib3
 import urllib3.contrib.pyopenssl
 
+__author__ = 'Jeff West @ ApigeeCorporation'
+
 urllib3.disable_warnings()
 urllib3.contrib.pyopenssl.inject_into_urllib3()
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
new file mode 100644
index 0000000..04328ab
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
@@ -0,0 +1,504 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
+from Queue import Empty
+import json
+import logging
+import sys
+from multiprocessing import Queue, Process
+import traceback
+from logging.handlers import RotatingFileHandler
+import time
+
+import argparse
+
+from usergrid import UsergridClient, UsergridError
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+logger = logging.getLogger('UsergridIterator')
+
+# SAMPLE CONFIG FILE for source and target
+sample_config = {
+    "endpoint": {
+        "api_url": "https://api.usergrid.com",
+        "limit": 100
+    },
+
+    "credentials": {
+        "myOrg": {
+            "client_id": "<<client_id>>",
+            "client_secret": "<<client_secret>>"
+        }
+    }
+}
+
+
+def init_logging(file_enabled=False, stdout_enabled=True):
+    root_logger = logging.getLogger()
+    root_logger.setLevel(logging.INFO)
+    logging.getLogger('urllib3.connectionpool').setLevel(logging.WARN)
+    logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARN)
+
+    log_formatter = logging.Formatter(fmt='%(asctime)s | %(name)s | %(processName)s | %(levelname)s | %(message)s',
+                                      datefmt='%m/%d/%Y %I:%M:%S %p')
+
+    if file_enabled:
+        log_file_name = './UsergridIterator.log'
+
+        rotating_file = logging.handlers.RotatingFileHandler(filename=log_file_name,
+                                                             mode='a',
+                                                             maxBytes=204857600,
+                                                             backupCount=10)
+        rotating_file.setFormatter(log_formatter)
+        rotating_file.setLevel(logging.INFO)
+
+        root_logger.addHandler(rotating_file)
+
+    if stdout_enabled:
+        stdout_logger = logging.StreamHandler(sys.stdout)
+        stdout_logger.setFormatter(log_formatter)
+        stdout_logger.setLevel(logging.INFO)
+
+        root_logger.addHandler(stdout_logger)
+
+
+config = {}
+
+
+class Worker(Process):
+    """
+    The worker is used to perform a set of handler functions in a chain.  Work is provided for the Worker thread(s) on
+    a JoinableQueue.  The thread will continue until either 1) it is explicitly terminated or 2) until it does not
+     receive work on the queue after a consecutive number of attempts (max_empty_count) using the specified timeout
+     (queue_timeout)
+    """
+
+    def __init__(self,
+                 queue,
+                 source_client,
+                 target_client,
+                 max_empty_count=3,
+                 queue_timeout=10,
+                 function_chain=None):
+        """
+        This is an example handler function which can transform an entity. Multiple handler functions can be used to
+        process a entity.  The response is an entity which will get passed to the next handler in the chain
+
+        :param queue: The queue on which to listen for work
+        :param source_client: The UsergridClient of the source Usergrid instance
+        :param target_client: The UsergridClient of the target Usergrid instance
+        :param max_empty_count: The maximum number of times for a worker to not receive work after checking the queue
+        :param queue_timeout: The timeout for waiting for work on the queue
+        :param function_chain: An array of function pointers which will be executed in array sequence, expeting the following parameters: org_name, app_name, collection_name, entity, source_client, target_client, attempts=0p
+        """
+
+        super(Worker, self).__init__()
+        logger.warning('Creating worker!')
+
+        if not function_chain:
+            function_chain = []
+
+        self.function_chain = function_chain
+        self.queue = queue
+        self.source_client = source_client
+        self.target_client = target_client
+        self.max_empty_count = max_empty_count
+        self.queue_timeout = queue_timeout
+
+    def run(self):
+        logger.info('starting run()...')
+        keep_going = True
+
+        count_processed = 0
+        count_failed = 0
+        empty_count = 0
+
+        while keep_going:
+
+            try:
+                org, app, collection_name, entity = self.queue.get(timeout=self.queue_timeout)
+
+                empty_count = 0
+                success = True
+                entity_param = entity
+
+                for handler in self.function_chain:
+
+                    if entity_param is not None:
+                        try:
+                            entity_param = handler(org, app, collection_name, entity_param, self.source_client,
+                                                   self.target_client)
+                        except Exception, e:
+                            logger.error(e)
+                            print traceback.format_exc()
+                            success = False
+
+                if success:
+                    count_processed += 1
+                    logger.info('Processed [%sth] SUCCESS app/collection/name/uuid = %s / %s / %s / %s' % (
+                        count_processed, app, collection_name, entity.get('name'), entity.get('uuid')))
+                else:
+                    count_failed += 1
+                    logger.warning('Processed [%sth] FAILURE app/collection/name/uuid = %s / %s / %s / %s' % (
+                        count_processed, app, collection_name, entity.get('name'), entity.get('uuid')))
+
+            except KeyboardInterrupt, e:
+                raise e
+
+            except Empty:
+                logger.warning(
+                    'No task received after timeout=[%s]! Empty Count=%s' % (self.queue_timeout, empty_count))
+
+                empty_count += 1
+
+                if empty_count >= self.max_empty_count:
+                    logger.warning('Stopping work after empty_count=[%s]' % empty_count)
+                    keep_going = False
+
+        logger.info('Worker finished!')
+
+
+def filter_entity(org_name, app_name, collection_name, entity_data, source_client, target_client, attempts=0):
+    """
+    This is an example handler function which can filter entities. Multiple handler functions can be used to
+    process a entity.  The response is an entity which will get passed to the next handler in the chain
+
+    :param org_name: The org name from whence this entity came
+    :param app_name: The app name from whence this entity came
+    :param collection_name: The collection name from whence this entity came
+    :param entity: The entity retrieved from the source instance
+    :param source_client: The UsergridClient for the source Usergrid instance
+    :param target_client: The UsergridClient for the target Usergrid instance
+    :param attempts: the number of previous attempts this function was run (manual, not part of the framework)
+    :return: an entity.  If response is None then the chain will stop.
+    """
+
+    # return None if you want to stop the chain (filter the entity out)
+    if 'blah' in entity_data:
+        return None
+
+    # return the entity to keep going
+    return entity_data
+
+
+def transform_entity(org_name, app_name, collection_name, entity_data, source_client, target_client, attempts=0):
+    """
+    This is an example handler function which can transform an entity. Multiple handler functions can be used to
+    process a entity.  The response is an entity which will get passed to the next handler in the chain
+
+    :param org_name: The org name from whence this entity came
+    :param app_name: The app name from whence this entity came
+    :param collection_name: The collection name from whence this entity came
+    :param entity: The entity retrieved from the source instance
+    :param source_client: The UsergridClient for the source Usergrid instance
+    :param target_client: The UsergridClient for the target Usergrid instance
+    :param attempts: the number of previous attempts this function was run (manual, not part of the framework)
+    :return: an entity.  If response is None then the chain will stop.
+    """
+    # this just returns the entity with no transform
+    return entity_data
+
+
+def create_new(org_name, app_name, collection_name, entity_data, source_client, target_client, attempts=0):
+    """
+    This is an example handler function which can be used to create a new entity in the target instance (based on the
+    target_client) parameter. Multiple handler functions can be used to process a entity.  The response is an entity
+    which will get passed to the next handler in the chain
+
+    :param org_name: The org name from whence this entity came
+    :param app_name: The app name from whence this entity came
+    :param collection_name: The collection name from whence this entity came
+    :param entity_data: The entity retrieved from the source instance
+    :param source_client: The UsergridClient for the source Usergrid instance
+    :param target_client: The UsergridClient for the target Usergrid instance
+    :param attempts: the number of previous attempts this function was run (manual, not part of the framework)
+    :return: an entity.  If response is None then the chain will stop.
+    """
+
+    attempts += 1
+
+    if 'metadata' in entity_data: entity_data.pop('metadata')
+
+    target_org = config.get('target_org')
+    target_app = config.get('app_mapping', {}).get(app_name, app_name)
+    target_collection = config.get('collection_mapping', {}).get(collection_name, collection_name)
+
+    if target_client:
+        try:
+            c = target_client.org(target_org).app(target_app).collection(target_collection)
+            e = c.entity_from_data(entity_data)
+            e.put()
+
+        except UsergridError, err:
+            logger.error(err)
+            raise err
+
+    return None
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Usergrid App/Collection Iterator')
+
+    parser.add_argument('-o', '--org',
+                        help='Name of the org to migrate',
+                        type=str,
+                        required=True)
+
+    parser.add_argument('-a', '--app',
+                        help='Multiple, name of apps to include, skip to include all',
+                        default=[],
+                        action='append')
+
+    parser.add_argument('-c', '--collection',
+                        help='Multiple, name of collections to include, skip to include all',
+                        default=[],
+                        action='append')
+
+    parser.add_argument('--ql',
+                        help='The Query string for processing the source collection(s)',
+                        type=str,
+                        default='select *')
+
+    parser.add_argument('-s', '--source_config',
+                        help='The configuration of the source endpoint/org',
+                        type=str,
+                        default='source.json')
+
+    parser.add_argument('-d', '--target_config',
+                        help='The configuration of the target endpoint/org',
+                        type=str,
+                        default='destination.json')
+
+    parser.add_argument('-w', '--workers',
+                        help='The number of worker threads',
+                        type=int,
+                        default=1)
+
+    parser.add_argument('-f', '--force',
+                        help='Force an update regardless of modified date',
+                        type=bool,
+                        default=False)
+
+    parser.add_argument('--max_empty_count',
+                        help='The number of iterations for an individual worker to receive no work before stopping',
+                        type=int,
+                        default=3)
+
+    parser.add_argument('--queue_timeout',
+                        help='The duration in seconds for an individual worker queue poll before Empty is raised',
+                        type=int,
+                        default=10)
+
+    parser.add_argument('--map_app',
+                        help="A colon-separated string such as 'apples:oranges' which indicates to put data from the app named 'apples' from the source endpoint into app named 'oranges' in the target endpoint",
+                        default=[],
+                        action='append')
+
+    parser.add_argument('--map_collection',
+                        help="A colon-separated string such as 'cats:dogs' which indicates to put data from collections named 'cats' from the source endpoint into a collection named 'dogs' in the target endpoint, applicable to all apps",
+                        default=[],
+                        action='append')
+
+    parser.add_argument('--target_org',
+                        help="The org name at the Usergrid destination instance",
+                        type=str)
+
+    my_args = parser.parse_args(sys.argv[1:])
+
+    return vars(my_args)
+
+
+def init():
+    global config
+
+    config['collection_mapping'] = {}
+    config['app_mapping'] = {}
+    config['org_mapping'] = {}
+
+    with open(config.get('source_config'), 'r') as f:
+        config['source_config'] = json.load(f)
+
+    with open(config.get('target_config'), 'r') as f:
+        config['target_config'] = json.load(f)
+
+    for mapping in config.get('map_collection', []):
+        parts = mapping.split(':')
+
+        if len(parts) == 2:
+            config['collection_mapping'][parts[0]] = parts[1]
+        else:
+            logger.warning('Skipping malformed Collection mapping: [%s]' % mapping)
+
+    for mapping in config.get('map_app', []):
+        parts = mapping.split(':')
+
+        if len(parts) == 2:
+            config['app_mapping'][parts[0]] = parts[1]
+        else:
+            logger.warning('Skipping malformed App mapping: [%s]' % mapping)
+
+    for mapping in config.get('map_org', []):
+        parts = mapping.split(':')
+
+        if len(parts) == 2:
+            config['org_mapping'][parts[0]] = parts[1]
+        else:
+            logger.warning('Skipping Org mapping: [%s]' % mapping)
+
+    if 'source_config' in config:
+        config['source_endpoint'] = config['source_config'].get('endpoint').copy()
+        config['source_endpoint'].update(config['source_config']['credentials'][config['org']])
+
+    config['target_org'] = config['target_org'] if config['target_org'] else config['org']
+
+    if 'target_config' in config:
+        config['target_endpoint'] = config['target_config'].get('endpoint').copy()
+        config['target_endpoint'].update(config['target_config']['credentials'][config['target_org']])
+
+
+def wait_for(arr_threads, sleep_time=3):
+    """
+    This function pauses the thread until the array of threads which is provided all stop working
+
+    :param arr_threads: an array of Process objects to monitor
+    :param sleep_time: the time to sleep between evaluating the array for completion
+    :return: None
+    """
+    threads_working = 100
+
+    while threads_working > 0:
+        threads_working = 0
+
+        for t in arr_threads:
+
+            if t.is_alive():
+                threads_working += 1
+
+        if threads_working > 0:
+            logger.warn('Waiting for [%s] threads to finish...' % threads_working)
+            time.sleep(sleep_time)
+
+    logger.warn('Worker Threads finished!')
+
+
+class UsergridIterator:
+    def __init__(self):
+        pass
+
+    def get_to_work(self):
+        global config
+
+        queue = Queue()
+        logger.warning('Starting workers...')
+
+        apps_to_process = config.get('app')
+        collections_to_process = config.get('collection')
+        source_org = config['org']
+        target_org = config.get('target_org', config.get('org'))
+
+        source_client = None
+        target_client = None
+
+        try:
+            source_client = UsergridClient(api_url=config['source_endpoint']['api_url'],
+                                           org_name=source_org)
+            source_client.authenticate_management_client(
+                client_credentials=config['source_config']['credentials'][source_org])
+
+        except UsergridError, e:
+            logger.critical(e)
+            exit()
+
+        if 'target_endpoint' in config:
+            try:
+                target_client = UsergridClient(api_url=config['target_endpoint']['api_url'],
+                                               org_name=target_org)
+                target_client.authenticate_management_client(
+                    client_credentials=config['target_config']['credentials'][target_org])
+
+            except UsergridError, e:
+                logger.critical(e)
+                exit()
+
+        function_chain = [filter_entity, transform_entity, create_new]
+
+        workers = [Worker(queue=queue,
+                          source_client=source_client,
+                          target_client=target_client,
+                          function_chain=function_chain,
+                          max_empty_count=config.get('max_empty_count', 3),
+                          queue_timeout=config.get('queue_timeout', 10))
+
+                   for x in xrange(config.get('workers', 1))]
+
+        [w.start() for w in workers]
+
+        for app in source_client.list_apps():
+
+            if len(apps_to_process) > 0 and app not in apps_to_process:
+                logger.warning('Skipping app=[%s]' % app)
+                continue
+
+            logger.warning('Processing app=[%s]' % app)
+
+            source_app = source_client.organization(source_org).application(app)
+
+            for collection_name, collection in source_app.list_collections().iteritems():
+
+                if collection_name in ['events', 'queues']:
+                    logger.warning('Skipping internal collection=[%s]' % collection_name)
+                    continue
+
+                if len(collections_to_process) > 0 and collection_name not in collections_to_process:
+                    logger.warning('Skipping collection=[%s]' % collection_name)
+                    continue
+
+                logger.warning('Processing collection=%s' % collection_name)
+
+                counter = 0
+
+                try:
+                    for entity in collection.query(ql=config.get('ql'),
+                                                   limit=config.get('source_endpoint', {}).get('limit', 100)):
+                        counter += 1
+                        queue.put((config.get('org'), app, collection_name, entity))
+
+                except KeyboardInterrupt:
+                    [w.terminate() for w in workers]
+
+            logger.info('Publishing entities complete!')
+
+        wait_for(workers)
+
+        logger.info('All done!!')
+
+
+def main():
+    global config
+    config = parse_args()
+    init()
+
+    init_logging()
+
+    UsergridIterator().get_to_work()
+
+
+if __name__ == '__main__':
+    main()

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/library_check.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/library_check.py b/utils/usergrid-util-python/usergrid_tools/library_check.py
index f326987..0fc6b2f 100644
--- a/utils/usergrid-util-python/usergrid_tools/library_check.py
+++ b/utils/usergrid-util-python/usergrid_tools/library_check.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import traceback
 
 url_data = {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
index 19d6f24..e374fbc 100644
--- a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
+++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import os
 import uuid
 from Queue import Empty

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
index 30ecc26..c99aa12 100644
--- a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
+++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import os
 import uuid
 from Queue import Empty
@@ -7,9 +26,7 @@ import logging
 import sys
 from multiprocessing import Queue, Process
 from sets import Set
-
 import time_uuid
-
 import datetime
 from cloghandler import ConcurrentRotatingFileHandler
 import requests
@@ -17,9 +34,7 @@ import traceback
 import redis
 import time
 from sys import platform as _platform
-
 import signal
-
 from requests.auth import HTTPBasicAuth
 from usergrid import UsergridQueryIterator
 import urllib3

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py b/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
index 3a2d864..0989401 100644
--- a/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
+++ b/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 import logging
 from logging.handlers import RotatingFileHandler

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py b/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
index e859843..95730d5 100644
--- a/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
+++ b/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *   http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied.  See the License for the
+#    * specific language governing permissions and limitations
+# * under the License.
+# */
+
 import json
 from multiprocessing import Pool
 


Mime
View raw message