asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject incubator-asterixdb git commit: Improve KV Generator
Date Wed, 16 Mar 2016 16:59:07 GMT
Repository: incubator-asterixdb
Updated Branches:
  refs/heads/master fc6448227 -> 333eae01c


Improve KV Generator

Currently, KV Generator produces the same documents when
running in parallel where the only difference is in the
key of the record. With this change, it produces different
Documents in each partition to produce more diverse set of
records.

Change-Id: Ia337be57f8394d74a98f3c72ebcaa584f277f34b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/724
Reviewed-by: Michael Blow <michael.blow@couchbase.com>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/333eae01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/333eae01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/333eae01

Branch: refs/heads/master
Commit: 333eae01cd8721b3dc4096880ac0cac69358aef5
Parents: fc64482
Author: Abdullah Alamoudi <bamousaa@gmail.com>
Authored: Wed Mar 16 12:19:26 2016 +0300
Committer: abdullah alamoudi <bamousaa@gmail.com>
Committed: Wed Mar 16 09:53:38 2016 -0700

----------------------------------------------------------------------
 .../external/input/record/reader/kv/KVTestReader.java       | 9 +++++----
 .../input/record/reader/kv/KVTestReaderFactory.java         | 4 +++-
 .../asterix/external/generator/test/DCPGeneratorTest.java   | 2 +-
 3 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/333eae01/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
index 9e797e3..f540f4e 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
@@ -45,6 +45,7 @@ public class KVTestReader implements IRecordReader<DCPRequest> {
     private final ArrayList<Short> assigned;
     private final int numberOfMutations;
     private int counter = 0;
+    private int upsertCounter = 0;
     private boolean stopped = false;
     // for deterministic data generation
     private int expiration = 7999;
@@ -59,14 +60,13 @@ public class KVTestReader implements IRecordReader<DCPRequest> {
     private short nextUpsertPartition;
     private final ByteBuf byteBuff;
     private final StringBuilder strBuilder = new StringBuilder();
-    private int upsertCounter = 0;
     private final String[] names = { "Michael Carey", "Till Westmann", "Michael Blow", "Chris
Hillary", "Yingyi Bu",
             "Ian Maxon", "Abdullah Alamoudi" };
 
-    public KVTestReader(final int partition, final String bucket, final int[] schedule,
-            final int numberOfMutations, final int deleteCycle, final int upsertCycle) {
+    public KVTestReader(final int partition, final String bucket, final int[] schedule, final
int numberOfMutations,
+            final int deleteCycle, final int upsertCycle, int counterStart) {
         this.bucket = bucket;
-        this.numberOfMutations = numberOfMutations;
+        this.numberOfMutations = numberOfMutations + counterStart;
         this.assigned = new ArrayList<>();
         this.deleteCycle = deleteCycle;
         this.upsertCycle = upsertCycle;
@@ -82,6 +82,7 @@ public class KVTestReader implements IRecordReader<DCPRequest> {
         this.byteBuff = ByteBufAllocator.DEFAULT.buffer(ExternalDataConstants.DEFAULT_BUFFER_SIZE);
         byteBuff.retain();
         this.record = new GenericRecord<DCPRequest>();
+        this.counter = counterStart;
     }
 
     private String generateKey() {

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/333eae01/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
index 8242554..adaa525 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
@@ -37,11 +37,13 @@ public class KVTestReaderFactory implements IRecordReaderFactory<DCPRequest>
{
     private int numOfRecords = 1000; // default = 1 Million
     private int deleteCycle = 0;
     private int upsertCycle = 0;
+    private int numOfReaders;
     private transient AlgebricksAbsolutePartitionConstraint clusterLocations;
 
     @Override
     public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() {
         clusterLocations = AsterixClusterProperties.INSTANCE.getClusterLocations();
+        numOfReaders = clusterLocations.getLocations().length;
         return clusterLocations;
     }
 
@@ -68,7 +70,7 @@ public class KVTestReaderFactory implements IRecordReaderFactory<DCPRequest>
{
     public IRecordReader<? extends DCPRequest> createRecordReader(final IHyracksTaskContext
ctx, final int partition) {
         return new KVTestReader(partition, bucket, schedule,
                 (int) Math.ceil((double) numOfRecords / (double) getPartitionConstraint().getLocations().length),
-                deleteCycle, upsertCycle);
+                deleteCycle, upsertCycle, (numOfRecords / numOfReaders) * partition);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/333eae01/asterix-external-data/src/test/java/org/apache/asterix/external/generator/test/DCPGeneratorTest.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/generator/test/DCPGeneratorTest.java
b/asterix-external-data/src/test/java/org/apache/asterix/external/generator/test/DCPGeneratorTest.java
index 14b6ed1..b08fc7d 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/generator/test/DCPGeneratorTest.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/generator/test/DCPGeneratorTest.java
@@ -32,7 +32,7 @@ public class DCPGeneratorTest {
     @Test
     public void runTest() throws Exception {
         try (KVTestReader cbreader = new KVTestReader(0, "TestBucket",
-                new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, 150,
0, 0)) {
+                new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, 150,
0, 0, 0)) {
             final UTF8StringPointable pointable = new UTF8StringPointable();
             final DCPRequestToRecordWithMetadataAndPKConverter converter = new DCPRequestToRecordWithMetadataAndPKConverter();
             while (cbreader.hasNext()) {


Mime
View raw message