incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [4/7] git commit: Adding thrift generated code.
Date Sun, 12 Apr 2015 02:44:11 GMT
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dec4fa7a/blur-thrift/src/main/java/org/apache/blur/thrift/generated/SafeClientGen.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/generated/SafeClientGen.java
b/blur-thrift/src/main/java/org/apache/blur/thrift/generated/SafeClientGen.java
index 84ccd64..8402bad 100644
--- a/blur-thrift/src/main/java/org/apache/blur/thrift/generated/SafeClientGen.java
+++ b/blur-thrift/src/main/java/org/apache/blur/thrift/generated/SafeClientGen.java
@@ -244,6 +244,13 @@ public void loadData(java.lang.String arg0, java.lang.String arg1) throws
org.ap
 }
 
 @Override
+public void loadIndex(java.lang.String arg0, java.util.List arg1) throws org.apache.blur.thrift.generated.BlurException,
org.apache.blur.thirdparty.thrift_0_9_0.TException {
+  _lock.errorFailLock(); try {
+    super.loadIndex(arg0, arg1);
+  } finally {_lock.unlock();}
+}
+
+@Override
 public void logging(java.lang.String arg0, org.apache.blur.thrift.generated.Level arg1) throws
org.apache.blur.thrift.generated.BlurException, org.apache.blur.thirdparty.thrift_0_9_0.TException
{
   _lock.errorFailLock(); try {
     super.logging(arg0, arg1);
@@ -510,6 +517,13 @@ public void recv_loadData() throws org.apache.blur.thrift.generated.BlurExceptio
 }
 
 @Override
+public void recv_loadIndex() throws org.apache.blur.thrift.generated.BlurException, org.apache.blur.thirdparty.thrift_0_9_0.TException
{
+  _lock.errorFailLock(); try {
+    super.recv_loadIndex();
+  } finally {_lock.unlock();}
+}
+
+@Override
 public void recv_logging() throws org.apache.blur.thrift.generated.BlurException, org.apache.blur.thirdparty.thrift_0_9_0.TException
{
   _lock.errorFailLock(); try {
     super.recv_logging();
@@ -706,6 +720,13 @@ public java.util.List recv_traceRequestList() throws org.apache.blur.thrift.gene
 }
 
 @Override
+public void recv_validateIndex() throws org.apache.blur.thrift.generated.BlurException, org.apache.blur.thirdparty.thrift_0_9_0.TException
{
+  _lock.errorFailLock(); try {
+    super.recv_validateIndex();
+  } finally {_lock.unlock();}
+}
+
+@Override
 public void refresh() throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
   _lock.errorFailLock(); try {
     super.refresh();
@@ -923,6 +944,13 @@ public void send_loadData(java.lang.String arg0, java.lang.String arg1)
throws o
 }
 
 @Override
+public void send_loadIndex(java.lang.String arg0, java.util.List arg1) throws org.apache.blur.thirdparty.thrift_0_9_0.TException
{
+  _lock.errorFailLock(); try {
+    super.send_loadIndex(arg0, arg1);
+  } finally {_lock.unlock();}
+}
+
+@Override
 public void send_logging(java.lang.String arg0, org.apache.blur.thrift.generated.Level arg1)
throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
   _lock.errorFailLock(); try {
     super.send_logging(arg0, arg1);
@@ -1140,6 +1168,13 @@ public void send_traceRequestList(java.lang.String arg0) throws org.apache.blur.
 }
 
 @Override
+public void send_validateIndex(java.lang.String arg0, java.util.List arg1) throws org.apache.blur.thirdparty.thrift_0_9_0.TException
{
+  _lock.errorFailLock(); try {
+    super.send_validateIndex(arg0, arg1);
+  } finally {_lock.unlock();}
+}
+
+@Override
 public void setUser(org.apache.blur.thrift.generated.User arg0) throws org.apache.blur.thirdparty.thrift_0_9_0.TException
{
   _lock.errorFailLock(); try {
     super.setUser(arg0);
@@ -1237,4 +1272,11 @@ public java.util.List traceRequestList(java.lang.String arg0) throws
org.apache.
   } finally {_lock.unlock();}
 }
 
+@Override
+public void validateIndex(java.lang.String arg0, java.util.List arg1) throws org.apache.blur.thrift.generated.BlurException,
org.apache.blur.thirdparty.thrift_0_9_0.TException {
+  _lock.errorFailLock(); try {
+    super.validateIndex(arg0, arg1);
+  } finally {_lock.unlock();}
+}
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dec4fa7a/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataBulkLoad.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataBulkLoad.java b/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataBulkLoad.java
new file mode 100644
index 0000000..619fc4b
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataBulkLoad.java
@@ -0,0 +1,177 @@
+package org.apache.blur.thrift.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.RecordMutation;
+import org.apache.blur.thrift.generated.RecordMutationType;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.RowMutationType;
+
+public class LoadDataBulkLoad {
+
+  private static final long _5_SECONDS = TimeUnit.SECONDS.toNanos(5);
+  private static Random random = new Random();
+  private static List<String> words = new ArrayList<String>();
+
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    final int numberOfColumns = 3;
+    int numberRows = 1000000;
+    final int numberRecordsPerRow = 3;
+    final int numberOfFamilies = 3;
+    final int numberOfWords = 30;
+    int batch = 1;
+    String connectionString = args[0];
+    String table = args[1];
+    Iface client = BlurClient.getClient(connectionString);
+    runLoad(client, false, table, numberRows, numberRecordsPerRow, numberOfFamilies, numberOfColumns,
numberOfWords,
+        batch, new PrintWriter(System.out));
+  }
+
+  public static void runLoad(Iface client, boolean enqueue, String table, int numberRows,
int numberRecordsPerRow,
+      int numberOfFamilies, int numberOfColumns, int numberOfWords, int batch, PrintWriter
out) throws IOException,
+      BlurException, TException {
+    loadWords();
+
+    int countRow = 0;
+    int countRecord = 0;
+    final long start = System.currentTimeMillis();
+    long s = start;
+
+    List<RowMutation> mutations = new ArrayList<RowMutation>();
+
+    String bulkId = UUID.randomUUID().toString();
+    client.bulkMutateStart(bulkId);
+
+    long ts = System.nanoTime();
+    for (int i = 0; i < numberRows; i++) {
+      long now = System.nanoTime();
+      if (ts + _5_SECONDS < now) {
+        printPerformance(out, countRow, countRecord, start, s, i);
+        countRow = 0;
+        countRecord = 0;
+        s = System.currentTimeMillis();
+        ts = System.nanoTime();
+      }
+
+      RowMutation mutation = new RowMutation();
+      mutation.setTable(table);
+      String rowId = getRowId();
+      mutation.setRowId(rowId);
+      mutation.setRowMutationType(RowMutationType.REPLACE_ROW);
+      for (int j = 0; j < numberRecordsPerRow; j++) {
+        mutation.addToRecordMutations(getRecordMutation(numberOfColumns, numberOfFamilies,
numberOfWords));
+        countRecord++;
+      }
+      mutations.add(mutation);
+      if (mutations.size() >= batch) {
+        client.bulkMutateAddMultiple(bulkId, mutations);
+        mutations.clear();
+      }
+      countRow++;
+    }
+    if (!mutations.isEmpty()) {
+      client.bulkMutateAddMultiple(bulkId, mutations);
+    }
+    client.bulkMutateFinish(bulkId, true, true);
+    printPerformance(out, countRow, countRecord, start, s, numberRows);
+  }
+
+  private static void printPerformance(PrintWriter out, int countRow, int countRecord, final
long start, long s, int i) {
+    double totalSeconds = (System.currentTimeMillis() - start) / 1000.0;
+    double seconds = (System.currentTimeMillis() - s) / 1000.0;
+    double recordRate = countRecord / seconds;
+    double rowRate = countRow / seconds;
+    double avgRowRate = i / totalSeconds;
+    out.printf("Rows indexed [%d] at Avg Rows [%f/s] Rows [%f/s] Records [%f/s]%n", i, avgRowRate,
rowRate, recordRate);
+    out.flush();
+  }
+
+  public static void loadWords() throws IOException {
+    InputStream inputStream = LoadDataBulkLoad.class.getResourceAsStream("words.txt");
+    BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+    String word;
+    while ((word = reader.readLine()) != null) {
+      words.add(word.trim());
+    }
+    reader.close();
+  }
+
+  protected static RecordMutation getRecordMutation(int numberOfColumns, int numberOfFamilies,
int numberOfWords) {
+    RecordMutation recordMutation = new RecordMutation();
+    recordMutation.setRecord(getRecord(numberOfColumns, numberOfFamilies, numberOfWords));
+    recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD);
+    return recordMutation;
+  }
+
+  private static Record getRecord(int numberOfColumns, int numberOfFamilies, int numberOfWords)
{
+    Record record = new Record();
+    record.setRecordId(getRowId());
+    record.setFamily(getFamily(numberOfFamilies));
+    for (int i = 0; i < numberOfColumns; i++) {
+      record.addToColumns(new Column("col" + i, getWords(numberOfWords)));
+    }
+    return record;
+  }
+
+  private static String getWords(int numberOfWords) {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < numberOfWords; i++) {
+      if (i != 0) {
+        builder.append(' ');
+      }
+      builder.append(getWord());
+    }
+    return builder.toString();
+  }
+
+  private static String getFamily(int numberOfFamilies) {
+    return "fam" + random.nextInt(numberOfFamilies);
+  }
+
+  public static String getWord() {
+    return makeUpperCaseRandomly(words.get(random.nextInt(words.size())), random);
+  }
+
+  private static String makeUpperCaseRandomly(String s, Random r) {
+    if (r.nextBoolean()) {
+      return s.substring(0, 1).toUpperCase() + s.substring(1);
+    }
+    return s;
+  }
+
+  protected static String getRowId() {
+    return Long.toString(Math.abs(random.nextLong())) + "-" + Long.toString(Math.abs(random.nextLong()));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dec4fa7a/blur-thrift/src/main/java/org/apache/blur/thrift/util/QueryTrace.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/util/QueryTrace.java b/blur-thrift/src/main/java/org/apache/blur/thrift/util/QueryTrace.java
new file mode 100644
index 0000000..b819472
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thrift/util/QueryTrace.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.thrift.util;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.apache.blur.thrift.generated.BlurResults;
+import org.apache.blur.thrift.generated.Query;
+import org.apache.blur.trace.Trace;
+
+public class QueryTrace {
+
+  public static void main(String[] args) throws BlurException, TException {
+    Iface client = BlurClient.getClient("localhost:40010");
+    
+    BlurQuery blurQuery = new BlurQuery();
+    Query query = new Query();
+    query.setQuery("*");
+    blurQuery.setQuery(query);
+    Trace.setupTrace(Long.toString(System.currentTimeMillis()));
+    BlurResults blurResults = client.query("test_hdfs", blurQuery);
+    System.out.println(blurResults);
+    Trace.tearDownTrace();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dec4fa7a/distribution/src/main/scripts/interface/gen-html/Blur.html
----------------------------------------------------------------------
diff --git a/distribution/src/main/scripts/interface/gen-html/Blur.html b/distribution/src/main/scripts/interface/gen-html/Blur.html
index 16dc9fd..cc9973b 100644
--- a/distribution/src/main/scripts/interface/gen-html/Blur.html
+++ b/distribution/src/main/scripts/interface/gen-html/Blur.html
@@ -37,6 +37,7 @@
 <li><a href="Blur.html#Fn_Blur_listInstalledCommands">listInstalledCommands</a></li>
 <li><a href="Blur.html#Fn_Blur_listSnapshots">listSnapshots</a></li>
 <li><a href="Blur.html#Fn_Blur_loadData">loadData</a></li>
+<li><a href="Blur.html#Fn_Blur_loadIndex">loadIndex</a></li>
 <li><a href="Blur.html#Fn_Blur_logging">logging</a></li>
 <li><a href="Blur.html#Fn_Blur_metrics">metrics</a></li>
 <li><a href="Blur.html#Fn_Blur_mutate">mutate</a></li>
@@ -68,6 +69,7 @@
 <li><a href="Blur.html#Fn_Blur_traceRemove">traceRemove</a></li>
 <li><a href="Blur.html#Fn_Blur_traceRequestFetch">traceRequestFetch</a></li>
 <li><a href="Blur.html#Fn_Blur_traceRequestList">traceRequestList</a></li>
+<li><a href="Blur.html#Fn_Blur_validateIndex">validateIndex</a></li>
 </ul>
 </td>
 <td><a href="Blur.html#Struct_ArgumentDescriptor">ArgumentDescriptor</a><br/>
@@ -884,7 +886,15 @@ the shard server, if any.
 </td></tr>
 <tr><td>location</td><td>Location of bulk data load.
 </td></tr>
-</table></div><div class="definition"><h4 id="Fn_Blur_mutate">Function:
Blur.mutate</h4>
+</table></div><div class="definition"><h4 id="Fn_Blur_validateIndex">Function:
Blur.validateIndex</h4>
+<pre><code>void</code> validateIndex(<code>string</code> table,
+                   <code>list&lt;<code>string</code>&gt;</code>
externalIndexPaths)
+    throws <code><a href="Blur.html#Struct_BlurException">BlurException</a></code>
+</pre></div><div class="definition"><h4 id="Fn_Blur_loadIndex">Function:
Blur.loadIndex</h4>
+<pre><code>void</code> loadIndex(<code>string</code> table,
+               <code>list&lt;<code>string</code>&gt;</code>
externalIndexPaths)
+    throws <code><a href="Blur.html#Struct_BlurException">BlurException</a></code>
+</pre></div><div class="definition"><h4 id="Fn_Blur_mutate">Function:
Blur.mutate</h4>
 <pre><code>void</code> mutate(<code><a href="Blur.html#Struct_RowMutation">RowMutation</a></code>
mutation)
     throws <code><a href="Blur.html#Struct_BlurException">BlurException</a></code>
 </pre>Mutates a Row given the RowMutation that is provided.

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/dec4fa7a/distribution/src/main/scripts/interface/gen-html/index.html
----------------------------------------------------------------------
diff --git a/distribution/src/main/scripts/interface/gen-html/index.html b/distribution/src/main/scripts/interface/gen-html/index.html
index 347e18e..b02f97b 100644
--- a/distribution/src/main/scripts/interface/gen-html/index.html
+++ b/distribution/src/main/scripts/interface/gen-html/index.html
@@ -33,6 +33,7 @@
 <li><a href="Blur.html#Fn_Blur_listInstalledCommands">listInstalledCommands</a></li>
 <li><a href="Blur.html#Fn_Blur_listSnapshots">listSnapshots</a></li>
 <li><a href="Blur.html#Fn_Blur_loadData">loadData</a></li>
+<li><a href="Blur.html#Fn_Blur_loadIndex">loadIndex</a></li>
 <li><a href="Blur.html#Fn_Blur_logging">logging</a></li>
 <li><a href="Blur.html#Fn_Blur_metrics">metrics</a></li>
 <li><a href="Blur.html#Fn_Blur_mutate">mutate</a></li>
@@ -64,6 +65,7 @@
 <li><a href="Blur.html#Fn_Blur_traceRemove">traceRemove</a></li>
 <li><a href="Blur.html#Fn_Blur_traceRequestFetch">traceRequestFetch</a></li>
 <li><a href="Blur.html#Fn_Blur_traceRequestList">traceRequestList</a></li>
+<li><a href="Blur.html#Fn_Blur_validateIndex">validateIndex</a></li>
 </ul>
 </td>
 <td><a href="Blur.html#Struct_ArgumentDescriptor">ArgumentDescriptor</a><br/>


Mime
View raw message