accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [2/2] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Date Thu, 05 Dec 2013 16:57:02 GMT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/00fb08b3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/00fb08b3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/00fb08b3

Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 00fb08b3c4eed71865107c40c3036da39772cb41
Parents: 0833f4f 513f4d2
Author: Christopher Tubbs <ctubbsii@apache.org>
Authored: Thu Dec 5 11:55:02 2013 -0500
Committer: Christopher Tubbs <ctubbsii@apache.org>
Committed: Thu Dec 5 11:55:02 2013 -0500

----------------------------------------------------------------------
 docs/examples/README.filedata                                    | 4 ++--
 .../apache/accumulo/examples/simple/filedata/FileDataIngest.java | 2 +-
 .../apache/accumulo/examples/simple/filedata/FileDataQuery.java  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/00fb08b3/docs/examples/README.filedata
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/00fb08b3/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java
index ef17f04,0000000..78fef0d
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java
@@@ -1,203 -1,0 +1,203 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.examples.simple.filedata;
 +
 +import java.io.FileInputStream;
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.security.MessageDigest;
 +import java.security.NoSuchAlgorithmException;
 +import java.util.ArrayList;
 +import java.util.List;
 +
 +import org.apache.accumulo.core.cli.BatchWriterOpts;
 +import org.apache.accumulo.core.cli.ClientOnRequiredTable;
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.data.ArrayByteSequence;
 +import org.apache.accumulo.core.data.ByteSequence;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.ColumnVisibility;
 +import org.apache.hadoop.io.Text;
 +
 +import com.beust.jcommander.Parameter;
 +
 +/**
-  * Takes a list of files and archives them into Accumulo keyed on the SHA1 hashes of the
files. See docs/examples/README.filedata for instructions.
++ * Takes a list of files and archives them into Accumulo keyed on hashes of the files. See
docs/examples/README.filedata for instructions.
 + */
 +public class FileDataIngest {
 +  public static final Text CHUNK_CF = new Text("~chunk");
 +  public static final Text REFS_CF = new Text("refs");
 +  public static final String REFS_ORIG_FILE = "name";
 +  public static final String REFS_FILE_EXT = "filext";
 +  public static final ByteSequence CHUNK_CF_BS = new ArrayByteSequence(CHUNK_CF.getBytes(),
0, CHUNK_CF.getLength());
 +  public static final ByteSequence REFS_CF_BS = new ArrayByteSequence(REFS_CF.getBytes(),
0, REFS_CF.getLength());
 +  
 +  int chunkSize;
 +  byte[] chunkSizeBytes;
 +  byte[] buf;
 +  MessageDigest md5digest;
 +  ColumnVisibility cv;
 +  
 +  public FileDataIngest(int chunkSize, ColumnVisibility colvis) {
 +    this.chunkSize = chunkSize;
 +    chunkSizeBytes = intToBytes(chunkSize);
 +    buf = new byte[chunkSize];
 +    try {
 +      md5digest = MessageDigest.getInstance("MD5");
 +    } catch (NoSuchAlgorithmException e) {
 +      throw new RuntimeException(e);
 +    }
 +    cv = colvis;
 +  }
 +  
 +  public String insertFileData(String filename, BatchWriter bw) throws MutationsRejectedException,
IOException {
 +    if (chunkSize == 0)
 +      return "";
 +    md5digest.reset();
 +    String uid = hexString(md5digest.digest(filename.getBytes()));
 +    
 +    // read through file once, calculating hashes
 +    md5digest.reset();
 +    InputStream fis = null;
 +    int numRead = 0;
 +    try {
 +	    fis = new FileInputStream(filename);
 +	    numRead = fis.read(buf);
 +	    while (numRead >= 0) {
 +	      if (numRead > 0) {
 +	        md5digest.update(buf, 0, numRead);
 +	      }
 +	      numRead = fis.read(buf);
 +	    }
 +    } finally {
 +      if (fis != null) {
 +    	  fis.close();
 +      }
 +    }
 +    
 +    String hash = hexString(md5digest.digest());
 +    Text row = new Text(hash);
 +    
 +    // write info to accumulo
 +    Mutation m = new Mutation(row);
 +    m.put(REFS_CF, KeyUtil.buildNullSepText(uid, REFS_ORIG_FILE), cv, new Value(filename.getBytes()));
 +    String fext = getExt(filename);
 +    if (fext != null)
 +      m.put(REFS_CF, KeyUtil.buildNullSepText(uid, REFS_FILE_EXT), cv, new Value(fext.getBytes()));
 +    bw.addMutation(m);
 +    
 +    // read through file again, writing chunks to accumulo
 +    int chunkCount = 0;
 +    try {
 +	    fis = new FileInputStream(filename);
 +	    numRead = fis.read(buf);
 +	    while (numRead >= 0) {
 +	      while (numRead < buf.length) {
 +	        int moreRead = fis.read(buf, numRead, buf.length - numRead);
 +	        if (moreRead > 0)
 +	          numRead += moreRead;
 +	        else if (moreRead < 0)
 +	          break;
 +	      }
 +	      m = new Mutation(row);
 +	      Text chunkCQ = new Text(chunkSizeBytes);
 +	      chunkCQ.append(intToBytes(chunkCount), 0, 4);
 +	      m.put(CHUNK_CF, chunkCQ, cv, new Value(buf, 0, numRead));
 +	      bw.addMutation(m);
 +	      if (chunkCount == Integer.MAX_VALUE)
 +	        throw new RuntimeException("too many chunks for file " + filename + ", try raising
chunk size");
 +	      chunkCount++;
 +	      numRead = fis.read(buf);
 +	    }
 +    } finally {
 +    	if (fis != null) {
 +    		fis.close();
 +    	}
 +    }
 +    m = new Mutation(row);
 +    Text chunkCQ = new Text(chunkSizeBytes);
 +    chunkCQ.append(intToBytes(chunkCount), 0, 4);
 +    m.put(new Text(CHUNK_CF), chunkCQ, cv, new Value(new byte[0]));
 +    bw.addMutation(m);
 +    return hash;
 +  }
 +  
 +  public static int bytesToInt(byte[] b, int offset) {
 +    if (b.length <= offset + 3)
 +      throw new NumberFormatException("couldn't pull integer from bytes at offset " + offset);
 +    int i = (((b[offset] & 255) << 24) + ((b[offset + 1] & 255) << 16)
+ ((b[offset + 2] & 255) << 8) + ((b[offset + 3] & 255) << 0));
 +    return i;
 +  }
 +  
 +  public static byte[] intToBytes(int l) {
 +    byte[] b = new byte[4];
 +    b[0] = (byte) (l >>> 24);
 +    b[1] = (byte) (l >>> 16);
 +    b[2] = (byte) (l >>> 8);
 +    b[3] = (byte) (l >>> 0);
 +    return b;
 +  }
 +  
 +  private static String getExt(String filename) {
 +    if (filename.indexOf(".") == -1)
 +      return null;
 +    return filename.substring(filename.lastIndexOf(".") + 1);
 +  }
 +  
 +  public String hexString(byte[] bytes) {
 +    StringBuilder sb = new StringBuilder();
 +    for (byte b : bytes) {
 +      sb.append(String.format("%02x", b));
 +    }
 +    return sb.toString();
 +  }
 +  
 +  public static class Opts extends ClientOnRequiredTable {
 +    @Parameter(names="--vis", description="use a given visibility for the new counts", converter=VisibilityConverter.class)
 +    ColumnVisibility visibility = new ColumnVisibility();
 +    
 +    @Parameter(names="--chunk", description="size of the chunks used to store partial files")
 +    int chunkSize = 64*1024;
 +    
 +    @Parameter(description="<file> { <file> ... }")
 +    List<String> files = new ArrayList<String>();
 +  }
 +  
 +  
 +  public static void main(String[] args) throws Exception {
 +    Opts opts = new Opts();
 +    BatchWriterOpts bwOpts = new BatchWriterOpts();
 +    opts.parseArgs(FileDataIngest.class.getName(), args, bwOpts);
 +    
 +    Connector conn = opts.getConnector();
 +    if (!conn.tableOperations().exists(opts.tableName)) {
 +      conn.tableOperations().create(opts.tableName);
 +      conn.tableOperations().attachIterator(opts.tableName, new IteratorSetting(1, ChunkCombiner.class));
 +    }
 +    BatchWriter bw = conn.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
 +    FileDataIngest fdi = new FileDataIngest(opts.chunkSize, opts.visibility);
 +    for (String filename : opts.files) {
 +      fdi.insertFileData(filename, bw);
 +    }
 +    bw.close();
 +    opts.stopTracing();
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/00fb08b3/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataQuery.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataQuery.java
index ecb42c7,0000000..77ad4db
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataQuery.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataQuery.java
@@@ -1,85 -1,0 +1,85 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.examples.simple.filedata;
 +
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.List;
 +import java.util.Map.Entry;
 +
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.ZooKeeperInstance;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.PeekingIterator;
 +
 +/**
-  * Retrieves file data based on the SHA1 hash of the file. Used by the {@link org.apache.accumulo.examples.simple.dirlist.Viewer}.
See README.dirlist for
++ * Retrieves file data based on the hash of the file. Used by the {@link org.apache.accumulo.examples.simple.dirlist.Viewer}.
See README.dirlist for
 + * instructions.
 + */
 +public class FileDataQuery {
 +  private Connector conn = null;
 +  List<Entry<Key,Value>> lastRefs;
 +  private ChunkInputStream cis;
 +  Scanner scanner;
 +  
 +  public FileDataQuery(String instanceName, String zooKeepers, String user, AuthenticationToken
token, String tableName, Authorizations auths) throws AccumuloException,
 +      AccumuloSecurityException, TableNotFoundException {
 +    ZooKeeperInstance instance = new ZooKeeperInstance(instanceName, zooKeepers);
 +    conn = instance.getConnector(user, token);
 +    lastRefs = new ArrayList<Entry<Key,Value>>();
 +    cis = new ChunkInputStream();
 +    scanner = conn.createScanner(tableName, auths);
 +  }
 +  
 +  public List<Entry<Key,Value>> getLastRefs() {
 +    return lastRefs;
 +  }
 +  
 +  public ChunkInputStream getData(String hash) throws IOException {
 +    scanner.setRange(new Range(hash));
 +    scanner.setBatchSize(1);
 +    lastRefs.clear();
 +    PeekingIterator<Entry<Key,Value>> pi = new PeekingIterator<Entry<Key,Value>>(scanner.iterator());
 +    if (pi.hasNext()) {
 +      while (!pi.peek().getKey().getColumnFamily().equals(FileDataIngest.CHUNK_CF)) {
 +        lastRefs.add(pi.peek());
 +        pi.next();
 +      }
 +    }
 +    cis.clear();
 +    cis.setSource(pi);
 +    return cis;
 +  }
 +  
 +  public String getSomeData(String hash, int numBytes) throws IOException {
 +    ChunkInputStream is = getData(hash);
 +    byte[] buf = new byte[numBytes];
 +    if (is.read(buf) >= 0) {
 +      return new String(buf);
 +    } else {
 +      return "";
 +    }
 +  }
 +}


Mime
View raw message