cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From eev...@apache.org
Subject svn commit: r1069125 - /cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
Date Wed, 09 Feb 2011 21:38:57 GMT
Author: eevans
Date: Wed Feb  9 21:38:57 2011
New Revision: 1069125

URL: http://svn.apache.org/viewvc?rev=1069125&view=rev
Log:
reverting the insanity of my last merge

Modified:
    cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java

Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java?rev=1069125&r1=1069124&r2=1069125&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java Wed Feb  9 21:38:57
2011
@@ -29,6 +29,11 @@ import org.apache.commons.cli.*;
 
 import org.apache.cassandra.config.ConfigurationException;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamily;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.ExpiringColumn;
+import org.apache.cassandra.db.IColumn;
+import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.io.sstable.*;
 
@@ -40,8 +45,7 @@ import static org.apache.cassandra.utils
  */
 public class SSTableExport
 {
-    // size of the columns page
-    private static final int PAGE_SIZE = 1000;
+    private static int INPUT_FILE_BUFFER_SIZE = 8 * 1024 * 1024;
 
     private static final String KEY_OPTION = "k";
     private static final String EXCLUDEKEY_OPTION = "x";
@@ -66,22 +70,12 @@ public class SSTableExport
         Option optEnumerate = new Option(ENUMERATEKEYS_OPTION, false, "enumerate keys only");
         options.addOption(optEnumerate);
     }
-
-    /**
-     * Wraps given string into quotes
-     * @param val string to quote
-     * @return quoted string
-     */
+    
     private static String quote(String val)
     {
         return String.format("\"%s\"", val);
     }
-
-    /**
-     * JSON Hash Key serializer
-     * @param val value to set as a key
-     * @return JSON Hash key
-     */
+    
     private static String asKey(String val)
     {
         return String.format("%s: ", quote(val));
@@ -89,9 +83,7 @@ public class SSTableExport
     
     private static void serializeColumns(PrintStream outs, Collection<IColumn> columns,
AbstractType comparator, CFMetaData cfMetaData)
     {
-        while (columns.hasNext())
-        {
-            serializeColumn(columns.next(), out);
+        outs.print("[");
 
         Iterator<IColumn> iter = columns.iterator();
 
@@ -126,18 +118,11 @@ public class SSTableExport
                 outs.print(", ");
             }
         }
-
-        out.print("]");
+        
+        outs.print("]");
     }
-
-    /**
-     * Get portion of the columns and serialize in loop while not more columns left in the
row
-     * @param reader SSTableReader for given SSTable
-     * @param row SSTableIdentityIterator row representation with Column Family
-     * @param key Decorated Key for the required row
-     * @param out output stream
-     */
-    private static void serializeRow(SSTableReader reader, SSTableIdentityIterator row, DecoratedKey
key, PrintStream out)
+    
+    private static void serializeRow(PrintStream outs, SSTableIdentityIterator row) throws
IOException
     {
         ColumnFamily columnFamily = row.getColumnFamilyWithColumns();
         CFMetaData cfMetaData = columnFamily.metadata();
@@ -149,12 +134,7 @@ public class SSTableExport
 
         if (columnFamily.isSuper())
         {
-            QueryFilter filter = QueryFilter.getSliceFilter(key,
-                                                            new QueryPath(columnFamily.metadata().tableName),
-                                                            startColumn,
-                                                            ByteBufferUtil.EMPTY_BYTE_BUFFER,
-                                                            false,
-                                                            PAGE_SIZE);
+            outs.print("{ ");
 
             Iterator<IColumn> iter = columnFamily.getSortedColumns().iterator();
             while (iter.hasNext())
@@ -179,55 +159,8 @@ public class SSTableExport
                     outs.print(", ");
                 }
             }
-
-            try
-            {
-                columns = filter.getSSTableColumnIterator(reader); // iterator reset
-                serializeRow(columns, isSuperCF, out);
-            }
-            catch (IOException e)
-            {
-                System.err.println("WARNING: Corrupt row " + key + " (skipping).");
-            }
-
-            if (columnCount < PAGE_SIZE)
-                break;
-        }
-
-        out.print(isSuperCF ? "}" : "]");
-    }
-
-    /**
-     * Serialize a row with already given column iterator
-     *
-     * @param columns columns of the row
-     * @param isSuper true if wrapping Column Family is Super
-     * @param out output stream
-     *
-     * @throws IOException on any I/O error.
-     */
-    private static void serializeRow(IColumnIterator columns, boolean isSuper, PrintStream
out) throws IOException
-    {
-        if (isSuper)
-        {
-            while (columns.hasNext())
-            {
-                IColumn column = columns.next();
-
-                out.print(asKey(bytesToHex(column.name())));
-                out.print("{");
-                out.print(asKey("deletedAt"));
-                out.print(column.getMarkedForDeleteAt());
-                out.print(", ");
-                out.print(asKey("subColumns"));
-                out.print("[");
-                serializeColumns(column.getSubColumns(), out);
-                out.print("]");
-                out.print("}");
-
-                if (columns.hasNext())
-                    out.print(", ");
-            }
+            
+            outs.print("}");
         }
         else
         {
@@ -266,10 +199,9 @@ public class SSTableExport
     /**
      * Export specific rows from an SSTable and write the resulting JSON to a PrintStream.
      * 
-     * @param ssTableFile the SSTableScanner to export the rows from
+     * @param ssTableFile the SSTable to export the rows from
      * @param outs PrintStream to write the output to
-     * @param toExport the keys corresponding to the rows to export
-     * @param excludes keys to exclude from export
+     * @param keys the keys corresponding to the rows to export
      * @param excludes the keys to exclude from export
      *
      * @throws IOException on failure to read/write input/output
@@ -277,23 +209,20 @@ public class SSTableExport
     public static void export(String ssTableFile, PrintStream outs, String[] keys, String[]
excludes) throws IOException
     {
         SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(ssTableFile));
-        SSTableScanner scanner = reader.getDirectScanner(BufferedRandomAccessFile.DEFAULT_BUFFER_SIZE);
-
-        IPartitioner<?> partitioner = StorageService.getPartitioner();
-
-        for (String toExclude : excludes)
-        {
-            toExport.remove(toExclude); // excluding key from export
-        }
+        SSTableScanner scanner = reader.getDirectScanner(INPUT_FILE_BUFFER_SIZE);
+        IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();    
+        Set<String> excludeSet = new HashSet<String>();
+        int i = 0;
 
+        if (excludes != null)
+            excludeSet = new HashSet<String>(Arrays.asList(excludes));
+        
         outs.println("{");
 
-        int i = 0;
-
-        // last key to compare order
+        // last key to compare order 
         DecoratedKey lastKey = null;
-
-        for (String key : toExport)
+        
+        for (String key : keys)
         {
             if (excludeSet.contains(key))
                 continue;
@@ -306,65 +235,84 @@ public class SSTableExport
 
             lastKey = dk;
 
-            lastKey = decoratedKey;
-
-            scanner.seekTo(decoratedKey);
-
-            if (!scanner.hasNext())
-                continue;
+            scanner.seekTo(dk);
+            
+            i++;
 
-            serializeRow(reader, (SSTableIdentityIterator) scanner.next(), decoratedKey,
outs);
+            if (scanner.hasNext())
+            {
+                SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next();
 
-            if (i != 0)
-                outs.println(",");
+                try
+                {
+                    serializeRow(outs, row);
+                }
+                catch (IOException ioexc)
+                {
+                    System.err.println("WARNING: Corrupt row " + key + " (skipping).");
+                    continue;
+                }
+                catch (OutOfMemoryError oom)
+                {
+                    System.err.println("ERROR: Out of memory deserializing row " + key);
+                    continue;
+                }
 
-            i++;
+                if (i != 1)
+                    outs.println(",");
+            }
         }
-
+        
         outs.println("\n}");
         outs.flush();
-
-        scanner.close();
     }
 
     // This is necessary to accommodate the test suite since you cannot open a Reader more
     // than once from within the same process.
     static void export(SSTableReader reader, PrintStream outs, String[] excludes) throws
IOException
     {
+        SSTableScanner scanner = reader.getDirectScanner(INPUT_FILE_BUFFER_SIZE);
         Set<String> excludeSet = new HashSet<String>();
 
         if (excludes != null)
             excludeSet = new HashSet<String>(Arrays.asList(excludes));
 
-
-        SSTableIdentityIterator row;
-        SSTableScanner scanner = reader.getDirectScanner(BufferedRandomAccessFile.DEFAULT_BUFFER_SIZE);
-
         outs.println("{");
 
-        int i = 0;
+        SSTableIdentityIterator row;
 
-        // collecting keys to export
+        boolean elementWritten = false;
         while (scanner.hasNext())
         {
             row = (SSTableIdentityIterator) scanner.next();
 
-            String currentKey = bytesToHex(row.getKey().key);
-
-            if (excludeSet.contains(currentKey))
+            if (excludeSet.contains(bytesToHex(row.getKey().key)))
                 continue;
-            else if (i != 0)
+            else if (elementWritten)
                 outs.println(",");
 
-            serializeRow(reader, row, row.getKey(), outs);
+            try
+            {
+                serializeRow(outs, row);
 
-            i++;
+                // used to decide should we put ',' after previous row or not
+                if (!elementWritten)
+                    elementWritten = true;
+            }
+            catch (IOException ioexcep)
+            {
+                System.err.println("WARNING: Corrupt row " + bytesToHex(row.getKey().key)
+ " (skipping).");
+                elementWritten = false;
+            }
+            catch (OutOfMemoryError oom)
+            {
+                System.err.println("ERROR: Out of memory deserializing row " + bytesToHex(row.getKey().key));
+                elementWritten = false;
+            }
         }
-
-        outs.println("\n}");
+        
+        outs.printf("%n}%n");
         outs.flush();
-
-        scanner.close();
     }
     
     /**
@@ -378,7 +326,8 @@ public class SSTableExport
      */
     public static void export(String ssTableFile, PrintStream outs, String[] excludes) throws
IOException
     {
-        export(SSTableReader.open(Descriptor.fromFilename(ssTableFile)), outs, excludes);
+        SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(ssTableFile));
+        export(reader, outs, excludes);
     }
 
     /**
@@ -399,7 +348,6 @@ public class SSTableExport
      * export the contents of the SSTable to JSON.
      *  
      * @param args command lines arguments
-     *
      * @throws IOException on failure to open/read/write files or output streams
      * @throws ConfigurationException if configuration is invalid
      */
@@ -447,7 +395,7 @@ public class SSTableExport
         else
         {
             if ((keys != null) && (keys.length > 0))
-                export(ssTableFileName, System.out, Arrays.asList(keys), excludes);
+                export(ssTableFileName, System.out, keys, excludes);
             else
                 export(ssTableFileName, excludes);
         }



Mime
View raw message