cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gdusba...@apache.org
Subject svn commit: r934157 - in /cassandra/trunk: src/java/org/apache/cassandra/tools/ test/resources/ test/unit/org/apache/cassandra/tools/
Date Wed, 14 Apr 2010 19:53:43 GMT
Author: gdusbabek
Date: Wed Apr 14 19:53:43 2010
New Revision: 934157

URL: http://svn.apache.org/viewvc?rev=934157&view=rev
Log:
use hex keys in sstable import/export. Patch by Stu Hood, reviewed by Gary Dusbabek. CASSANDRA-767

Modified:
    cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
    cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
    cassandra/trunk/test/resources/SimpleCF.json
    cassandra/trunk/test/resources/SuperCF.json
    cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java

Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java?rev=934157&r1=934156&r2=934157&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java Wed Apr 14 19:53:43
2010
@@ -38,7 +38,7 @@ import org.apache.cassandra.service.Stor
 import org.apache.cassandra.utils.FBUtilities;
 
 import static org.apache.cassandra.utils.FBUtilities.bytesToHex;
-import static org.apache.cassandra.utils.FBUtilities.UTF8;
+import static org.apache.cassandra.utils.FBUtilities.hexToBytes;
 import org.apache.commons.cli.*;
 
 /**
@@ -82,13 +82,6 @@ public class SSTableExport
         return String.format("%s: ", quote(val));
     }
     
-    @Deprecated
-    private static String asStr(byte[] val)
-    {
-        // FIXME: should not interpret as a string
-        return new String(val, FBUtilities.UTF8);
-    }
-    
     private static String serializeColumns(Collection<IColumn> cols, AbstractType comp)
     {
         StringBuilder json = new StringBuilder("[");
@@ -119,7 +112,7 @@ public class SSTableExport
     {
         ColumnFamily cf = row.getColumnFamily();
         AbstractType comparator = cf.getComparator();
-        StringBuilder json = new StringBuilder(asKey(asStr(row.getKey().key)));
+        StringBuilder json = new StringBuilder(asKey(bytesToHex(row.getKey().key)));
         
         if (cf.isSuper())
         {
@@ -167,7 +160,7 @@ public class SSTableExport
         {
             DecoratedKey decoratedKey = partitioner.convertFromDiskFormat(FBUtilities.readShortByteArray(input));
             long dataPosition = input.readLong();
-            outs.println(asStr(decoratedKey.key));
+            outs.println(bytesToHex(decoratedKey.key));
         }
 
         outs.flush();
@@ -213,8 +206,7 @@ public class SSTableExport
         {
             if (excludeSet.contains(key))
                 continue;
-            // FIXME: assuming string keys
-            DecoratedKey<?> dk = partitioner.decorateKey(key.getBytes(UTF8));
+            DecoratedKey<?> dk = partitioner.decorateKey(hexToBytes(key));
             scanner.seekTo(dk);
             
             i++;
@@ -275,7 +267,7 @@ public class SSTableExport
         while(scanner.hasNext())
         {
             SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next();
-            if (excludeSet.contains(asStr(row.getKey().key)))
+            if (excludeSet.contains(bytesToHex(row.getKey().key)))
                 continue;
             try
             {
@@ -288,12 +280,12 @@ public class SSTableExport
             }
             catch (IOException ioexcep)
             {
-                System.err.println("WARNING: Corrupt row " + asStr(row.getKey().key) + "
(skipping).");
+                System.err.println("WARNING: Corrupt row " + bytesToHex(row.getKey().key)
+ " (skipping).");
                 continue;
             }
             catch (OutOfMemoryError oom)
             {
-                System.err.println("ERROR: Out of memory deserializing row " + asStr(row.getKey().key));
+                System.err.println("ERROR: Out of memory deserializing row " + bytesToHex(row.getKey().key));
                 continue;
             }
         }

Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java?rev=934157&r1=934156&r2=934157&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java Wed Apr 14 19:53:43
2010
@@ -31,7 +31,6 @@ import org.apache.cassandra.io.util.Data
 import org.apache.cassandra.io.sstable.SSTableWriter;
 import org.apache.cassandra.utils.FBUtilities;
 import static org.apache.cassandra.utils.FBUtilities.hexToBytes;
-import static org.apache.cassandra.utils.FBUtilities.UTF8;
 import org.apache.commons.cli.*;
 import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
@@ -77,13 +76,6 @@ public class SSTableImport
         }
     }
 
-    @Deprecated
-    private static String asStr(byte[] val)
-    {
-        // FIXME: should not interpret as a string
-        return new String(val, FBUtilities.UTF8);
-    }
-    
     /**
      * Add columns to a column family.
      * 
@@ -151,22 +143,21 @@ public class SSTableImport
             JSONObject json = (JSONObject)JSONValue.parseWithException(new FileReader(jsonFile));
             
             SSTableWriter writer = new SSTableWriter(ssTablePath, json.size(), partitioner);
-            List<DecoratedKey<?>> decoratedKeys = new ArrayList<DecoratedKey<?>>();
+            SortedMap<DecoratedKey,String> decoratedKeys = new TreeMap<DecoratedKey,String>();
             
+            // sort by dk representation, but hold onto the hex version
             for (String key : (Set<String>)json.keySet())
-                // FIXME: assuming string keys
-                decoratedKeys.add(partitioner.decorateKey(key.getBytes(UTF8)));
-            Collections.sort(decoratedKeys);
+                decoratedKeys.put(partitioner.decorateKey(hexToBytes(key)), key);
 
-            for (DecoratedKey<?> rowKey : decoratedKeys)
+            for (Map.Entry<DecoratedKey, String> rowKey : decoratedKeys.entrySet())
             {
                 if (cfType.equals("Super"))
-                    addToSuperCF((JSONObject)json.get(asStr(rowKey.key)), cfamily);
+                    addToSuperCF((JSONObject)json.get(rowKey.getValue()), cfamily);
                 else
-                    addToStandardCF((JSONArray)json.get(asStr(rowKey.key)), cfamily);
+                    addToStandardCF((JSONArray)json.get(rowKey.getValue()), cfamily);
                            
                 ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
-                writer.append(rowKey, dob);
+                writer.append(rowKey.getKey(), dob);
                 dob.reset();
                 cfamily.clear();
             }

Modified: cassandra/trunk/test/resources/SimpleCF.json
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/resources/SimpleCF.json?rev=934157&r1=934156&r2=934157&view=diff
==============================================================================
--- cassandra/trunk/test/resources/SimpleCF.json (original)
+++ cassandra/trunk/test/resources/SimpleCF.json Wed Apr 14 19:53:43 2010
@@ -1,4 +1,4 @@
 {
- "rowA": [["636f6c4141", "76616c4141", 1, false], ["636f6c4142", "76616c4142", 1, false]],
- "rowB": [["636f6c4241", "76616c4241", 1, false], ["636f6c4242", "76616c4242", 1, false]]
+ "726f7741": [["636f6c4141", "76616c4141", 1, false], ["636f6c4142", "76616c4142", 1, false]],
+ "726f7742": [["636f6c4241", "76616c4241", 1, false], ["636f6c4242", "76616c4242", 1, false]]
 }

Modified: cassandra/trunk/test/resources/SuperCF.json
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/resources/SuperCF.json?rev=934157&r1=934156&r2=934157&view=diff
==============================================================================
--- cassandra/trunk/test/resources/SuperCF.json (original)
+++ cassandra/trunk/test/resources/SuperCF.json Wed Apr 14 19:53:43 2010
@@ -1,4 +1,4 @@
 {
-  "rowA": {"737570657241": {"deletedAt": -9223372036854775808, "subColumns": [["636f6c4141",
"76616c75654141", 1, false], ["636f6c4142", "76616c75654142", 1, false]]}},
-  "rowB": {"737570657242": {"deletedAt": -9223372036854775808, "subColumns": [["636f6c4241",
"76616c75654241", 1, false], ["636f6c4242", "76616c75654242", 1, false]]}}
+  "726f7741": {"737570657241": {"deletedAt": -9223372036854775808, "subColumns": [["636f6c4141",
"76616c75654141", 1, false], ["636f6c4142", "76616c75654142", 1, false]]}},
+  "726f7742": {"737570657242": {"deletedAt": -9223372036854775808, "subColumns": [["636f6c4241",
"76616c75654241", 1, false], ["636f6c4242", "76616c75654242", 1, false]]}}
 }

Modified: cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java?rev=934157&r1=934156&r2=934157&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java Wed Apr 14
19:53:43 2010
@@ -35,6 +35,7 @@ import org.apache.cassandra.io.sstable.S
 import org.apache.cassandra.io.sstable.SSTableWriter;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import static org.apache.cassandra.io.sstable.SSTableUtils.tempSSTableFile;
+import static org.apache.cassandra.utils.FBUtilities.bytesToHex;
 import static org.apache.cassandra.utils.FBUtilities.hexToBytes;
 import static org.junit.Assert.assertTrue;
 
@@ -48,6 +49,11 @@ import org.junit.Test;
 
 public class SSTableExportTest extends SchemaLoader
 {
+    public String asHex(String str)
+    {
+        return bytesToHex(str.getBytes());
+    }
+
     @Test
     public void testEnumeratekeys() throws IOException
     {
@@ -84,8 +90,7 @@ public class SSTableExportTest extends S
         String output = new String(buf);
 
         String sep = System.getProperty("line.separator");
-        // FIXME: string keys
-        assert output.equals("rowA" + sep + "rowB" + sep) : output;
+        assert output.equals(asHex("rowA") + sep + asHex("rowB") + sep) : output;
     }
 
     @Test
@@ -121,20 +126,19 @@ public class SSTableExportTest extends S
         
         // Export to JSON and verify
         File tempJson = File.createTempFile("Standard1", ".json");
-        SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{"rowExclude"});
+        SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")});
         
         JSONObject json = (JSONObject)JSONValue.parse(new FileReader(tempJson));
         
-        // FIXME: string keys
-        JSONArray rowA = (JSONArray)json.get("rowA");
+        JSONArray rowA = (JSONArray)json.get(asHex("rowA"));
         JSONArray colA = (JSONArray)rowA.get(0);
         assert Arrays.equals(hexToBytes((String)colA.get(1)), "valA".getBytes());
         
-        JSONArray rowB = (JSONArray)json.get("rowB");
+        JSONArray rowB = (JSONArray)json.get(asHex("rowB"));
         JSONArray colB = (JSONArray)rowB.get(0);
         assert !(Boolean)colB.get(3);
 
-        JSONArray rowExclude = (JSONArray)json.get("rowExclude");
+        JSONArray rowExclude = (JSONArray)json.get(asHex("rowExclude"));
         assert rowExclude == null;
     }
 
@@ -172,15 +176,15 @@ public class SSTableExportTest extends S
         
         // Export to JSON and verify
         File tempJson = File.createTempFile("Super4", ".json");
-        SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{"rowExclude"});
+        SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")});
         
         JSONObject json = (JSONObject)JSONValue.parse(new FileReader(tempJson));
         
-        JSONObject rowA = (JSONObject)json.get("rowA");
+        JSONObject rowA = (JSONObject)json.get(asHex("rowA"));
         JSONObject superA = (JSONObject)rowA.get(cfamily.getComparator().getString("superA".getBytes()));
         JSONArray subColumns = (JSONArray)superA.get("subColumns");
         JSONArray colA = (JSONArray)subColumns.get(0);
-        JSONObject rowExclude = (JSONObject)json.get("rowExclude");
+        JSONObject rowExclude = (JSONObject)json.get(asHex("rowExclude"));
         assert Arrays.equals(hexToBytes((String)colA.get(1)), "valA".getBytes());
         assert !(Boolean)colA.get(3);
         assert rowExclude == null;
@@ -213,7 +217,7 @@ public class SSTableExportTest extends S
         
         // Export to JSON and verify
         File tempJson = File.createTempFile("Standard1", ".json");
-        SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{"rowExclude"});
+        SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")});
         
         // Import JSON to another SSTable file
         File tempSS2 = tempSSTableFile("Keyspace1", "Standard1");



Mime
View raw message