Return-Path: Delivered-To: apmail-cassandra-commits-archive@www.apache.org Received: (qmail 9468 invoked from network); 14 Apr 2010 19:54:05 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 14 Apr 2010 19:54:05 -0000 Received: (qmail 64708 invoked by uid 500); 14 Apr 2010 19:54:05 -0000 Delivered-To: apmail-cassandra-commits-archive@cassandra.apache.org Received: (qmail 64694 invoked by uid 500); 14 Apr 2010 19:54:05 -0000 Mailing-List: contact commits-help@cassandra.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cassandra.apache.org Delivered-To: mailing list commits@cassandra.apache.org Received: (qmail 64686 invoked by uid 99); 14 Apr 2010 19:54:05 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 14 Apr 2010 19:54:05 +0000 X-ASF-Spam-Status: No, hits=-1889.0 required=10.0 tests=ALL_TRUSTED,AWL X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 14 Apr 2010 19:54:03 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 771D5238890B; Wed, 14 Apr 2010 19:53:43 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r934157 - in /cassandra/trunk: src/java/org/apache/cassandra/tools/ test/resources/ test/unit/org/apache/cassandra/tools/ Date: Wed, 14 Apr 2010 19:53:43 -0000 To: commits@cassandra.apache.org From: gdusbabek@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100414195343.771D5238890B@eris.apache.org> Author: gdusbabek Date: Wed Apr 14 19:53:43 2010 New Revision: 934157 URL: http://svn.apache.org/viewvc?rev=934157&view=rev Log: use hex keys in sstable import/export. Patch by Stu Hood, reviewed by Gary Dusbabek. CASSANDRA-767 Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java cassandra/trunk/test/resources/SimpleCF.json cassandra/trunk/test/resources/SuperCF.json cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java?rev=934157&r1=934156&r2=934157&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java Wed Apr 14 19:53:43 2010 @@ -38,7 +38,7 @@ import org.apache.cassandra.service.Stor import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.utils.FBUtilities.bytesToHex; -import static org.apache.cassandra.utils.FBUtilities.UTF8; +import static org.apache.cassandra.utils.FBUtilities.hexToBytes; import org.apache.commons.cli.*; /** @@ -82,13 +82,6 @@ public class SSTableExport return String.format("%s: ", quote(val)); } - @Deprecated - private static String asStr(byte[] val) - { - // FIXME: should not interpret as a string - return new String(val, FBUtilities.UTF8); - } - private static String serializeColumns(Collection cols, AbstractType comp) { StringBuilder json = new StringBuilder("["); @@ -119,7 +112,7 @@ public class SSTableExport { ColumnFamily cf = row.getColumnFamily(); AbstractType comparator = cf.getComparator(); - StringBuilder json = new StringBuilder(asKey(asStr(row.getKey().key))); + StringBuilder json = new StringBuilder(asKey(bytesToHex(row.getKey().key))); if (cf.isSuper()) { @@ -167,7 +160,7 @@ public class SSTableExport { DecoratedKey decoratedKey = partitioner.convertFromDiskFormat(FBUtilities.readShortByteArray(input)); long dataPosition = input.readLong(); - outs.println(asStr(decoratedKey.key)); + outs.println(bytesToHex(decoratedKey.key)); } outs.flush(); @@ -213,8 +206,7 @@ public class SSTableExport { if (excludeSet.contains(key)) continue; - // FIXME: assuming string keys - DecoratedKey dk = partitioner.decorateKey(key.getBytes(UTF8)); + DecoratedKey dk = partitioner.decorateKey(hexToBytes(key)); scanner.seekTo(dk); i++; @@ -275,7 +267,7 @@ public class SSTableExport while(scanner.hasNext()) { SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next(); - if (excludeSet.contains(asStr(row.getKey().key))) + if (excludeSet.contains(bytesToHex(row.getKey().key))) continue; try { @@ -288,12 +280,12 @@ public class SSTableExport } catch (IOException ioexcep) { - System.err.println("WARNING: Corrupt row " + asStr(row.getKey().key) + " (skipping)."); + System.err.println("WARNING: Corrupt row " + bytesToHex(row.getKey().key) + " (skipping)."); continue; } catch (OutOfMemoryError oom) { - System.err.println("ERROR: Out of memory deserializing row " + asStr(row.getKey().key)); + System.err.println("ERROR: Out of memory deserializing row " + bytesToHex(row.getKey().key)); continue; } } Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java?rev=934157&r1=934156&r2=934157&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java Wed Apr 14 19:53:43 2010 @@ -31,7 +31,6 @@ import org.apache.cassandra.io.util.Data import org.apache.cassandra.io.sstable.SSTableWriter; import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.utils.FBUtilities.hexToBytes; -import static org.apache.cassandra.utils.FBUtilities.UTF8; import org.apache.commons.cli.*; import org.json.simple.JSONArray; import org.json.simple.JSONObject; @@ -77,13 +76,6 @@ public class SSTableImport } } - @Deprecated - private static String asStr(byte[] val) - { - // FIXME: should not interpret as a string - return new String(val, FBUtilities.UTF8); - } - /** * Add columns to a column family. * @@ -151,22 +143,21 @@ public class SSTableImport JSONObject json = (JSONObject)JSONValue.parseWithException(new FileReader(jsonFile)); SSTableWriter writer = new SSTableWriter(ssTablePath, json.size(), partitioner); - List> decoratedKeys = new ArrayList>(); + SortedMap decoratedKeys = new TreeMap(); + // sort by dk representation, but hold onto the hex version for (String key : (Set)json.keySet()) - // FIXME: assuming string keys - decoratedKeys.add(partitioner.decorateKey(key.getBytes(UTF8))); - Collections.sort(decoratedKeys); + decoratedKeys.put(partitioner.decorateKey(hexToBytes(key)), key); - for (DecoratedKey rowKey : decoratedKeys) + for (Map.Entry rowKey : decoratedKeys.entrySet()) { if (cfType.equals("Super")) - addToSuperCF((JSONObject)json.get(asStr(rowKey.key)), cfamily); + addToSuperCF((JSONObject)json.get(rowKey.getValue()), cfamily); else - addToStandardCF((JSONArray)json.get(asStr(rowKey.key)), cfamily); + addToStandardCF((JSONArray)json.get(rowKey.getValue()), cfamily); ColumnFamily.serializer().serializeWithIndexes(cfamily, dob); - writer.append(rowKey, dob); + writer.append(rowKey.getKey(), dob); dob.reset(); cfamily.clear(); } Modified: cassandra/trunk/test/resources/SimpleCF.json URL: http://svn.apache.org/viewvc/cassandra/trunk/test/resources/SimpleCF.json?rev=934157&r1=934156&r2=934157&view=diff ============================================================================== --- cassandra/trunk/test/resources/SimpleCF.json (original) +++ cassandra/trunk/test/resources/SimpleCF.json Wed Apr 14 19:53:43 2010 @@ -1,4 +1,4 @@ { - "rowA": [["636f6c4141", "76616c4141", 1, false], ["636f6c4142", "76616c4142", 1, false]], - "rowB": [["636f6c4241", "76616c4241", 1, false], ["636f6c4242", "76616c4242", 1, false]] + "726f7741": [["636f6c4141", "76616c4141", 1, false], ["636f6c4142", "76616c4142", 1, false]], + "726f7742": [["636f6c4241", "76616c4241", 1, false], ["636f6c4242", "76616c4242", 1, false]] } Modified: cassandra/trunk/test/resources/SuperCF.json URL: http://svn.apache.org/viewvc/cassandra/trunk/test/resources/SuperCF.json?rev=934157&r1=934156&r2=934157&view=diff ============================================================================== --- cassandra/trunk/test/resources/SuperCF.json (original) +++ cassandra/trunk/test/resources/SuperCF.json Wed Apr 14 19:53:43 2010 @@ -1,4 +1,4 @@ { - "rowA": {"737570657241": {"deletedAt": -9223372036854775808, "subColumns": [["636f6c4141", "76616c75654141", 1, false], ["636f6c4142", "76616c75654142", 1, false]]}}, - "rowB": {"737570657242": {"deletedAt": -9223372036854775808, "subColumns": [["636f6c4241", "76616c75654241", 1, false], ["636f6c4242", "76616c75654242", 1, false]]}} + "726f7741": {"737570657241": {"deletedAt": -9223372036854775808, "subColumns": [["636f6c4141", "76616c75654141", 1, false], ["636f6c4142", "76616c75654142", 1, false]]}}, + "726f7742": {"737570657242": {"deletedAt": -9223372036854775808, "subColumns": [["636f6c4241", "76616c75654241", 1, false], ["636f6c4242", "76616c75654242", 1, false]]}} } Modified: cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java?rev=934157&r1=934156&r2=934157&view=diff ============================================================================== --- cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java (original) +++ cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java Wed Apr 14 19:53:43 2010 @@ -35,6 +35,7 @@ import org.apache.cassandra.io.sstable.S import org.apache.cassandra.io.sstable.SSTableWriter; import org.apache.cassandra.io.util.DataOutputBuffer; import static org.apache.cassandra.io.sstable.SSTableUtils.tempSSTableFile; +import static org.apache.cassandra.utils.FBUtilities.bytesToHex; import static org.apache.cassandra.utils.FBUtilities.hexToBytes; import static org.junit.Assert.assertTrue; @@ -48,6 +49,11 @@ import org.junit.Test; public class SSTableExportTest extends SchemaLoader { + public String asHex(String str) + { + return bytesToHex(str.getBytes()); + } + @Test public void testEnumeratekeys() throws IOException { @@ -84,8 +90,7 @@ public class SSTableExportTest extends S String output = new String(buf); String sep = System.getProperty("line.separator"); - // FIXME: string keys - assert output.equals("rowA" + sep + "rowB" + sep) : output; + assert output.equals(asHex("rowA") + sep + asHex("rowB") + sep) : output; } @Test @@ -121,20 +126,19 @@ public class SSTableExportTest extends S // Export to JSON and verify File tempJson = File.createTempFile("Standard1", ".json"); - SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{"rowExclude"}); + SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")}); JSONObject json = (JSONObject)JSONValue.parse(new FileReader(tempJson)); - // FIXME: string keys - JSONArray rowA = (JSONArray)json.get("rowA"); + JSONArray rowA = (JSONArray)json.get(asHex("rowA")); JSONArray colA = (JSONArray)rowA.get(0); assert Arrays.equals(hexToBytes((String)colA.get(1)), "valA".getBytes()); - JSONArray rowB = (JSONArray)json.get("rowB"); + JSONArray rowB = (JSONArray)json.get(asHex("rowB")); JSONArray colB = (JSONArray)rowB.get(0); assert !(Boolean)colB.get(3); - JSONArray rowExclude = (JSONArray)json.get("rowExclude"); + JSONArray rowExclude = (JSONArray)json.get(asHex("rowExclude")); assert rowExclude == null; } @@ -172,15 +176,15 @@ public class SSTableExportTest extends S // Export to JSON and verify File tempJson = File.createTempFile("Super4", ".json"); - SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{"rowExclude"}); + SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")}); JSONObject json = (JSONObject)JSONValue.parse(new FileReader(tempJson)); - JSONObject rowA = (JSONObject)json.get("rowA"); + JSONObject rowA = (JSONObject)json.get(asHex("rowA")); JSONObject superA = (JSONObject)rowA.get(cfamily.getComparator().getString("superA".getBytes())); JSONArray subColumns = (JSONArray)superA.get("subColumns"); JSONArray colA = (JSONArray)subColumns.get(0); - JSONObject rowExclude = (JSONObject)json.get("rowExclude"); + JSONObject rowExclude = (JSONObject)json.get(asHex("rowExclude")); assert Arrays.equals(hexToBytes((String)colA.get(1)), "valA".getBytes()); assert !(Boolean)colA.get(3); assert rowExclude == null; @@ -213,7 +217,7 @@ public class SSTableExportTest extends S // Export to JSON and verify File tempJson = File.createTempFile("Standard1", ".json"); - SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{"rowExclude"}); + SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")}); // Import JSON to another SSTable file File tempSS2 = tempSSTableFile("Keyspace1", "Standard1");