Return-Path: Delivered-To: apmail-incubator-cassandra-commits-archive@minotaur.apache.org Received: (qmail 85455 invoked from network); 3 Mar 2010 16:51:56 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 3 Mar 2010 16:51:56 -0000 Received: (qmail 59294 invoked by uid 500); 3 Mar 2010 16:51:49 -0000 Delivered-To: apmail-incubator-cassandra-commits-archive@incubator.apache.org Received: (qmail 59280 invoked by uid 500); 3 Mar 2010 16:51:49 -0000 Mailing-List: contact cassandra-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: cassandra-dev@incubator.apache.org Delivered-To: mailing list cassandra-commits@incubator.apache.org Received: (qmail 59272 invoked by uid 99); 3 Mar 2010 16:51:49 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 03 Mar 2010 16:51:49 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 03 Mar 2010 16:51:46 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 51A9A23889B3; Wed, 3 Mar 2010 16:51:25 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r918566 [2/2] - in /incubator/cassandra/trunk: ./ conf/ interface/thrift/gen-java/org/apache/cassandra/thrift/ src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/io/ src/java/org/apache/cas... Date: Wed, 03 Mar 2010 16:51:24 -0000 To: cassandra-commits@incubator.apache.org From: jbellis@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100303165125.51A9A23889B3@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java?rev=918566&r1=918565&r2=918566&view=diff ============================================================================== --- incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java (original) +++ incubator/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java Wed Mar 3 16:51:24 2010 @@ -21,8 +21,8 @@ import java.io.File; import java.io.IOException; import java.io.PrintStream; -import java.util.Collection; -import java.util.Iterator; +import java.util.*; + import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamily; import org.apache.cassandra.db.DecoratedKey; @@ -47,6 +47,7 @@ private static int INPUT_FILE_BUFFER_SIZE = 8 * 1024 * 1024; private static final String KEY_OPTION = "k"; + private static final String EXCLUDEKEY_OPTION = "x"; private static final String ENUMERATEKEYS_OPTION = "e"; private static Options options; private static CommandLine cmd; @@ -60,6 +61,11 @@ optKey.setArgs(500); options.addOption(optKey); + Option excludeKey = new Option(EXCLUDEKEY_OPTION, true, "Excluded row key"); + // Number of times -x can be passed on the command line. + excludeKey.setArgs(500); + options.addOption(excludeKey); + Option optEnumerate = new Option(ENUMERATEKEYS_OPTION, false, "enumerate keys only"); options.addOption(optEnumerate); } @@ -180,18 +186,21 @@ * @param keys the keys corresponding to the rows to export * @throws IOException on failure to read/write input/output */ - public static void export(String ssTableFile, PrintStream outs, String[] keys) + public static void export(String ssTableFile, PrintStream outs, String[] keys, String[] excludes) throws IOException { SSTableReader reader = SSTableReader.open(ssTableFile); SSTableScanner scanner = reader.getScanner(INPUT_FILE_BUFFER_SIZE); IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + Set excludeSet = new HashSet(Arrays.asList(excludes)); int i = 0; outs.println("{"); for (String key : keys) { + if (excludeSet.contains(key)) + continue; DecoratedKey dk = partitioner.decorateKey(key); scanner.seekTo(dk); @@ -232,23 +241,26 @@ * @param keys the keys corresponding to the rows to export * @throws IOException on failure to read/write input/output */ - public static void export(String ssTableFile, String outFile, String[] keys) throws IOException + public static void export(String ssTableFile, String outFile, String[] keys, String[] excludes) throws IOException { PrintStream outs = new PrintStream(outFile); - export(ssTableFile, outs, keys); + export(ssTableFile, outs, keys, excludes); } // This is necessary to accommodate the test suite since you cannot open a Reader more // than once from within the same process. - static void export(SSTableReader reader, PrintStream outs) throws IOException + static void export(SSTableReader reader, PrintStream outs, String[] excludes) throws IOException { SSTableScanner scanner = reader.getScanner(INPUT_FILE_BUFFER_SIZE); - + Set excludeSet = new HashSet(Arrays.asList(excludes)); + outs.println("{"); while(scanner.hasNext()) { IteratingRow row = scanner.next(); + if (excludeSet.contains(row.getKey().key)) + continue; try { String jsonOut = serializeRow(row); @@ -281,10 +293,10 @@ * @param outs PrintStream to write the output to * @throws IOException on failure to read/write input/output */ - public static void export(String ssTableFile, PrintStream outs) throws IOException + public static void export(String ssTableFile, PrintStream outs, String[] excludes) throws IOException { SSTableReader reader = SSTableReader.open(ssTableFile); - export(reader, outs); + export(reader, outs, excludes); } /** @@ -294,10 +306,10 @@ * @param outFile file to write output to * @throws IOException on failure to read/write SSTable/output file */ - public static void export(String ssTableFile, String outFile) throws IOException + public static void export(String ssTableFile, String outFile, String[] excludes) throws IOException { PrintStream outs = new PrintStream(outFile); - export(ssTableFile, outs); + export(ssTableFile, outs, excludes); } /** @@ -306,9 +318,9 @@ * @param ssTableFile SSTable to export * @throws IOException on failure to read/write SSTable/standard out */ - public static void export(String ssTableFile) throws IOException + public static void export(String ssTableFile, String[] excludes) throws IOException { - export(ssTableFile, System.out); + export(ssTableFile, System.out, excludes); } /** @@ -320,7 +332,7 @@ */ public static void main(String[] args) throws IOException { - String usage = String.format("Usage: %s [-k key [-k key [...]]]%n", SSTableExport.class.getName()); + String usage = String.format("Usage: %s [-k key [-k key [...]] -x key [-x key [...]]]%n", SSTableExport.class.getName()); CommandLineParser parser = new PosixParser(); try @@ -343,15 +355,16 @@ String[] keys = cmd.getOptionValues(KEY_OPTION); + String[] excludes = cmd.getOptionValues(EXCLUDEKEY_OPTION); String ssTableFileName = new File(cmd.getArgs()[0]).getAbsolutePath(); if (cmd.hasOption(ENUMERATEKEYS_OPTION)) enumeratekeys(ssTableFileName, System.out); else { if ((keys != null) && (keys.length > 0)) - export(ssTableFileName, System.out, keys); + export(ssTableFileName, System.out, keys, excludes); else - export(ssTableFileName); + export(ssTableFileName, excludes); } System.exit(0); } Modified: incubator/cassandra/trunk/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java?rev=918566&r1=918565&r2=918566&view=diff ============================================================================== --- incubator/cassandra/trunk/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java (original) +++ incubator/cassandra/trunk/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java Wed Mar 3 16:51:24 2010 @@ -52,7 +52,7 @@ @Test public void testKSMetaDataSerialization() throws IOException { - for (KSMetaData ksm : DatabaseDescriptor.tables_.values()) + for (KSMetaData ksm : DatabaseDescriptor.tables.values()) { byte[] ser = KSMetaData.serialize(ksm); KSMetaData ksmDupe = KSMetaData.deserialize(new ByteArrayInputStream(ser)); Modified: incubator/cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java?rev=918566&r1=918565&r2=918566&view=diff ============================================================================== --- incubator/cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java (original) +++ incubator/cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java Wed Mar 3 16:51:24 2010 @@ -103,12 +103,19 @@ writer.append(partitioner.decorateKey("rowB"), dob); dob.reset(); cfamily.clear(); - + + // Add rowExclude + cfamily.addColumn(new QueryPath("Standard1", null, "colX".getBytes()), "valX".getBytes(), 1, false); + ColumnFamily.serializer().serializeWithIndexes(cfamily, dob); + writer.append(partitioner.decorateKey("rowExclude"), dob); + dob.reset(); + cfamily.clear(); + SSTableReader reader = writer.closeAndOpenReader(); // Export to JSON and verify File tempJson = File.createTempFile("Standard1", ".json"); - SSTableExport.export(reader, new PrintStream(tempJson.getPath())); + SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{"rowExclude"}); JSONObject json = (JSONObject)JSONValue.parse(new FileReader(tempJson)); @@ -119,6 +126,9 @@ JSONArray rowB = (JSONArray)json.get("rowB"); JSONArray colB = (JSONArray)rowB.get(0); assert !(Boolean)colB.get(3); + + JSONArray rowExclude = (JSONArray)json.get("rowExclude"); + assert rowExclude == null; } @Test @@ -143,12 +153,19 @@ writer.append(partitioner.decorateKey("rowB"), dob); dob.reset(); cfamily.clear(); - + + // Add rowExclude + cfamily.addColumn(new QueryPath("Super4", "superX".getBytes(), "colX".getBytes()), "valX".getBytes(), 1, false); + ColumnFamily.serializer().serializeWithIndexes(cfamily, dob); + writer.append(partitioner.decorateKey("rowExclude"), dob); + dob.reset(); + cfamily.clear(); + SSTableReader reader = writer.closeAndOpenReader(); // Export to JSON and verify File tempJson = File.createTempFile("Super4", ".json"); - SSTableExport.export(reader, new PrintStream(tempJson.getPath())); + SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{"rowExclude"}); JSONObject json = (JSONObject)JSONValue.parse(new FileReader(tempJson)); @@ -156,9 +173,10 @@ JSONObject superA = (JSONObject)rowA.get(cfamily.getComparator().getString("superA".getBytes())); JSONArray subColumns = (JSONArray)superA.get("subColumns"); JSONArray colA = (JSONArray)subColumns.get(0); - + JSONObject rowExclude = (JSONObject)json.get("rowExclude"); assert Arrays.equals(hexToBytes((String)colA.get(1)), "valA".getBytes()); - assert !(Boolean)colA.get(3); + assert !(Boolean)colA.get(3); + assert rowExclude == null; } @Test @@ -176,12 +194,19 @@ writer.append(partitioner.decorateKey("rowA"), dob); dob.reset(); cfamily.clear(); - + + // Add rowExclude + cfamily.addColumn(new QueryPath("Standard1", null, "name".getBytes()), "val".getBytes(), 1, false); + ColumnFamily.serializer().serializeWithIndexes(cfamily, dob); + writer.append(partitioner.decorateKey("rowExclude"), dob); + dob.reset(); + cfamily.clear(); + SSTableReader reader = writer.closeAndOpenReader(); // Export to JSON and verify File tempJson = File.createTempFile("Standard1", ".json"); - SSTableExport.export(reader, new PrintStream(tempJson.getPath())); + SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{"rowExclude"}); // Import JSON to another SSTable file File tempSS2 = tempSSTableFile("Keyspace1", "Standard1"); @@ -192,6 +217,11 @@ ColumnFamily cf = qf.getSSTableColumnIterator(reader).getColumnFamily(); assertTrue(cf != null); assertTrue(Arrays.equals(cf.getColumn("name".getBytes()).value(), hexToBytes("76616c"))); + + qf = new NamesQueryFilter("rowExclude", new QueryPath("Standard1", null, null), "name".getBytes()); + cf = qf.getSSTableColumnIterator(reader).getColumnFamily(); + assert cf == null; + } }