phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jamestay...@apache.org
Subject [1/2] PHOENIX-129 Improve MapReduce-based import (GabrielReid)
Date Sun, 16 Mar 2014 18:22:04 GMT
Repository: incubator-phoenix
Updated Branches:
  refs/heads/3.0 6451b1df4 -> ed9f58bae


http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/ed9f58ba/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolTest.java
new file mode 100644
index 0000000..31fc71c
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvBulkLoadToolTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce;
+
+import org.apache.commons.cli.CommandLine;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class CsvBulkLoadToolTest {
+
+    private CsvBulkLoadTool bulkLoadTool;
+
+    @Before
+    public void setUp() {
+        bulkLoadTool = new CsvBulkLoadTool();
+    }
+
+    @Test
+    public void testParseOptions() {
+        CommandLine cmdLine = bulkLoadTool.parseOptions(new String[] { "--input", "/input",
+                "--table", "mytable" });
+
+        assertEquals("mytable", cmdLine.getOptionValue(CsvBulkLoadTool.TABLE_NAME_OPT.getOpt()));
+        assertEquals("/input", cmdLine.getOptionValue(CsvBulkLoadTool.INPUT_PATH_OPT.getOpt()));
+    }
+
+    @Test(expected=IllegalStateException.class)
+    public void testParseOptions_ExtraArguments() {
+        bulkLoadTool.parseOptions(new String[] { "--input", "/input",
+                "--table", "mytable", "these", "shouldnt", "be", "here" });
+    }
+
+    @Test(expected=IllegalStateException.class)
+    public void testParseOptions_NoInput() {
+        bulkLoadTool.parseOptions(new String[] { "--table", "mytable" });
+    }
+
+    @Test(expected=IllegalStateException.class)
+    public void testParseOptions_NoTable() {
+        bulkLoadTool.parseOptions(new String[] { "--input", "/input" });
+    }
+
+    @Test
+    public void testGetQualifiedTableName() {
+        assertEquals("MYSCHEMA.MYTABLE", CsvBulkLoadTool.getQualifiedTableName("mySchema",
"myTable"));
+    }
+
+    @Test
+    public void testGetQualifiedTableName_NullSchema() {
+        assertEquals("MYTABLE", CsvBulkLoadTool.getQualifiedTableName(null, "myTable"));
+    }
+
+    @Test
+    public void testGetJdbcUrl_WithQuorumSupplied() {
+        assertEquals("jdbc:phoenix:myzkhost:2181", bulkLoadTool.getJdbcUrl("myzkhost:2181"));
+    }
+
+    @Test
+    public void testGetJdbcUrl_NoQuorumSupplied() {
+        assertEquals("jdbc:phoenix:localhost:2181", bulkLoadTool.getJdbcUrl(null));
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/ed9f58ba/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapperTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapperTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapperTest.java
new file mode 100644
index 0000000..96157a6
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapperTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.mapreduce;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.apache.commons.csv.CSVRecord;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.phoenix.schema.PDataType;
+import org.apache.phoenix.util.ColumnInfo;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class CsvToKeyValueMapperTest {
+
+    @Test
+    public void testCsvLineParser() throws IOException {
+        CsvToKeyValueMapper.CsvLineParser lineParser = new CsvToKeyValueMapper.CsvLineParser(';');
+        CSVRecord parsed = lineParser.parse("one;two");
+
+        assertEquals("one", parsed.get(0));
+        assertEquals("two", parsed.get(1));
+        assertTrue(parsed.isConsistent());
+        assertEquals(1, parsed.getRecordNumber());
+    }
+
+
+    @Test
+    public void testBuildColumnInfoList() {
+        List<ColumnInfo> columnInfoList = ImmutableList.of(
+                new ColumnInfo("idCol", PDataType.INTEGER.getSqlType()),
+                new ColumnInfo("unsignedIntCol", PDataType.UNSIGNED_INT.getSqlType()),
+                new ColumnInfo("stringArrayCol", PDataType.INTEGER_ARRAY.getSqlType()));
+
+        Configuration conf = new Configuration();
+        CsvToKeyValueMapper.configureColumnInfoList(conf, columnInfoList);
+        List<ColumnInfo> fromConfig = CsvToKeyValueMapper.buildColumnInfoList(conf);
+
+        assertEquals(columnInfoList, fromConfig);
+    }
+
+    @Test
+    public void testBuildColumnInfoList_ContainingNulls() {
+        // A null value in the column info list means "skip that column in the input"
+        List<ColumnInfo> columnInfoListWithNull = Lists.newArrayList(
+                new ColumnInfo("idCol", PDataType.INTEGER.getSqlType()),
+                null,
+                new ColumnInfo("unsignedIntCol", PDataType.UNSIGNED_INT.getSqlType()),
+                new ColumnInfo("stringArrayCol", PDataType.INTEGER_ARRAY.getSqlType()));
+
+        Configuration conf = new Configuration();
+        CsvToKeyValueMapper.configureColumnInfoList(conf, columnInfoListWithNull);
+        List<ColumnInfo> fromConfig = CsvToKeyValueMapper.buildColumnInfoList(conf);
+
+        assertEquals(columnInfoListWithNull, fromConfig);
+    }
+
+    @Test
+    public void testGetJdbcUrl() {
+        Configuration conf = new Configuration();
+        conf.set(HConstants.ZOOKEEPER_QUORUM, "myzkclient:2181");
+        String jdbcUrl = CsvToKeyValueMapper.getJdbcUrl(conf);
+
+        assertEquals("jdbc:phoenix:myzkclient:2181", jdbcUrl);
+    }
+
+    @Test(expected=IllegalStateException.class)
+    public void testGetJdbcUrl_NotConfigured() {
+        Configuration conf = new Configuration();
+        CsvToKeyValueMapper.getJdbcUrl(conf);
+    }
+
+    @Test
+    public void testLoadPreUpdateProcessor() {
+        Configuration conf = new Configuration();
+        conf.setClass(CsvToKeyValueMapper.UPSERT_HOOK_CLASS_CONFKEY, MockUpsertProcessor.class,
+                ImportPreUpsertKeyValueProcessor.class);
+
+        ImportPreUpsertKeyValueProcessor processor = CsvToKeyValueMapper.loadPreUpsertProcessor(conf);
+        assertEquals(MockUpsertProcessor.class, processor.getClass());
+    }
+
+    @Test
+    public void testLoadPreUpdateProcessor_NotConfigured() {
+
+        Configuration conf = new Configuration();
+        ImportPreUpsertKeyValueProcessor processor = CsvToKeyValueMapper.loadPreUpsertProcessor(conf);
+
+        assertEquals(CsvToKeyValueMapper.DefaultImportPreUpsertKeyValueProcessor.class,
+                processor.getClass());
+    }
+
+    @Test(expected=IllegalStateException.class)
+    public void testLoadPreUpdateProcessor_ClassNotFound() {
+        Configuration conf = new Configuration();
+        conf.set(CsvToKeyValueMapper.UPSERT_HOOK_CLASS_CONFKEY, "MyUndefinedClass");
+
+        CsvToKeyValueMapper.loadPreUpsertProcessor(conf);
+    }
+
+
+    static class MockUpsertProcessor implements ImportPreUpsertKeyValueProcessor {
+
+        @Override
+        public List<KeyValue> preUpsert(byte[] rowKey, List<KeyValue> keyValues)
{
+            throw new UnsupportedOperationException("Not yet implemented");
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/ed9f58ba/phoenix-core/src/test/java/org/apache/phoenix/util/ColumnInfoTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/ColumnInfoTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/ColumnInfoTest.java
new file mode 100644
index 0000000..85f3b44
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/ColumnInfoTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import org.apache.phoenix.schema.IllegalDataException;
+import org.junit.Test;
+
+import java.sql.Types;
+
+import static org.junit.Assert.assertEquals;
+
+public class ColumnInfoTest {
+
+    @Test
+    public void testToFromStringRoundTrip() {
+        ColumnInfo columnInfo = new ColumnInfo("myColumn", Types.INTEGER);
+        assertEquals(columnInfo, ColumnInfo.fromString(columnInfo.toString()));
+    }
+
+    @Test(expected=IllegalArgumentException.class)
+    public void testFromString_InvalidString() {
+        ColumnInfo.fromString("invalid");
+    }
+
+    @Test(expected= IllegalDataException.class)
+    public void testFromString_InvalidDataType() {
+        ColumnInfo.fromString("COLNAME:badType");
+    }
+}


Mime
View raw message