Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@hbase.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: appy@apache.org
To: commits@hbase.apache.org
Date: Tue, 11 Oct 2016 23:38:17 -0000
Message-Id: <3fc2a372b1b04d40a1ab0ecc52589ba8@git.apache.org>
Subject: [1/2] hbase git commit: HBASE-16811 Remove mob sweep job.
archived-at: Tue, 11 Oct 2016 23:38:19 -0000

Repository: hbase
Updated Branches:
  refs/heads/master ec87b4bfe -> eb52e2682


http://git-wip-us.apache.org/repos/asf/hbase/blob/eb52e268/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/mapreduce/TestMobSweepReducer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/mapreduce/TestMobSweepReducer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/mapreduce/TestMobSweepReducer.java
deleted file mode 100644
index abe0d3a..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/mapreduce/TestMobSweepReducer.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.mob.mapreduce;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
-import org.apache.hadoop.hbase.master.TableLockManager;
-import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.mob.mapreduce.SweepJob.DummyMobAbortable;
-import org.apache.hadoop.hbase.mob.mapreduce.SweepJob.SweepCounter;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.serializer.JavaSerialization;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.counters.GenericCounter;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.mockito.Matchers;
-
-@Category(MediumTests.class)
-public class TestMobSweepReducer {
-
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private final static String tableName = "testSweepReducer";
-  private final static String row = "row";
-  private final static String family = "family";
-  private final static String qf = "qf";
-  private static BufferedMutator table;
-  private static Admin admin;
-
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
-
-    TEST_UTIL.startMiniCluster(1);
-  }
-
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-    TEST_UTIL.shutdownMiniCluster();
-  }
-
-  @SuppressWarnings("deprecation")
-  @Before
-  public void setUp() throws Exception {
-    HTableDescriptor desc = new HTableDescriptor(tableName);
-    HColumnDescriptor hcd = new HColumnDescriptor(family);
-    hcd.setMobEnabled(true);
-    hcd.setMobThreshold(3L);
-    hcd.setMaxVersions(4);
-    desc.addFamily(hcd);
-
-    admin = TEST_UTIL.getHBaseAdmin();
-    admin.createTable(desc);
-    table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())
-            .getBufferedMutator(TableName.valueOf(tableName));
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    admin.disableTable(TableName.valueOf(tableName));
-    admin.deleteTable(TableName.valueOf(tableName));
-    admin.close();
-  }
-
-  private List<String> getKeyFromSequenceFile(FileSystem fs, Path path,
-                                              Configuration conf) throws Exception {
-    List<String> list = new ArrayList<String>();
-    SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(path));
-
-    String next = (String) reader.next((String) null);
-    while (next != null) {
-      list.add(next);
-      next = (String) reader.next((String) null);
-    }
-    reader.close();
-    return list;
-  }
-
-  @Test
-  public void testRun() throws Exception {
-
-    TableName tn = TableName.valueOf(tableName);
-    byte[] mobValueBytes = new byte[100];
-
-    //get the path where mob files lie in
-    Path mobFamilyPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, family);
-
-    Put put = new Put(Bytes.toBytes(row));
-    put.addColumn(Bytes.toBytes(family), Bytes.toBytes(qf), 1, mobValueBytes);
-    Put put2 = new Put(Bytes.toBytes(row + "ignore"));
-    put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(qf), 1, mobValueBytes);
-    table.mutate(put);
-    table.mutate(put2);
-    table.flush();
-    admin.flush(tn);
-
-    FileStatus[] fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath);
-    //check the generation of a mob file
-    assertEquals(1, fileStatuses.length);
-
-    String mobFile1 = fileStatuses[0].getPath().getName();
-
-    Configuration configuration = new Configuration(TEST_UTIL.getConfiguration());
-    configuration.setFloat(MobConstants.MOB_SWEEP_TOOL_COMPACTION_RATIO, 0.6f);
-    configuration.setStrings(TableInputFormat.INPUT_TABLE, tableName);
-    configuration.setStrings(TableInputFormat.SCAN_COLUMN_FAMILY, family);
-    configuration.setStrings(SweepJob.WORKING_VISITED_DIR_KEY, "jobWorkingNamesDir");
-    configuration.setStrings(SweepJob.WORKING_FILES_DIR_KEY, "compactionFileDir");
-    configuration.setStrings(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY,
-            JavaSerialization.class.getName());
-    configuration.set(SweepJob.WORKING_VISITED_DIR_KEY, "compactionVisitedDir");
-    configuration.setLong(MobConstants.MOB_SWEEP_TOOL_COMPACTION_START_DATE,
-        System.currentTimeMillis() + 24 * 3600 * 1000);
-
-    ZooKeeperWatcher zkw = new ZooKeeperWatcher(configuration, "1", new DummyMobAbortable());
-    TableName lockName = MobUtils.getTableLockName(tn);
-    String znode = ZKUtil.joinZNode(zkw.znodePaths.tableLockZNode, lockName.getNameAsString());
-    configuration.set(SweepJob.SWEEP_JOB_ID, "1");
-    configuration.set(SweepJob.SWEEP_JOB_TABLE_NODE, znode);
-    ServerName serverName = SweepJob.getCurrentServerName(configuration);
-    configuration.set(SweepJob.SWEEP_JOB_SERVERNAME, serverName.toString());
-
-    TableLockManager tableLockManager = TableLockManager.createTableLockManager(configuration, zkw,
-        serverName);
-    TableLock lock = tableLockManager.writeLock(lockName, "Run sweep tool");
-    lock.acquire();
-    try {
-      // use the same counter when mocking
-      Counter counter = new GenericCounter();
-      Reducer<Text, KeyValue, Writable, Writable>.Context ctx = mock(Reducer.Context.class);
-      when(ctx.getConfiguration()).thenReturn(configuration);
-      when(ctx.getCounter(Matchers.any(SweepCounter.class))).thenReturn(counter);
-      when(ctx.nextKey()).thenReturn(true).thenReturn(false);
-      when(ctx.getCurrentKey()).thenReturn(new Text(mobFile1));
-
-      byte[] refBytes = Bytes.toBytes(mobFile1);
-      long valueLength = refBytes.length;
-      byte[] newValue = Bytes.add(Bytes.toBytes(valueLength), refBytes);
-      KeyValue kv2 = new KeyValue(Bytes.toBytes(row), Bytes.toBytes(family), Bytes.toBytes(qf), 1,
-        KeyValue.Type.Put, newValue);
-      List<KeyValue> list = new ArrayList<KeyValue>();
-      list.add(kv2);
-
-      when(ctx.getValues()).thenReturn(list);
-
-      SweepReducer reducer = new SweepReducer();
-      reducer.run(ctx);
-    } finally {
-      lock.release();
-    }
-    FileStatus[] filsStatuses2 = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath);
-    String mobFile2 = filsStatuses2[0].getPath().getName();
-    //new mob file is generated, old one has been archived
-    assertEquals(1, filsStatuses2.length);
-    assertEquals(false, mobFile2.equalsIgnoreCase(mobFile1));
-
-    //test sequence file
-    String workingPath = configuration.get(SweepJob.WORKING_VISITED_DIR_KEY);
-    FileStatus[] statuses = TEST_UTIL.getTestFileSystem().listStatus(new Path(workingPath));
-    Set<String> files = new TreeSet<String>();
-    for (FileStatus st : statuses) {
-      files.addAll(getKeyFromSequenceFile(TEST_UTIL.getTestFileSystem(),
-              st.getPath(), configuration));
-    }
-    assertEquals(1, files.size());
-    assertEquals(true, files.contains(mobFile1));
-  }
-}