Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 115C51054B for ; Wed, 3 Jul 2013 18:33:43 +0000 (UTC) Received: (qmail 23923 invoked by uid 500); 3 Jul 2013 18:33:42 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 23889 invoked by uid 500); 3 Jul 2013 18:33:42 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 23882 invoked by uid 99); 3 Jul 2013 18:33:42 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 03 Jul 2013 18:33:42 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 03 Jul 2013 18:33:21 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 527E82388980; Wed, 3 Jul 2013 18:32:58 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1499510 [2/7] - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/client/ core/src/main/java/org/apache/accumulo/core/client/admin/ core/src/main/java/org/apache/accumulo/core/client/impl/ core/src/main/java/org/apache/accumu... Date: Wed, 03 Jul 2013 18:32:53 -0000 To: commits@accumulo.apache.org From: ctubbsii@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20130703183258.527E82388980@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Added: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForUserTables.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForUserTables.java?rev=1499510&view=auto ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForUserTables.java (added) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForUserTables.java Wed Jul 3 18:32:51 2013 @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.metadata; + +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.security.thrift.TCredentials; + +/** + * A metadata servicer for user tables.
+ * Metadata for user tables are serviced in the metadata table. + */ +class ServicerForUserTables extends TableMetadataServicer { + + public ServicerForUserTables(Instance instance, TCredentials credentials, String tableId) { + super(instance, credentials, MetadataTable.NAME, tableId); + } + +} Propchange: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForUserTables.java ------------------------------------------------------------------------------ svn:mime-type = text/plain Added: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/TableMetadataServicer.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/TableMetadataServicer.java?rev=1499510&view=auto ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/TableMetadataServicer.java (added) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/TableMetadataServicer.java Wed Jul 3 18:32:51 2013 @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.metadata; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.SortedMap; +import java.util.SortedSet; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.KeyExtent; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.CredentialHelper; +import org.apache.accumulo.core.security.thrift.TCredentials; +import org.apache.hadoop.io.Text; + +/** + * A {@link MetadataServicer} that is backed by a table + */ +abstract class TableMetadataServicer extends MetadataServicer { + + private Instance instance; + private TCredentials credentials; + private String tableIdBeingServiced; + private String serviceTableName; + + public TableMetadataServicer(Instance instance, TCredentials credentials, String serviceTableName, String tableIdBeingServiced) { + this.instance = instance; + this.credentials = credentials; + this.serviceTableName = serviceTableName; + this.tableIdBeingServiced = tableIdBeingServiced; + } + + @Override + public String getServicedTableId() { + return tableIdBeingServiced; + } + + public String getServicingTableName() { + return serviceTableName; + } + + @Override + public void getTabletLocations(SortedMap tablets) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + + Scanner scanner = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(getServicingTableName(), + Authorizations.EMPTY); + + TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); + scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); + + // position at first entry in metadata table for given table + scanner.setRange(TabletsSection.getRange(getServicedTableId())); + + Text colf = new Text(); + Text colq = new Text(); + + KeyExtent currentKeyExtent = null; + String location = null; + Text row = null; + // acquire this table's tablets from the metadata table which services it + for (Entry entry : scanner) { + if (row != null) { + if (!row.equals(entry.getKey().getRow())) { + currentKeyExtent = null; + location = null; + row = entry.getKey().getRow(); + } + } else { + row = entry.getKey().getRow(); + } + + colf = entry.getKey().getColumnFamily(colf); + colq = entry.getKey().getColumnQualifier(colq); + + if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(colf, colq)) { + currentKeyExtent = new KeyExtent(entry.getKey().getRow(), entry.getValue()); + tablets.put(currentKeyExtent, location); + currentKeyExtent = null; + location = null; + } else if (colf.equals(TabletsSection.CurrentLocationColumnFamily.NAME)) { + location = entry.getValue().toString(); + } + + } + + validateEntries(tablets); + } + + private void validateEntries(SortedMap tablets) throws AccumuloException { + SortedSet tabletsKeys = (SortedSet) tablets.keySet(); + // sanity check of metadata table entries + // make sure tablets has no holes, and that it starts and ends w/ null + if (tabletsKeys.size() == 0) + throw new AccumuloException("No entries found in metadata table for table " + getServicedTableId()); + + if (tabletsKeys.first().getPrevEndRow() != null) + throw new AccumuloException("Problem with metadata table, first entry for table " + getServicedTableId() + "- " + tabletsKeys.first() + + " - has non null prev end row"); + + if (tabletsKeys.last().getEndRow() != null) + throw new AccumuloException("Problem with metadata table, last entry for table " + getServicedTableId() + "- " + tabletsKeys.first() + + " - has non null end row"); + + Iterator tabIter = tabletsKeys.iterator(); + Text lastEndRow = tabIter.next().getEndRow(); + while (tabIter.hasNext()) { + KeyExtent tabke = tabIter.next(); + + if (tabke.getPrevEndRow() == null) + throw new AccumuloException("Problem with metadata table, it has null prev end row in middle of table " + tabke); + + if (!tabke.getPrevEndRow().equals(lastEndRow)) + throw new AccumuloException("Problem with metadata table, it has a hole " + tabke.getPrevEndRow() + " != " + lastEndRow); + + lastEndRow = tabke.getEndRow(); + } + + // end METADATA table sanity check + } + +} Added: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java?rev=1499510&view=auto ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java (added) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java Wed Jul 3 18:32:51 2013 @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.metadata.schema; + +public class DataFileValue { + private long size; + private long numEntries; + private long time = -1; + + public DataFileValue(long size, long numEntries, long time) { + this.size = size; + this.numEntries = numEntries; + this.time = time; + } + + public DataFileValue(long size, long numEntries) { + this.size = size; + this.numEntries = numEntries; + this.time = -1; + } + + public DataFileValue(byte[] encodedDFV) { + String[] ba = new String(encodedDFV).split(","); + + size = Long.parseLong(ba[0]); + numEntries = Long.parseLong(ba[1]); + + if (ba.length == 3) + time = Long.parseLong(ba[2]); + else + time = -1; + } + + public long getSize() { + return size; + } + + public long getNumEntries() { + return numEntries; + } + + public boolean isTimeSet() { + return time >= 0; + } + + public long getTime() { + return time; + } + + public byte[] encode() { + if (time >= 0) + return ("" + size + "," + numEntries + "," + time).getBytes(); + return ("" + size + "," + numEntries).getBytes(); + } + + @Override + public boolean equals(Object o) { + if (o instanceof DataFileValue) { + DataFileValue odfv = (DataFileValue) o; + + return size == odfv.size && numEntries == odfv.numEntries; + } + + return false; + } + + @Override + public int hashCode() { + return Long.valueOf(size + numEntries).hashCode(); + } + + @Override + public String toString() { + return size + " " + numEntries; + } + + public void setTime(long time) { + if (time < 0) + throw new IllegalArgumentException(); + this.time = time; + } +} Propchange: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java ------------------------------------------------------------------------------ svn:mime-type = text/plain Added: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java?rev=1499510&view=auto ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java (added) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java Wed Jul 3 18:32:51 2013 @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.metadata.schema; + +import org.apache.accumulo.core.client.admin.TimeType; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.PartialKey; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.util.ColumnFQ; +import org.apache.hadoop.io.Text; + +/** + * Describes the table schema used for metadata tables + */ +public class MetadataSchema { + + private static final String RESERVED_PREFIX = "~"; + + private static class Section { + private String rowPrefix; + private Range range; + + private Section(String startRow, boolean startInclusive, String endRow, boolean endInclusive) { + rowPrefix = startRow; + range = new Range(startRow, startInclusive, endRow, endInclusive); + } + } + + /** + * Used for storing information about tablets + */ + public static class TabletsSection { + private static final Section section = new Section(null, false, RESERVED_PREFIX, false); + + public static Range getRange() { + return section.range; + } + + public static Range getRange(String tableId) { + return new Range(new Key(tableId + ';'), true, new Key(tableId + '<').followingKey(PartialKey.ROW), false); + } + + public static Text getRow(Text tableId, Text endRow) { + Text entry = new Text(tableId); + + if (endRow == null) { + // append delimiter for default tablet + entry.append(new byte[] {'<'}, 0, 1); + } else { + // append delimiter for regular tablets + entry.append(new byte[] {';'}, 0, 1); + entry.append(endRow.getBytes(), 0, endRow.getLength()); + } + + return entry; + } + + /** + * Column family for storing the tablet information needed by clients + */ + public static class TabletColumnFamily { + /** + * This needs to sort after all other column families for that tablet, because the {@link PREV_ROW_COLUMN} sits in this and that needs to sort last + * because the {@link SimpleGarbageCollector} relies on this. + */ + public static final Text NAME = new Text("~tab"); + /** + * README : very important that prevRow sort last to avoid race conditions between garbage collector and split this needs to sort after everything else + * for that tablet + */ + public static final ColumnFQ PREV_ROW_COLUMN = new ColumnFQ(NAME, new Text("~pr")); + /** + * A temporary field in case a split fails and we need to roll back + */ + public static final ColumnFQ OLD_PREV_ROW_COLUMN = new ColumnFQ(NAME, new Text("oldprevrow")); + /** + * A temporary field for splits to optimize certain operations + */ + public static final ColumnFQ SPLIT_RATIO_COLUMN = new ColumnFQ(NAME, new Text("splitRatio")); + } + + /** + * Column family for recording information used by the TServer + */ + public static class ServerColumnFamily { + public static final Text NAME = new Text("srv"); + /** + * Holds the location of the tablet in the DFS file system + */ + public static final ColumnFQ DIRECTORY_COLUMN = new ColumnFQ(NAME, new Text("dir")); + /** + * Holds the {@link TimeType} + */ + public static final ColumnFQ TIME_COLUMN = new ColumnFQ(NAME, new Text("time")); + /** + * Holds flush IDs to enable waiting on a flush to complete + */ + public static final ColumnFQ FLUSH_COLUMN = new ColumnFQ(NAME, new Text("flush")); + /** + * Holds compact IDs to enable waiting on a compaction to complete + */ + public static final ColumnFQ COMPACT_COLUMN = new ColumnFQ(NAME, new Text("compact")); + /** + * Holds lock IDs to enable a sanity check to ensure that the TServer writing to the metadata tablet is not dead + */ + public static final ColumnFQ LOCK_COLUMN = new ColumnFQ(NAME, new Text("lock")); + } + + /** + * Column family for storing entries created by the TServer to indicate it has loaded a tablet that it was assigned + */ + public static class CurrentLocationColumnFamily { + public static final Text NAME = new Text("loc"); + } + + /** + * Column family for storing the assigned location + */ + public static class FutureLocationColumnFamily { + public static final Text NAME = new Text("future"); + } + + /** + * Column family for storing last location, as a hint for assignment + */ + public static class LastLocationColumnFamily { + public static final Text NAME = new Text("last"); + } + + /** + * Temporary markers that indicate a tablet loaded a bulk file + */ + public static class BulkFileColumnFamily { + public static final Text NAME = new Text("loaded"); + } + + /** + * Temporary marker that indicates a tablet was successfully cloned + */ + public static class ClonedColumnFamily { + public static final Text NAME = new Text("!cloned"); + } + + /** + * Column family for storing files used by a tablet + */ + public static class DataFileColumnFamily { + public static final Text NAME = new Text("file"); + } + + /** + * Column family for storing the set of files scanned with an isolated scanner, to prevent them from being deleted + */ + public static class ScanFileColumnFamily { + public static final Text NAME = new Text("scan"); + } + + /** + * Column family for storing write-ahead log entries + */ + public static class LogColumnFamily { + public static final Text NAME = new Text("log"); + } + + /** + * Column family for indicating that the files in a tablet have been trimmed to only include data for the current tablet, so that they are safe to merge + */ + public static class ChoppedColumnFamily { + public static final Text NAME = new Text("chopped"); + public static final ColumnFQ CHOPPED_COLUMN = new ColumnFQ(NAME, new Text("chopped")); + } + } + + /** + * Contains additional metadata in a reserved area not for tablets + */ + public static class ReservedSection { + private static final Section section = new Section(RESERVED_PREFIX, true, null, false); + + public static Range getRange() { + return section.range; + } + + public static String getRowPrefix() { + return section.rowPrefix; + } + + } + + /** + * Holds delete markers for potentially unused files/directories + */ + public static class DeletesSection { + private static final Section section = new Section(RESERVED_PREFIX + "del", true, RESERVED_PREFIX + "dem", false); + + public static Range getRange() { + return section.range; + } + + public static String getRowPrefix() { + return section.rowPrefix; + } + + } + + /** + * Holds bulk-load-in-progress processing flags + */ + public static class BlipSection { + private static final Section section = new Section(RESERVED_PREFIX + "blip", true, RESERVED_PREFIX + "bliq", false); + + public static Range getRange() { + return section.range; + } + + public static String getRowPrefix() { + return section.rowPrefix; + } + + } + +} Propchange: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java ------------------------------------------------------------------------------ svn:mime-type = text/plain Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/ColumnFQ.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/ColumnFQ.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/ColumnFQ.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/ColumnFQ.java Wed Jul 3 18:32:51 2013 @@ -69,30 +69,6 @@ public class ColumnFQ implements Compara m.putDelete(colf, colq); } - /** - * @deprecated since 1.5, use {@link #fetch(ScannerBase)} instead - */ - @Deprecated - public static void fetch(ScannerBase sb, ColumnFQ cfq) { - sb.fetchColumn(cfq.colf, cfq.colq); - } - - /** - * @deprecated since 1.5, use {@link #put(Mutation, Value)} instead - */ - @Deprecated - public static void put(Mutation m, ColumnFQ cfq, Value v) { - m.put(cfq.colf, cfq.colq, v); - } - - /** - * @deprecated since 1.5, use {@link #putDelete(Mutation)} instead - */ - @Deprecated - public static void putDelete(Mutation m, ColumnFQ cfq) { - m.putDelete(cfq.colf, cfq.colq); - } - @Override public boolean equals(Object o) { if (!(o instanceof ColumnFQ)) Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/Merge.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/Merge.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/Merge.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/Merge.java Wed Jul 3 18:32:51 2013 @@ -31,6 +31,9 @@ import org.apache.accumulo.core.conf.Pro import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.KeyExtent; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.security.Authorizations; import org.apache.commons.cli.ParseException; import org.apache.hadoop.io.Text; @@ -216,8 +219,8 @@ public class Merge { throw new MergeException(e); } scanner.setRange(new KeyExtent(new Text(tableId), end, start).toMetadataRange()); - scanner.fetchColumnFamily(MetadataTable.DATAFILE_COLUMN_FAMILY); - MetadataTable.PREV_ROW_COLUMN.fetch(scanner); + scanner.fetchColumnFamily(DataFileColumnFamily.NAME); + TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); final Iterator> iterator = scanner.iterator(); Iterator result = new Iterator() { @@ -233,12 +236,12 @@ public class Merge { while (iterator.hasNext()) { Entry entry = iterator.next(); Key key = entry.getKey(); - if (key.getColumnFamily().equals(MetadataTable.DATAFILE_COLUMN_FAMILY)) { + if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) { String[] sizeEntries = new String(entry.getValue().get()).split(","); if (sizeEntries.length == 2) { tabletSize += Long.parseLong(sizeEntries[0]); } - } else if (MetadataTable.PREV_ROW_COLUMN.hasColumns(key)) { + } else if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) { KeyExtent extent = new KeyExtent(key.getRow(), entry.getValue()); return new Size(extent, tabletSize); } Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/FlushCommand.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/FlushCommand.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/FlushCommand.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/FlushCommand.java Wed Jul 3 18:32:51 2013 @@ -19,7 +19,7 @@ package org.apache.accumulo.core.util.sh import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.util.MetadataTable; +import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.util.shell.Shell; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/GetSplitsCommand.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/GetSplitsCommand.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/GetSplitsCommand.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/GetSplitsCommand.java Wed Jul 3 18:32:51 2013 @@ -30,9 +30,10 @@ import org.apache.accumulo.core.data.Key import org.apache.accumulo.core.data.KeyExtent; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.RootTable; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.util.MetadataTable; -import org.apache.accumulo.core.util.RootTable; import org.apache.accumulo.core.util.TextUtil; import org.apache.accumulo.core.util.format.BinaryFormatter; import org.apache.accumulo.core.util.shell.Shell; @@ -72,14 +73,14 @@ public class GetSplitsCommand extends Co } else { String systemTableToCheck = MetadataTable.NAME.equals(tableName) ? RootTable.NAME : MetadataTable.NAME; final Scanner scanner = shellState.getConnector().createScanner(systemTableToCheck, Authorizations.EMPTY); - MetadataTable.PREV_ROW_COLUMN.fetch(scanner); + TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); final Text start = new Text(shellState.getConnector().tableOperations().tableIdMap().get(tableName)); final Text end = new Text(start); end.append(new byte[] {'<'}, 0, 1); scanner.setRange(new Range(start, end)); for (Iterator> iterator = scanner.iterator(); iterator.hasNext();) { final Entry next = iterator.next(); - if (MetadataTable.PREV_ROW_COLUMN.hasColumns(next.getKey())) { + if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(next.getKey())) { KeyExtent extent = new KeyExtent(next.getKey().getRow(), next.getValue()); final String pr = encode(encode, extent.getPrevEndRow()); final String er = encode(encode, extent.getEndRow()); Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/OfflineCommand.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/OfflineCommand.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/OfflineCommand.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/OfflineCommand.java Wed Jul 3 18:32:51 2013 @@ -19,7 +19,7 @@ package org.apache.accumulo.core.util.sh import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.util.MetadataTable; +import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.util.shell.Shell; public class OfflineCommand extends TableOperation { Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/OnlineCommand.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/OnlineCommand.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/OnlineCommand.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/OnlineCommand.java Wed Jul 3 18:32:51 2013 @@ -19,7 +19,7 @@ package org.apache.accumulo.core.util.sh import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.util.MetadataTable; +import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.util.shell.Shell; public class OnlineCommand extends TableOperation { Modified: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java (original) +++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java Wed Jul 3 18:32:51 2013 @@ -47,10 +47,12 @@ import org.apache.accumulo.core.data.Mut import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.metadata.MetadataLocationObtainer; +import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.RootTable; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.security.thrift.TCredentials; -import org.apache.accumulo.core.util.MetadataTable; import org.apache.accumulo.core.util.Pair; -import org.apache.accumulo.core.util.RootTable; import org.apache.hadoop.io.Text; public class TabletLocatorImplTest extends TestCase { @@ -454,7 +456,7 @@ public class TabletLocatorImplTest exten public Connector getConnector(org.apache.accumulo.core.security.thrift.AuthInfo auth) throws AccumuloException, AccumuloSecurityException { return getConnector(auth.user, auth.getPassword()); } - + @Override public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException { throw new UnsupportedOperationException(); @@ -474,7 +476,8 @@ public class TabletLocatorImplTest exten } @Override - public TabletLocations lookupTablet(TabletLocation src, Text row, Text stopRow, TabletLocator parent, TCredentials credentials) throws AccumuloSecurityException { + public TabletLocations lookupTablet(TabletLocation src, Text row, Text stopRow, TabletLocator parent, TCredentials credentials) + throws AccumuloSecurityException { // System.out.println("lookupTablet("+src+","+row+","+stopRow+","+ parent+")"); // System.out.println(tservers); @@ -503,7 +506,7 @@ public class TabletLocatorImplTest exten SortedMap results = tabletData.tailMap(startKey).headMap(stopKey); - Pair,List> metadata = MetadataTable.getMetadataLocationEntries(results); + Pair,List> metadata = MetadataLocationObtainer.getMetadataLocationEntries(results); for (Entry entry : metadata.getFirst().entrySet()) { list.add(new TabletLocation(entry.getKey(), entry.getValue().toString())); @@ -559,7 +562,7 @@ public class TabletLocatorImplTest exten if (failures.size() > 0) parent.invalidateCache(failures); - SortedMap metadata = MetadataTable.getMetadataLocationEntries(results).getFirst(); + SortedMap metadata = MetadataLocationObtainer.getMetadataLocationEntries(results).getFirst(); for (Entry entry : metadata.entrySet()) { list.add(new TabletLocation(entry.getKey(), entry.getValue().toString())); @@ -606,18 +609,19 @@ public class TabletLocatorImplTest exten if (location != null) { if (instance == null) instance = ""; - Key lk = new Key(mr, MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY, new Text(instance)); + Key lk = new Key(mr, TabletsSection.CurrentLocationColumnFamily.NAME, new Text(instance)); tabletData.put(lk, new Value(location.getBytes())); } - Key pk = new Key(mr, MetadataTable.PREV_ROW_COLUMN.getColumnFamily(), MetadataTable.PREV_ROW_COLUMN.getColumnQualifier()); + Key pk = new Key(mr, TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnFamily(), + TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnQualifier()); tabletData.put(pk, per); } static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyExtent ke, String location) { setLocation(tservers, server, tablet, ke, location, ""); } - + static void deleteServer(TServers tservers, String server) { tservers.tservers.remove(server); @@ -1274,7 +1278,6 @@ public class TabletLocatorImplTest exten } catch (Exception e) { } - - + } } Copied: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/metadata/MetadataServicerTest.java (from r1497400, accumulo/trunk/core/src/test/java/org/apache/accumulo/core/util/MetadataTableTest.java) URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/metadata/MetadataServicerTest.java?p2=accumulo/trunk/core/src/test/java/org/apache/accumulo/core/metadata/MetadataServicerTest.java&p1=accumulo/trunk/core/src/test/java/org/apache/accumulo/core/util/MetadataTableTest.java&r1=1497400&r2=1499510&rev=1499510&view=diff ============================================================================== --- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/util/MetadataTableTest.java (original) +++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/metadata/MetadataServicerTest.java Wed Jul 3 18:32:51 2013 @@ -14,20 +14,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.accumulo.core.util; +package org.apache.accumulo.core.metadata; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; -import java.util.SortedSet; -import java.util.TreeSet; - -import org.apache.accumulo.core.data.KeyExtent; -import org.apache.hadoop.io.Text; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.mock.MockInstance; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.security.CredentialHelper; +import org.apache.accumulo.core.security.thrift.TCredentials; import org.junit.Test; -public class MetadataTableTest { +public class MetadataServicerTest { @Test public void checkSystemTableIdentifiers() { @@ -35,75 +40,47 @@ public class MetadataTableTest { assertNotEquals(RootTable.NAME, MetadataTable.NAME); } - private KeyExtent createKeyExtent(String tname, String er, String per) { - return new KeyExtent(new Text(tname), er == null ? null : new Text(er), per == null ? null : new Text(per)); - } - - private SortedSet createKeyExtents(String data[][]) { - - TreeSet extents = new TreeSet(); - for (String[] exdata : data) { - extents.add(createKeyExtent(exdata[0], exdata[1], exdata[2])); - } - - return extents; - } - - private void runTest(String beginRange, String endRange) { - KeyExtent ke = createKeyExtent("foo", endRange, beginRange); - - SortedSet children = createKeyExtents(new String[][] {new String[] {"foo", endRange, beginRange}}); - - assertTrue(MetadataTable.isContiguousRange(ke, children)); - - children = createKeyExtents(new String[][] {new String[] {"foo", "r1", beginRange}, new String[] {"foo", endRange, "r1"}}); - - assertTrue(MetadataTable.isContiguousRange(ke, children)); - - children = createKeyExtents(new String[][] {new String[] {"foo", "r1", beginRange}, new String[] {"foo", endRange, "r2"}}); - - assertFalse(MetadataTable.isContiguousRange(ke, children)); - - children = createKeyExtents(new String[][] {new String[] {"foo", "r1", beginRange}, new String[] {"foo", (endRange == null ? "r2" : endRange + "Z"), "r1"}}); - - assertFalse(MetadataTable.isContiguousRange(ke, children)); - - children = createKeyExtents(new String[][] {new String[] {"foo", "r1", (beginRange == null ? "r0" : "a" + beginRange)}, - new String[] {"foo", endRange, "r1"}}); - - assertFalse(MetadataTable.isContiguousRange(ke, children)); - - children = createKeyExtents(new String[][] {new String[] {"foo", "r1", beginRange}, new String[] {"foo", "r2", "r1"}, new String[] {"foo", endRange, "r2"}}); - - assertTrue(MetadataTable.isContiguousRange(ke, children)); - - children = createKeyExtents(new String[][] {new String[] {"foo", "r1", beginRange}, new String[] {"foo", "r2", "r1"}, new String[] {"foo", endRange, "r1"}}); - - assertFalse(MetadataTable.isContiguousRange(ke, children)); - - children = createKeyExtents(new String[][] {new String[] {"foo", "r1", beginRange}, new String[] {"foo", "r2", null}, new String[] {"foo", endRange, "r2"}}); - - assertFalse(MetadataTable.isContiguousRange(ke, children)); - - if (endRange == null) { - children = createKeyExtents(new String[][] {new String[] {"foo", "r1", beginRange}, new String[] {"foo", null, "r1"}, - new String[] {"foo", endRange, "r2"}}); - - assertFalse(MetadataTable.isContiguousRange(ke, children)); - } - - children = createKeyExtents(new String[][] {new String[] {"foo", "r1", beginRange}, new String[] {"foo", "r2", "r1"}, new String[] {"foo", "r3", "r2"}, - new String[] {"foo", endRange, "r3"}}); - - assertTrue(MetadataTable.isContiguousRange(ke, children)); - - } - @Test - public void testICR1() { - runTest(null, null); - runTest(null, "r4"); - runTest("r0", null); - runTest("r0", "r4"); + public void testGetCorrectServicer() throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException { + String userTableName = "A"; + MockInstance instance = new MockInstance("metadataTest"); + Connector connector = instance.getConnector("root", new PasswordToken("")); + connector.tableOperations().create(userTableName); + String userTableId = connector.tableOperations().tableIdMap().get(userTableName); + TCredentials credentials = CredentialHelper.createSquelchError("root", new PasswordToken(""), instance.getInstanceID()); + + MetadataServicer ms = MetadataServicer.forTableId(instance, credentials, RootTable.ID); + assertTrue(ms instanceof ServicerForRootTable); + assertFalse(ms instanceof TableMetadataServicer); + assertEquals(RootTable.ID, ms.getServicedTableId()); + + ms = MetadataServicer.forTableId(instance, credentials, MetadataTable.ID); + assertTrue(ms instanceof ServicerForMetadataTable); + assertTrue(ms instanceof TableMetadataServicer); + assertEquals(RootTable.NAME, ((TableMetadataServicer) ms).getServicingTableName()); + assertEquals(MetadataTable.ID, ms.getServicedTableId()); + + ms = MetadataServicer.forTableId(instance, credentials, userTableId); + assertTrue(ms instanceof ServicerForUserTables); + assertTrue(ms instanceof TableMetadataServicer); + assertEquals(MetadataTable.NAME, ((TableMetadataServicer) ms).getServicingTableName()); + assertEquals(userTableId, ms.getServicedTableId()); + + ms = MetadataServicer.forTableName(instance, credentials, RootTable.NAME); + assertTrue(ms instanceof ServicerForRootTable); + assertFalse(ms instanceof TableMetadataServicer); + assertEquals(RootTable.ID, ms.getServicedTableId()); + + ms = MetadataServicer.forTableName(instance, credentials, MetadataTable.NAME); + assertTrue(ms instanceof ServicerForMetadataTable); + assertTrue(ms instanceof TableMetadataServicer); + assertEquals(RootTable.NAME, ((TableMetadataServicer) ms).getServicingTableName()); + assertEquals(MetadataTable.ID, ms.getServicedTableId()); + + ms = MetadataServicer.forTableName(instance, credentials, userTableName); + assertTrue(ms instanceof ServicerForUserTables); + assertTrue(ms instanceof TableMetadataServicer); + assertEquals(MetadataTable.NAME, ((TableMetadataServicer) ms).getServicingTableName()); + assertEquals(userTableId, ms.getServicedTableId()); } } Modified: accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java URL: http://svn.apache.org/viewvc/accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java (original) +++ accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java Wed Jul 3 18:32:51 2013 @@ -50,7 +50,7 @@ import org.apache.accumulo.core.iterator import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.user.SummingCombiner; import org.apache.accumulo.core.iterators.user.VersioningIterator; -import org.apache.accumulo.core.util.MetadataTable; +import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint; import org.apache.accumulo.minicluster.MiniAccumuloCluster; @@ -147,7 +147,6 @@ public class SimpleTest { props.put("tokenClass", PasswordToken.class.getName()); protocolClass = getRandomProtocol(); - System.out.println(protocolClass.getName()); proxyPort = PortUtils.getRandomFreePort(); proxyServer = Proxy.createProxyServer(org.apache.accumulo.proxy.thrift.AccumuloProxy.class, org.apache.accumulo.proxy.ProxyServer.class, proxyPort, @@ -1001,14 +1000,14 @@ public class SimpleTest { client.closeScanner(scanner); assertEquals(10, more.getResults().size()); client.deleteTable(creds, "test2"); - + // don't know how to test this, call it just for fun client.clearLocatorCache(creds, TABLE_TEST); - + // compact client.compactTable(creds, TABLE_TEST, null, null, null, true, true); assertEquals(1, countFiles(TABLE_TEST)); - + // get disk usage client.cloneTable(creds, TABLE_TEST, "test2", true, null, null); Set tablesToScan = new HashSet(); @@ -1028,7 +1027,7 @@ public class SimpleTest { assertEquals(1, diskUsage.get(2).getTables().size()); client.deleteTable(creds, "foo"); client.deleteTable(creds, "test2"); - + // export/import String dir = folder.getRoot() + "/test"; String destDir = folder.getRoot() + "/test_dest"; Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/ServerConstants.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/ServerConstants.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/ServerConstants.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/ServerConstants.java Wed Jul 3 18:32:51 2013 @@ -17,9 +17,9 @@ package org.apache.accumulo.server; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.util.CachedConfiguration; -import org.apache.accumulo.core.util.MetadataTable; -import org.apache.accumulo.core.util.RootTable; import org.apache.accumulo.server.conf.ServerConfiguration; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -28,8 +28,12 @@ public class ServerConstants { // versions should never be negative public static final Integer WIRE_VERSION = 2; - public static final int DATA_VERSION = 5; - public static final int PREV_DATA_VERSION = 4; + + /** + * current version reflects the addition of a separate root table (ACCUMULO-1481) + */ + public static final int DATA_VERSION = 6; + public static final int PREV_DATA_VERSION = 5; // these are functions to delay loading the Accumulo configuration unless we must public static String[] getBaseDirs() { @@ -92,6 +96,6 @@ public class ServerConstants { } public static String getRootTabletDir() { - return prefix(getRootTableDirs(), RootTable.ZROOT_TABLET)[0]; + return prefix(getRootTableDirs(), RootTable.ROOT_TABLET_LOCATION)[0]; } } Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/BulkImporter.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/BulkImporter.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/BulkImporter.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/BulkImporter.java Wed Jul 3 18:32:51 2013 @@ -51,11 +51,11 @@ import org.apache.accumulo.core.data.thr import org.apache.accumulo.core.file.FileOperations; import org.apache.accumulo.core.file.FileSKVIterator; import org.apache.accumulo.core.file.FileUtil; +import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.security.thrift.TCredentials; import org.apache.accumulo.core.tabletserver.thrift.TabletClientService; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.LoggingRunnable; -import org.apache.accumulo.core.util.MetadataTable; import org.apache.accumulo.core.util.NamingThreadFactory; import org.apache.accumulo.core.util.StopWatch; import org.apache.accumulo.core.util.ThriftUtil; @@ -132,7 +132,7 @@ public class BulkImporter { } ClientService.Client client = null; - final TabletLocator locator = TabletLocator.getInstance(instance, new Text(tableId)); + final TabletLocator locator = TabletLocator.getLocator(instance, new Text(tableId)); try { final Map> assignments = Collections.synchronizedSortedMap(new TreeMap>()); Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java Wed Jul 3 18:32:51 2013 @@ -32,11 +32,11 @@ import org.apache.accumulo.core.client.s import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.security.CredentialHelper; import org.apache.accumulo.core.security.thrift.TCredentials; import org.apache.accumulo.core.util.ByteBufferUtil; import org.apache.accumulo.core.util.OpTimer; -import org.apache.accumulo.core.util.RootTable; import org.apache.accumulo.core.util.StringUtil; import org.apache.accumulo.core.util.TextUtil; import org.apache.accumulo.core.zookeeper.ZooUtil; Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java Wed Jul 3 18:32:51 2013 @@ -28,9 +28,15 @@ import org.apache.accumulo.core.data.Col import org.apache.accumulo.core.data.KeyExtent; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.schema.DataFileValue; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily; import org.apache.accumulo.core.util.ColumnFQ; -import org.apache.accumulo.core.util.MetadataTable; -import org.apache.accumulo.core.util.MetadataTable.DataFileValue; import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.fate.zookeeper.TransactionWatcher.Arbitrator; import org.apache.accumulo.server.client.HdfsZooInstance; @@ -55,14 +61,15 @@ public class MetadataConstraints impleme } } - private static final HashSet validColumnQuals = new HashSet(Arrays.asList(new ColumnFQ[] {MetadataTable.PREV_ROW_COLUMN, - MetadataTable.OLD_PREV_ROW_COLUMN, MetadataTable.DIRECTORY_COLUMN, MetadataTable.SPLIT_RATIO_COLUMN, MetadataTable.TIME_COLUMN, - MetadataTable.LOCK_COLUMN, MetadataTable.FLUSH_COLUMN, MetadataTable.COMPACT_COLUMN})); - - private static final HashSet validColumnFams = new HashSet(Arrays.asList(new Text[] {MetadataTable.BULKFILE_COLUMN_FAMILY, - MetadataTable.LOG_COLUMN_FAMILY, MetadataTable.SCANFILE_COLUMN_FAMILY, MetadataTable.DATAFILE_COLUMN_FAMILY, - MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY, MetadataTable.LAST_LOCATION_COLUMN_FAMILY, MetadataTable.FUTURE_LOCATION_COLUMN_FAMILY, - MetadataTable.CHOPPED_COLUMN_FAMILY, MetadataTable.CLONED_COLUMN_FAMILY})); + private static final HashSet validColumnQuals = new HashSet(Arrays.asList(new ColumnFQ[] { + TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN, TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN, + TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN, TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN, TabletsSection.ServerColumnFamily.TIME_COLUMN, + TabletsSection.ServerColumnFamily.LOCK_COLUMN, TabletsSection.ServerColumnFamily.FLUSH_COLUMN, TabletsSection.ServerColumnFamily.COMPACT_COLUMN})); + + private static final HashSet validColumnFams = new HashSet(Arrays.asList(new Text[] {TabletsSection.BulkFileColumnFamily.NAME, + LogColumnFamily.NAME, ScanFileColumnFamily.NAME, DataFileColumnFamily.NAME, + TabletsSection.CurrentLocationColumnFamily.NAME, TabletsSection.LastLocationColumnFamily.NAME, TabletsSection.FutureLocationColumnFamily.NAME, + ChoppedColumnFamily.NAME, ClonedColumnFamily.NAME})); private static boolean isValidColumn(ColumnUpdate cu) { @@ -78,19 +85,20 @@ public class MetadataConstraints impleme static private ArrayList addViolation(ArrayList lst, int violation) { if (lst == null) lst = new ArrayList(); - lst.add((short)violation); + lst.add((short) violation); return lst; } static private ArrayList addIfNotPresent(ArrayList lst, int intViolation) { if (lst == null) return addViolation(lst, intViolation); - short violation = (short)intViolation; + short violation = (short) intViolation; if (!lst.contains(violation)) return addViolation(lst, intViolation); return lst; } + @Override public List check(Environment env, Mutation mutation) { ArrayList violations = null; @@ -144,7 +152,7 @@ public class MetadataConstraints impleme } boolean checkedBulk = false; - + for (ColumnUpdate columnUpdate : colUpdates) { Text columnFamily = new Text(columnUpdate.getColumnFamily()); @@ -155,11 +163,11 @@ public class MetadataConstraints impleme continue; } - if (columnUpdate.getValue().length == 0 && !columnFamily.equals(MetadataTable.SCANFILE_COLUMN_FAMILY)) { + if (columnUpdate.getValue().length == 0 && !columnFamily.equals(ScanFileColumnFamily.NAME)) { violations = addViolation(violations, 6); } - if (columnFamily.equals(MetadataTable.DATAFILE_COLUMN_FAMILY)) { + if (columnFamily.equals(DataFileColumnFamily.NAME)) { try { DataFileValue dfv = new DataFileValue(columnUpdate.getValue()); @@ -171,33 +179,33 @@ public class MetadataConstraints impleme } catch (ArrayIndexOutOfBoundsException aiooe) { violations = addViolation(violations, 1); } - } else if (columnFamily.equals(MetadataTable.SCANFILE_COLUMN_FAMILY)) { + } else if (columnFamily.equals(ScanFileColumnFamily.NAME)) { - } else if (columnFamily.equals(MetadataTable.BULKFILE_COLUMN_FAMILY)) { + } else if (columnFamily.equals(TabletsSection.BulkFileColumnFamily.NAME)) { if (!columnUpdate.isDeleted() && !checkedBulk) { // splits, which also write the time reference, are allowed to write this reference even when // the transaction is not running because the other half of the tablet is holding a reference // to the file. boolean isSplitMutation = false; - // When a tablet is assigned, it re-writes the metadata. It should probably only update the location information, - // but it writes everything. We allow it to re-write the bulk information if it is setting the location. - // See ACCUMULO-1230. + // When a tablet is assigned, it re-writes the metadata. It should probably only update the location information, + // but it writes everything. We allow it to re-write the bulk information if it is setting the location. + // See ACCUMULO-1230. boolean isLocationMutation = false; HashSet dataFiles = new HashSet(); HashSet loadedFiles = new HashSet(); - + String tidString = new String(columnUpdate.getValue()); int otherTidCount = 0; - + for (ColumnUpdate update : mutation.getUpdates()) { - if (new ColumnFQ(update).equals(MetadataTable.DIRECTORY_COLUMN)) { + if (new ColumnFQ(update).equals(TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN)) { isSplitMutation = true; - } else if (new Text(update.getColumnFamily()).equals(MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY)) { + } else if (new Text(update.getColumnFamily()).equals(TabletsSection.CurrentLocationColumnFamily.NAME)) { isLocationMutation = true; - } else if (new Text(update.getColumnFamily()).equals(MetadataTable.DATAFILE_COLUMN_FAMILY)) { + } else if (new Text(update.getColumnFamily()).equals(DataFileColumnFamily.NAME)) { dataFiles.add(new Text(update.getColumnQualifier())); - } else if (new Text(update.getColumnFamily()).equals(MetadataTable.BULKFILE_COLUMN_FAMILY)) { + } else if (new Text(update.getColumnFamily()).equals(TabletsSection.BulkFileColumnFamily.NAME)) { loadedFiles.add(new Text(update.getColumnQualifier())); if (!new String(update.getValue()).equals(tidString)) { @@ -223,7 +231,7 @@ public class MetadataConstraints impleme } else { if (!isValidColumn(columnUpdate)) { violations = addViolation(violations, 2); - } else if (new ColumnFQ(columnUpdate).equals(MetadataTable.PREV_ROW_COLUMN) && columnUpdate.getValue().length > 0 + } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN) && columnUpdate.getValue().length > 0 && (violations == null || !violations.contains((short) 4))) { KeyExtent ke = new KeyExtent(new Text(mutation.getRow()), (Text) null); @@ -234,7 +242,7 @@ public class MetadataConstraints impleme if (!prevEndRowLessThanEndRow) { violations = addViolation(violations, 3); } - } else if (new ColumnFQ(columnUpdate).equals(MetadataTable.LOCK_COLUMN)) { + } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.ServerColumnFamily.LOCK_COLUMN)) { if (zooCache == null) { zooCache = new ZooCache(); } @@ -263,7 +271,8 @@ public class MetadataConstraints impleme if (violations != null) { log.debug("violating metadata mutation : " + new String(mutation.getRow())); for (ColumnUpdate update : mutation.getUpdates()) { - log.debug(" update: " + new String(update.getColumnFamily()) + ":" + new String(update.getColumnQualifier()) + " value " + (update.isDeleted() ? "[delete]" : new String(update.getValue()))); + log.debug(" update: " + new String(update.getColumnFamily()) + ":" + new String(update.getColumnQualifier()) + " value " + + (update.isDeleted() ? "[delete]" : new String(update.getValue()))); } } @@ -273,7 +282,8 @@ public class MetadataConstraints impleme protected Arbitrator getArbitrator() { return new ZooArbitrator(); } - + + @Override public String getViolationDescription(short violationCode) { switch (violationCode) { case 1: @@ -296,6 +306,7 @@ public class MetadataConstraints impleme return null; } + @Override protected void finalize() { if (zooCache != null) zooCache.clear(); Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java Wed Jul 3 18:32:51 2013 @@ -42,8 +42,8 @@ import org.apache.accumulo.server.Server import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.server.security.SecurityConstants; import org.apache.accumulo.server.util.AddressUtil; -import org.apache.accumulo.server.util.MetadataTable; -import org.apache.accumulo.server.util.MetadataTable.LogEntry; +import org.apache.accumulo.server.util.MetadataTableUtil; +import org.apache.accumulo.server.util.MetadataTableUtil.LogEntry; import org.apache.accumulo.server.zookeeper.ZooReaderWriter; import org.apache.accumulo.trace.instrument.Span; import org.apache.accumulo.trace.instrument.Trace; @@ -223,7 +223,7 @@ public class GarbageCollectWriteAheadLog private static int removeMetadataEntries(Map fileToServerMap, Set sortedWALogs, GCStatus status) throws IOException, KeeperException, InterruptedException { int count = 0; - Iterator iterator = MetadataTable.getLogEntries(SecurityConstants.getSystemCredentials()); + Iterator iterator = MetadataTableUtil.getLogEntries(SecurityConstants.getSystemCredentials()); while (iterator.hasNext()) { for (String filename : iterator.next().logSet) { Path path; Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java Wed Jul 3 18:32:51 2013 @@ -61,11 +61,16 @@ import org.apache.accumulo.core.gc.thrif import org.apache.accumulo.core.gc.thrift.GCStatus; import org.apache.accumulo.core.gc.thrift.GcCycleStats; import org.apache.accumulo.core.master.state.tables.TableState; +import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.RootTable; +import org.apache.accumulo.core.metadata.schema.MetadataSchema; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.CredentialHelper; import org.apache.accumulo.core.security.SecurityUtil; import org.apache.accumulo.core.security.thrift.TCredentials; -import org.apache.accumulo.core.util.MetadataTable; import org.apache.accumulo.core.util.NamingThreadFactory; import org.apache.accumulo.core.util.ServerServices; import org.apache.accumulo.core.util.ServerServices.Service; @@ -308,6 +313,7 @@ public class SimpleGarbageCollector impl try { Connector connector = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)); connector.tableOperations().compact(MetadataTable.NAME, null, null, true, true); + connector.tableOperations().compact(RootTable.NAME, null, null, true, true); } catch (Exception e) { log.warn(e, e); } @@ -452,19 +458,24 @@ public class SimpleGarbageCollector impl } checkForBulkProcessingFiles = false; - Range range = MetadataTable.DELETED_RANGE; - candidates.addAll(getBatch(MetadataTable.DELETED_RANGE.getStartKey().getRow().toString(), range)); + candidates.addAll(getBatch(RootTable.NAME)); if (candidateMemExceeded) return candidates; - range = MetadataTable.DELETED_RANGE; - candidates.addAll(getBatch(MetadataTable.DELETED_RANGE.getStartKey().getRow().toString(), range)); + candidates.addAll(getBatch(MetadataTable.NAME)); return candidates; } - private Collection getBatch(String prefix, Range range) throws Exception { + /** + * Gets a batch of delete markers from the specified table + * + * @param tableName + * the name of the system table to scan (either {@link RootTable.NAME} or {@link MetadataTable.NAME}) + */ + private Collection getBatch(String tableName) throws Exception { // want to ensure GC makes progress... if the 1st N deletes are stable and we keep processing them, // then will never inspect deletes after N + Range range = MetadataSchema.DeletesSection.getRange(); if (continueKey != null) { if (!range.contains(continueKey)) { // continue key is for some other range @@ -474,13 +485,13 @@ public class SimpleGarbageCollector impl continueKey = null; } - Scanner scanner = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(MetadataTable.NAME, + Scanner scanner = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(tableName, Authorizations.EMPTY); scanner.setRange(range); List result = new ArrayList(); // find candidates for deletion; chop off the prefix for (Entry entry : scanner) { - String cand = entry.getKey().getRow().toString().substring(prefix.length()); + String cand = entry.getKey().getRow().toString().substring(MetadataSchema.DeletesSection.getRowPrefix().length()); result.add(cand); checkForBulkProcessingFiles |= cand.toLowerCase(Locale.ENGLISH).contains(Constants.BULK_PREFIX); if (almostOutOfMemory()) { @@ -504,7 +515,11 @@ public class SimpleGarbageCollector impl * selected 2. They are still in use in the file column family in the METADATA table */ public void confirmDeletes(SortedSet candidates) throws AccumuloException { - + confirmDeletes(RootTable.NAME, candidates); + confirmDeletes(MetadataTable.NAME, candidates); + } + + private void confirmDeletes(String tableName, SortedSet candidates) throws AccumuloException { Scanner scanner; if (offline) { // TODO @@ -516,8 +531,8 @@ public class SimpleGarbageCollector impl // } } else { try { - scanner = new IsolatedScanner(instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner( - MetadataTable.NAME, Authorizations.EMPTY)); + scanner = new IsolatedScanner(instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(tableName, + Authorizations.EMPTY)); } catch (AccumuloSecurityException ex) { throw new AccumuloException(ex); } catch (TableNotFoundException ex) { @@ -530,14 +545,14 @@ public class SimpleGarbageCollector impl log.debug("Checking for bulk processing flags"); - scanner.setRange(MetadataTable.BLIP_KEYSPACE); + scanner.setRange(MetadataSchema.BlipSection.getRange()); // WARNING: This block is IMPORTANT // You MUST REMOVE candidates that are in the same folder as a bulk // processing flag! for (Entry entry : scanner) { - String blipPath = entry.getKey().getRow().toString().substring(MetadataTable.BLIP_FLAG_PREFIX.length()); + String blipPath = entry.getKey().getRow().toString().substring(MetadataSchema.BlipSection.getRowPrefix().length()); Iterator tailIter = candidates.tailSet(blipPath).iterator(); int count = 0; while (tailIter.hasNext()) { @@ -558,17 +573,17 @@ public class SimpleGarbageCollector impl // skip candidates that are still in use in the file column family in // the metadata table scanner.clearColumns(); - scanner.fetchColumnFamily(MetadataTable.DATAFILE_COLUMN_FAMILY); - scanner.fetchColumnFamily(MetadataTable.SCANFILE_COLUMN_FAMILY); - MetadataTable.DIRECTORY_COLUMN.fetch(scanner); - TabletIterator tabletIterator = new TabletIterator(scanner, MetadataTable.KEYSPACE, false, true); + scanner.fetchColumnFamily(DataFileColumnFamily.NAME); + scanner.fetchColumnFamily(ScanFileColumnFamily.NAME); + TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner); + TabletIterator tabletIterator = new TabletIterator(scanner, MetadataSchema.TabletsSection.getRange(), false, true); while (tabletIterator.hasNext()) { Map tabletKeyValues = tabletIterator.next(); for (Entry entry : tabletKeyValues.entrySet()) { - if (entry.getKey().getColumnFamily().equals(MetadataTable.DATAFILE_COLUMN_FAMILY) - || entry.getKey().getColumnFamily().equals(MetadataTable.SCANFILE_COLUMN_FAMILY)) { + if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME) + || entry.getKey().getColumnFamily().equals(ScanFileColumnFamily.NAME)) { String cf = entry.getKey().getColumnQualifier().toString(); String delete = cf; @@ -586,16 +601,16 @@ public class SimpleGarbageCollector impl // WARNING: This line is EXTREMELY IMPORTANT. // You MUST REMOVE candidates that are still in use if (candidates.remove(delete)) - log.debug("Candidate was still in use in the METADATA table: " + delete); + log.debug("Candidate was still in use in the " + tableName + " table: " + delete); String path = delete.substring(0, delete.lastIndexOf('/')); if (candidates.remove(path)) - log.debug("Candidate was still in use in the METADATA table: " + path); - } else if (MetadataTable.DIRECTORY_COLUMN.hasColumns(entry.getKey())) { + log.debug("Candidate was still in use in the " + tableName + " table: " + path); + } else if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(entry.getKey())) { String table = new String(KeyExtent.tableOfMetadataRow(entry.getKey().getRow())); String delete = "/" + table + entry.getValue().toString(); if (candidates.remove(delete)) - log.debug("Candidate was still in use in the METADATA table: " + delete); + log.debug("Candidate was still in use in the " + tableName + " table: " + delete); } else throw new AccumuloException("Scanner over metadata table returned unexpected column : " + entry.getKey()); } @@ -604,16 +619,12 @@ public class SimpleGarbageCollector impl final static String METADATA_TABLE_DIR = "/" + MetadataTable.ID; - private static void putMarkerDeleteMutation(final String delete, final BatchWriter writer, final BatchWriter rootWriter) throws MutationsRejectedException { - if (delete.contains(METADATA_TABLE_DIR)) { - Mutation m = new Mutation(new Text(MetadataTable.DELETED_RANGE.getStartKey().getRow().toString() + delete)); - m.putDelete(EMPTY_TEXT, EMPTY_TEXT); - rootWriter.addMutation(m); - } else { - Mutation m = new Mutation(new Text(MetadataTable.DELETED_RANGE.getStartKey().getRow().toString() + delete)); - m.putDelete(EMPTY_TEXT, EMPTY_TEXT); - writer.addMutation(m); - } + private static void putMarkerDeleteMutation(final String delete, final BatchWriter metadataWriter, final BatchWriter rootWriter) + throws MutationsRejectedException { + BatchWriter writer = delete.contains(METADATA_TABLE_DIR) ? rootWriter : metadataWriter; + Mutation m = new Mutation(MetadataSchema.DeletesSection.getRowPrefix() + delete); + m.putDelete(EMPTY_TEXT, EMPTY_TEXT); + writer.addMutation(m); } /** @@ -629,9 +640,13 @@ public class SimpleGarbageCollector impl try { c = instance.getConnector(SecurityConstants.SYSTEM_PRINCIPAL, SecurityConstants.getSystemToken()); writer = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig()); - rootWriter = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig()); - } catch (Exception e) { - log.error("Unable to create writer to remove file from the " + MetadataTable.NAME + " table", e); + rootWriter = c.createBatchWriter(RootTable.NAME, new BatchWriterConfig()); + } catch (AccumuloException e) { + log.error("Unable to connect to Accumulo to write deletes", e); + } catch (AccumuloSecurityException e) { + log.error("Unable to connect to Accumulo to write deletes", e); + } catch (TableNotFoundException e) { + log.error("Unable to create writer to remove file from the " + e.getTableName() + " table", e); } } // when deleting a dir and all files in that dir, only need to delete the dir Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilter.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilter.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilter.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilter.java Wed Jul 3 18:32:51 2013 @@ -27,7 +27,7 @@ import org.apache.accumulo.core.iterator import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; -import org.apache.accumulo.core.util.MetadataTable; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.fate.zookeeper.TransactionWatcher.Arbitrator; import org.apache.accumulo.server.zookeeper.TransactionWatcher.ZooArbitrator; import org.apache.log4j.Logger; @@ -48,7 +48,7 @@ public class MetadataBulkLoadFilter exte @Override public boolean accept(Key k, Value v) { - if (!k.isDeleted() && k.compareColumnFamily(MetadataTable.BULKFILE_COLUMN_FAMILY) == 0) { + if (!k.isDeleted() && k.compareColumnFamily(TabletsSection.BulkFileColumnFamily.NAME) == 0) { long txid = Long.valueOf(v.toString()); Status status = bulkTxStatusCache.get(txid); @@ -69,10 +69,10 @@ public class MetadataBulkLoadFilter exte return status == Status.ACTIVE; } - + return true; } - + @Override public void init(SortedKeyValueIterator source, Map options, IteratorEnvironment env) throws IOException { super.init(source, options, env); @@ -80,7 +80,7 @@ public class MetadataBulkLoadFilter exte if (env.getIteratorScope() == IteratorScope.scan) { throw new IOException("This iterator not intended for use at scan time"); } - + bulkTxStatusCache = new HashMap(); arbitrator = getArbitrator(); } Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1499510&r1=1499509&r2=1499510&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java Wed Jul 3 18:32:51 2013 @@ -69,12 +69,15 @@ import org.apache.accumulo.core.master.t import org.apache.accumulo.core.master.thrift.TabletLoadState; import org.apache.accumulo.core.master.thrift.TabletServerStatus; import org.apache.accumulo.core.master.thrift.TabletSplit; +import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.RootTable; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.SecurityUtil; import org.apache.accumulo.core.security.thrift.TCredentials; import org.apache.accumulo.core.util.ByteBufferUtil; import org.apache.accumulo.core.util.Daemon; -import org.apache.accumulo.core.util.RootTable; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.fate.AgeOffStore; @@ -131,7 +134,7 @@ import org.apache.accumulo.server.securi import org.apache.accumulo.server.util.AddressUtil; import org.apache.accumulo.server.util.DefaultMap; import org.apache.accumulo.server.util.Halt; -import org.apache.accumulo.server.util.MetadataTable; +import org.apache.accumulo.server.util.MetadataTableUtil; import org.apache.accumulo.server.util.SystemPropUtil; import org.apache.accumulo.server.util.TServerUtils; import org.apache.accumulo.server.util.TablePropUtil; @@ -288,7 +291,7 @@ public class Master implements LiveTServ @Override public void run() { try { - MetadataTable.moveMetaDeleteMarkers(instance, SecurityConstants.getSystemCredentials()); + MetadataTableUtil.moveMetaDeleteMarkers(instance, SecurityConstants.getSystemCredentials()); Accumulo.updateAccumuloVersion(fs); log.info("Upgrade complete"); @@ -379,8 +382,8 @@ public class Master implements LiveTServ } private void checkNotMetadataTable(String tableName, TableOperation operation) throws ThriftTableOperationException { - if (tableName.compareTo(MetadataTable.NAME) == 0) { - String why = "Table names cannot be == " + MetadataTable.NAME; + if (MetadataTable.NAME.equals(tableName) || RootTable.NAME.equals(tableName)) { + String why = "Table names cannot be == " + RootTable.NAME + " or " + MetadataTable.NAME; log.warn(why); throw new ThriftTableOperationException(null, tableName, operation, TableOperationExceptionType.OTHER, why); } @@ -527,10 +530,10 @@ public class Master implements LiveTServ try { Connector conn = getConnector(); Scanner scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)); - MetadataTable.FLUSH_COLUMN.fetch(scanner); - MetadataTable.DIRECTORY_COLUMN.fetch(scanner); - scanner.fetchColumnFamily(MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY); - scanner.fetchColumnFamily(MetadataTable.LOG_COLUMN_FAMILY); + TabletsSection.ServerColumnFamily.FLUSH_COLUMN.fetch(scanner); + TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner); + scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); + scanner.fetchColumnFamily(LogColumnFamily.NAME); scanner.setRange(new KeyExtent(new Text(tableId), null, ByteBufferUtil.toText(startRow)).toMetadataRange()); RowIterator ri = new RowIterator(scanner); @@ -553,14 +556,14 @@ public class Master implements LiveTServ entry = row.next(); Key key = entry.getKey(); - if (MetadataTable.FLUSH_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier())) { + if (TabletsSection.ServerColumnFamily.FLUSH_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier())) { tabletFlushID = Long.parseLong(entry.getValue().toString()); } - if (MetadataTable.LOG_COLUMN_FAMILY.equals(key.getColumnFamily())) + if (LogColumnFamily.NAME.equals(key.getColumnFamily())) logs++; - if (MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY.equals(key.getColumnFamily())) { + if (TabletsSection.CurrentLocationColumnFamily.NAME.equals(key.getColumnFamily())) { online = true; server = new TServerInstance(entry.getValue(), key.getColumnQualifier()); } @@ -899,10 +902,6 @@ public class Master implements LiveTServ Text startRow = ByteBufferUtil.toText(arguments.get(1)); Text endRow = ByteBufferUtil.toText(arguments.get(2)); final String tableId = checkTableId(tableName, TableOperation.MERGE); - if (tableId.equals(RootTable.ID)) { - throw new ThriftTableOperationException(null, tableName, TableOperation.MERGE, TableOperationExceptionType.OTHER, - "cannot merge or split the root table"); - } log.debug("Creating merge op: " + tableId + " " + startRow + " " + endRow); if (!security.canMerge(c, tableId)) @@ -1027,8 +1026,6 @@ public class Master implements LiveTServ } public MergeInfo getMergeInfo(KeyExtent tablet) { - if (tablet.isRootTablet()) - return new MergeInfo(); return getMergeInfo(tablet.getTableId()); } @@ -1251,7 +1248,7 @@ public class Master implements LiveTServ private void cleanupMutations() throws AccumuloException, AccumuloSecurityException, TableNotFoundException { Connector connector = getConnector(); Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY); - MetadataTable.PREV_ROW_COLUMN.fetch(scanner); + TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); Set found = new HashSet(); for (Entry entry : scanner) { KeyExtent extent = new KeyExtent(entry.getKey().getRow(), entry.getValue());