accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [accumulo] 01/01: Merge branch '2.0'
Date Mon, 08 Jul 2019 18:45:38 GMT
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit f26ca535bab0434b36d83a3f8b7ef926de091b9c
Merge: 0ca9cfe ea58037
Author: Keith Turner <kturner@apache.org>
AuthorDate: Mon Jul 8 14:44:06 2019 -0400

    Merge branch '2.0'

 .../java/org/apache/accumulo/fate/AgeOffStore.java |  4 +-
 .../main/java/org/apache/accumulo/fate/Fate.java   | 20 ++++----
 .../java/org/apache/accumulo/fate/FateTxId.java    | 59 ++++++++++++++++++++++
 .../java/org/apache/accumulo/fate/ZooStore.java    |  8 +--
 .../server/constraints/MetadataConstraints.java    |  3 +-
 .../server/iterators/MetadataBulkLoadFilter.java   |  3 +-
 .../server/metadata/TabletMutatorBase.java         |  3 +-
 .../accumulo/server/util/MetadataTableUtil.java    | 51 ++++++++-----------
 .../master/tableOps/bulkVer1/BulkImport.java       | 13 +++--
 .../tableOps/bulkVer1/CleanUpBulkImport.java       |  3 +-
 .../master/tableOps/bulkVer1/CopyFailed.java       | 13 +++--
 .../master/tableOps/bulkVer1/LoadFiles.java        | 13 +++--
 .../master/tableOps/bulkVer2/BulkImportMove.java   | 20 +++++---
 .../tableOps/bulkVer2/CleanUpBulkImport.java       |  3 +-
 .../master/tableOps/bulkVer2/LoadFiles.java        | 11 ++--
 .../apache/accumulo/tserver/tablet/TabletData.java |  9 ++--
 .../accumulo/shell/commands/FateCommand.java       | 13 ++++-
 .../accumulo/test/functional/BulkFailureIT.java    |  3 +-
 18 files changed, 164 insertions(+), 88 deletions(-)

diff --cc server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java
index 3aaa3cf,0000000..db7dce7
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java
@@@ -1,202 -1,0 +1,203 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +
 +package org.apache.accumulo.server.metadata;
 +
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.dataImpl.KeyExtent;
 +import org.apache.accumulo.core.metadata.schema.Ample;
 +import org.apache.accumulo.core.metadata.schema.DataFileValue;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
 +import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType;
 +import org.apache.accumulo.core.tabletserver.log.LogEntry;
++import org.apache.accumulo.fate.FateTxId;
 +import org.apache.accumulo.fate.zookeeper.ZooLock;
 +import org.apache.accumulo.server.ServerContext;
 +import org.apache.hadoop.io.Text;
 +
 +import com.google.common.base.Preconditions;
 +
 +public abstract class TabletMutatorBase implements Ample.TabletMutator {
 +
 +  private final ServerContext context;
 +  private final KeyExtent extent;
 +  private final Mutation mutation;
 +  protected AutoCloseable closeAfterMutate;
 +  private boolean updatesEnabled = true;
 +
 +  protected TabletMutatorBase(ServerContext ctx, KeyExtent extent) {
 +    this.extent = extent;
 +    this.context = ctx;
 +    mutation = new Mutation(extent.getMetadataEntry());
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator putPrevEndRow(Text per) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.put(mutation,
 +        KeyExtent.encodePrevEndRow(extent.getPrevEndRow()));
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator putDir(String dir) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mutation, new Value(dir));
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator putFile(Ample.FileMeta path, DataFileValue dfv) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    mutation.put(DataFileColumnFamily.NAME, path.meta(), new Value(dfv.encode()));
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator deleteFile(Ample.FileMeta path) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    mutation.putDelete(DataFileColumnFamily.NAME, path.meta());
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator putScan(Ample.FileMeta path) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    mutation.put(ScanFileColumnFamily.NAME, path.meta(), new Value(new byte[0]));
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator deleteScan(Ample.FileMeta path) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    mutation.putDelete(ScanFileColumnFamily.NAME, path.meta());
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator putCompactionId(long compactionId) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    TabletsSection.ServerColumnFamily.COMPACT_COLUMN.put(mutation,
 +        new Value(Long.toString(compactionId)));
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator putFlushId(long flushId) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    TabletsSection.ServerColumnFamily.FLUSH_COLUMN.put(mutation, new Value(Long.toString(flushId)));
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator putTime(String time) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    TabletsSection.ServerColumnFamily.TIME_COLUMN.put(mutation, new Value(time));
 +    return this;
 +  }
 +
 +  private String getLocationFamily(LocationType type) {
 +    switch (type) {
 +      case CURRENT:
 +        return TabletsSection.CurrentLocationColumnFamily.STR_NAME;
 +      case FUTURE:
 +        return TabletsSection.FutureLocationColumnFamily.STR_NAME;
 +      case LAST:
 +        return TabletsSection.LastLocationColumnFamily.STR_NAME;
 +      default:
 +        throw new IllegalArgumentException();
 +    }
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator putLocation(Ample.TServer tsi, LocationType type) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    mutation.put(getLocationFamily(type), tsi.getSession(), tsi.getLocation().toString());
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator deleteLocation(Ample.TServer tsi, LocationType type) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    mutation.putDelete(getLocationFamily(type), tsi.getSession());
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator putZooLock(ZooLock zooLock) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    TabletsSection.ServerColumnFamily.LOCK_COLUMN.put(mutation,
 +        new Value(zooLock.getLockID().serialize(context.getZooKeeperRoot() + "/")));
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator putWal(LogEntry logEntry) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    mutation.put(logEntry.getColumnFamily(), logEntry.getColumnQualifier(), logEntry.getValue());
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator deleteWal(LogEntry logEntry) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    mutation.putDelete(logEntry.getColumnFamily(), logEntry.getColumnQualifier());
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator deleteWal(String wal) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    mutation.putDelete(MetadataSchema.TabletsSection.LogColumnFamily.STR_NAME, wal);
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator putBulkFile(Ample.FileMeta bulkref, long tid) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    mutation.put(TabletsSection.BulkFileColumnFamily.NAME, bulkref.meta(),
-         new Value(Long.toString(tid)));
++        new Value(FateTxId.formatTid(tid)));
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator deleteBulkFile(Ample.FileMeta bulkref) {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    mutation.putDelete(TabletsSection.BulkFileColumnFamily.NAME, bulkref.meta());
 +    return this;
 +  }
 +
 +  @Override
 +  public Ample.TabletMutator putChopped() {
 +    Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
 +    TabletsSection.ChoppedColumnFamily.CHOPPED_COLUMN.put(mutation, new Value("chopped"));
 +    return this;
 +  }
 +
 +  protected Mutation getMutation() {
 +    updatesEnabled = false;
 +    return mutation;
 +  }
 +
 +  public void setCloseAfterMutate(AutoCloseable closeable) {
 +    this.closeAfterMutate = closeable;
 +  }
 +}
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index da72619,8576bf7..18d50ca
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@@ -81,7 -74,11 +80,8 @@@ import org.apache.accumulo.core.tablets
  import org.apache.accumulo.core.util.ColumnFQ;
  import org.apache.accumulo.core.util.FastFormat;
  import org.apache.accumulo.core.util.Pair;
+ import org.apache.accumulo.fate.FateTxId;
 -import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
  import org.apache.accumulo.fate.zookeeper.ZooLock;
 -import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 -import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
  import org.apache.accumulo.server.ServerConstants;
  import org.apache.accumulo.server.ServerContext;
  import org.apache.accumulo.server.fs.FileRef;
@@@ -715,12 -922,23 +715,24 @@@ public class MetadataTableUtil 
    }
  
    public static void chopped(ServerContext context, KeyExtent extent, ZooLock zooLock) {
 -    Mutation m = new Mutation(extent.getMetadataEntry());
 -    ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("chopped".getBytes(UTF_8)));
 -    update(context, zooLock, m, extent);
 +    TabletMutator tablet = context.getAmple().mutateTablet(extent);
 +    tablet.putChopped();
 +    tablet.putZooLock(zooLock);
 +    tablet.mutate();
    }
  
+   public static long getBulkLoadTid(Value v) {
+     String vs = v.toString();
+ 
+     if (FateTxId.isFormatedTid(vs)) {
+       return FateTxId.fromString(vs);
+     } else {
+       // a new serialization format was introduce in 2.0. This code support deserializing
the old
+       // format.
+       return Long.parseLong(vs);
+     }
+   }
+ 
    public static void removeBulkLoadEntries(AccumuloClient client, TableId tableId, long
tid)
        throws Exception {
      try (


Mime
View raw message