bookkeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From si...@apache.org
Subject bookkeeper git commit: BOOKKEEPER-879: Record ledger creation time
Date Tue, 23 Feb 2016 18:16:11 GMT
Repository: bookkeeper
Updated Branches:
  refs/heads/master 9c937f5d8 -> bac3ed360


BOOKKEEPER-879: Record ledger creation time

Author: eolivelli <eolivelli@gmail.com>

Reviewers: Sijie Guo <sijie@apache.org>, Flavio Junqueira <fpj@apache.org>

Closes #2 from eolivelli/master and squashes the following commits:

509ad4f [eolivelli] BOOKKEEPER-879: Record ledger creation time
227d50c [eolivelli] BOOKKEEPER-879: Record ledger creation time
94cdee6 [eolivelli] BOOKKEEPER-879: Record ledger creation time
2a8e700 [eolivelli] BOOKKEEPER-879: Record ledger creation time
c43f0b4 [eolivelli] BOOKKEEPER-879: Record ledger creation time
e33f2e0 [eolivelli] BOOKKEEPER-879: Record ledger creation time


Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/bac3ed36
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/bac3ed36
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/bac3ed36

Branch: refs/heads/master
Commit: bac3ed3603f9dc9864beb610bea215c186ef4fab
Parents: 9c937f5
Author: eolivelli <eolivelli@gmail.com>
Authored: Tue Feb 23 10:16:06 2016 -0800
Committer: Sijie Guo <sijie@apache.org>
Committed: Tue Feb 23 10:16:06 2016 -0800

----------------------------------------------------------------------
 .../bookkeeper/client/LedgerMetadata.java       | 26 +++++-
 .../meta/AbstractZkLedgerManager.java           |  9 +-
 .../bookkeeper/meta/MSLedgerManagerFactory.java |  3 +-
 .../apache/bookkeeper/proto/DataFormats.java    | 86 ++++++++++++++++----
 .../src/main/proto/DataFormats.proto            |  2 +
 .../bookkeeper/client/ListLedgersTest.java      | 25 ++++++
 6 files changed, 131 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
index a20f34a..a58adba 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
@@ -37,6 +37,7 @@ import java.util.SortedMap;
 import java.util.TreeMap;
 
 import static com.google.common.base.Charsets.UTF_8;
+import com.google.common.base.Optional;
 
 /**
  * This class encapsulates all the ledger metadata that is persistently stored
@@ -66,6 +67,7 @@ public class LedgerMetadata {
     private int ackQuorumSize;
     private long length;
     private long lastEntryId;
+    private long ctime;
 
     private LedgerMetadataFormat.State state;
     private SortedMap<Long, ArrayList<BookieSocketAddress>> ensembles =
@@ -82,6 +84,7 @@ public class LedgerMetadata {
         this.ensembleSize = ensembleSize;
         this.writeQuorumSize = writeQuorumSize;
         this.ackQuorumSize = ackQuorumSize;
+        this.ctime = System.currentTimeMillis();
 
         /*
          * It is set in PendingReadOp.readEntryComplete, and
@@ -104,6 +107,7 @@ public class LedgerMetadata {
     LedgerMetadata(LedgerMetadata other) {
         this.ensembleSize = other.ensembleSize;
         this.writeQuorumSize = other.writeQuorumSize;
+        this.ctime = other.ctime;
         this.ackQuorumSize = other.ackQuorumSize;
         this.length = other.length;
         this.lastEntryId = other.lastEntryId;
@@ -150,6 +154,14 @@ public class LedgerMetadata {
         return writeQuorumSize;
     }
 
+    /**
+     * Get the creation timestamp of the ledger
+     * @return 
+     */
+    public long getCtime() {
+        return ctime;
+    }        
+
     public int getAckQuorumSize() {
         return ackQuorumSize;
     }
@@ -255,7 +267,7 @@ public class LedgerMetadata {
         LedgerMetadataFormat.Builder builder = LedgerMetadataFormat.newBuilder();
         builder.setQuorumSize(writeQuorumSize).setAckQuorumSize(ackQuorumSize)
             .setEnsembleSize(ensembleSize).setLength(length)
-            .setState(state).setLastEntryId(lastEntryId);
+            .setState(state).setLastEntryId(lastEntryId).setCtime(ctime);
 
         if (hasPassword) {
             builder.setDigestType(digestType).setPassword(ByteString.copyFrom(password));
@@ -308,11 +320,13 @@ public class LedgerMetadata {
      *            byte array to parse
      * @param version
      *            version of the ledger metadata
+     * @param msCtime
+     *            metadata store creation time, used for legacy ledgers
      * @return LedgerConfig
      * @throws IOException
      *             if the given byte[] cannot be parsed
      */
-    public static LedgerMetadata parseConfig(byte[] bytes, Version version) throws IOException
{
+    public static LedgerMetadata parseConfig(byte[] bytes, Version version, Optional<Long>
msCtime) throws IOException {
         LedgerMetadata lc = new LedgerMetadata();
         lc.version = version;
 
@@ -351,7 +365,12 @@ public class LedgerMetadata {
         LedgerMetadataFormat.Builder builder = LedgerMetadataFormat.newBuilder();
         TextFormat.merge(reader, builder);
         LedgerMetadataFormat data = builder.build();
-        lc.writeQuorumSize = data.getQuorumSize();
+        lc.writeQuorumSize = data.getQuorumSize();        
+        if (data.hasCtime()) {
+            lc.ctime = data.getCtime();
+        } else if (msCtime.isPresent()) {
+            lc.ctime = msCtime.get();
+        }        
         if (data.hasAckQuorumSize()) {
             lc.ackQuorumSize = data.getAckQuorumSize();
         } else {
@@ -463,6 +482,7 @@ public class LedgerMetadata {
         if (metadataFormatVersion != newMeta.metadataFormatVersion ||
             ensembleSize != newMeta.ensembleSize ||
             writeQuorumSize != newMeta.writeQuorumSize ||
+            ctime != newMeta.ctime ||
             ackQuorumSize != newMeta.ackQuorumSize ||
             length != newMeta.length ||
             state != newMeta.state ||

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java
index 1d7c591..fbf38ad 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java
@@ -17,6 +17,7 @@
  */
 package org.apache.bookkeeper.meta;
 
+import com.google.common.base.Optional;
 import java.io.IOException;
 import java.util.HashSet;
 import java.util.List;
@@ -355,10 +356,14 @@ abstract class AbstractZkLedgerManager implements LedgerManager, Watcher
{
                     readCb.operationComplete(BKException.Code.ZKException, null);
                     return;
                 }
-
+                if (stat == null) {
+                    LOG.error("Could not parse ledger metadata for ledger: " + ledgerId+".
Stat object is null");
+                    readCb.operationComplete(BKException.Code.ZKException, null);
+                    return;
+                }
                 LedgerMetadata metadata;
                 try {
-                    metadata = LedgerMetadata.parseConfig(data, new ZkVersion(stat.getVersion()));
+                    metadata = LedgerMetadata.parseConfig(data, new ZkVersion(stat.getVersion()),
Optional.of(stat.getCtime()));
                 } catch (IOException e) {
                     LOG.error("Could not parse ledger metadata for ledger: " + ledgerId,
e);
                     readCb.operationComplete(BKException.Code.ZKException, null);

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java
index 9f7ef38..8a82d41 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java
@@ -17,6 +17,7 @@
  */
 package org.apache.bookkeeper.meta;
 
+import com.google.common.base.Optional;
 import static org.apache.bookkeeper.metastore.MetastoreTable.ALL_FIELDS;
 import static org.apache.bookkeeper.metastore.MetastoreTable.NON_FIELDS;
 
@@ -408,7 +409,7 @@ public class MSLedgerManagerFactory extends LedgerManagerFactory {
                     LedgerMetadata metadata;
                     try {
                         metadata = LedgerMetadata
-                                .parseConfig(value.getValue().getField(META_FIELD), value.getVersion());
+                                .parseConfig(value.getValue().getField(META_FIELD), value.getVersion(),
Optional.<Long>absent());
                     } catch (IOException e) {
                         LOG.error("Could not parse ledger metadata for ledger " + ledgerId
+ " : ", e);
                         readCb.operationComplete(BKException.Code.MetaStoreException, null);

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java
index 65d5444..4552144 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java
@@ -52,6 +52,10 @@ public final class DataFormats {
     // optional int32 ackQuorumSize = 9;
     boolean hasAckQuorumSize();
     int getAckQuorumSize();
+    
+    // optional int64 ctime = 10;
+    boolean hasCtime();
+    long getCtime();
   }
   public static final class LedgerMetadataFormat extends
       com.google.protobuf.GeneratedMessage
@@ -781,6 +785,16 @@ public final class DataFormats {
       return ackQuorumSize_;
     }
     
+    // optional int64 ctime = 10;
+    public static final int CTIME_FIELD_NUMBER = 10;
+    private long ctime_;
+    public boolean hasCtime() {
+      return ((bitField0_ & 0x00000100) == 0x00000100);
+    }
+    public long getCtime() {
+      return ctime_;
+    }
+    
     private void initFields() {
       quorumSize_ = 0;
       ensembleSize_ = 0;
@@ -791,6 +805,7 @@ public final class DataFormats {
       digestType_ = org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.DigestType.CRC32;
       password_ = com.google.protobuf.ByteString.EMPTY;
       ackQuorumSize_ = 0;
+      ctime_ = 0L;
     }
     private byte memoizedIsInitialized = -1;
     public final boolean isInitialized() {
@@ -853,6 +868,9 @@ public final class DataFormats {
       if (((bitField0_ & 0x00000080) == 0x00000080)) {
         output.writeInt32(9, ackQuorumSize_);
       }
+      if (((bitField0_ & 0x00000100) == 0x00000100)) {
+        output.writeInt64(10, ctime_);
+      }
       getUnknownFields().writeTo(output);
     }
     
@@ -898,6 +916,10 @@ public final class DataFormats {
         size += com.google.protobuf.CodedOutputStream
           .computeInt32Size(9, ackQuorumSize_);
       }
+      if (((bitField0_ & 0x00000100) == 0x00000100)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(10, ctime_);
+      }
       size += getUnknownFields().getSerializedSize();
       memoizedSerializedSize = size;
       return size;
@@ -1045,6 +1067,8 @@ public final class DataFormats {
         bitField0_ = (bitField0_ & ~0x00000080);
         ackQuorumSize_ = 0;
         bitField0_ = (bitField0_ & ~0x00000100);
+        ctime_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000200);
         return this;
       }
       
@@ -1124,6 +1148,10 @@ public final class DataFormats {
           to_bitField0_ |= 0x00000080;
         }
         result.ackQuorumSize_ = ackQuorumSize_;
+        if (((from_bitField0_ & 0x00000200) == 0x00000200)) {
+          to_bitField0_ |= 0x00000100;
+        }
+        result.ctime_ = ctime_;
         result.bitField0_ = to_bitField0_;
         onBuilt();
         return result;
@@ -1190,6 +1218,9 @@ public final class DataFormats {
         if (other.hasAckQuorumSize()) {
           setAckQuorumSize(other.getAckQuorumSize());
         }
+        if (other.hasCtime()) {
+          setCtime(other.getCtime());
+        }
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
@@ -1301,6 +1332,11 @@ public final class DataFormats {
               ackQuorumSize_ = input.readInt32();
               break;
             }
+            case 80: {
+              bitField0_ |= 0x00000200;
+              ctime_ = input.readInt64();
+              break;
+            }
           }
         }
       }
@@ -1670,6 +1706,27 @@ public final class DataFormats {
         return this;
       }
       
+      // optional int64 ctime = 10;
+      private long ctime_ ;
+      public boolean hasCtime() {
+        return ((bitField0_ & 0x00000200) == 0x00000200);
+      }
+      public long getCtime() {
+        return ctime_;
+      }
+      public Builder setCtime(long value) {
+        bitField0_ |= 0x00000200;
+        ctime_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearCtime() {
+        bitField0_ = (bitField0_ & ~0x00000200);
+        ctime_ = 0L;
+        onChanged();
+        return this;
+      }
+      
       // @@protoc_insertion_point(builder_scope:LedgerMetadataFormat)
     }
     
@@ -3993,7 +4050,7 @@ public final class DataFormats {
       descriptor;
   static {
     java.lang.String[] descriptorData = {
-      "\n src/main/proto/DataFormats.proto\"\262\003\n\024L" +
+      "\n src/main/proto/DataFormats.proto\"\301\003\n\024L" +
       "edgerMetadataFormat\022\022\n\nquorumSize\030\001 \002(\005\022" +
       "\024\n\014ensembleSize\030\002 \002(\005\022\016\n\006length\030\003 \002(\003\022\023\n"
+
       "\013lastEntryId\030\004 \001(\003\0220\n\005state\030\005 \002(\0162\033.Ledg" +
@@ -4001,18 +4058,19 @@ public final class DataFormats {
       "\006 \003(\0132\035.LedgerMetadataFormat.Segment\0224\n\n" +
       "digestType\030\007 \001(\0162 .LedgerMetadataFormat." +
       "DigestType\022\020\n\010password\030\010 \001(\014\022\025\n\rackQuoru" +
-      "mSize\030\t \001(\005\0327\n\007Segment\022\026\n\016ensembleMember" +
-      "\030\001 \003(\t\022\024\n\014firstEntryId\030\002 \002(\003\".\n\005State\022\010\n",
-      "\004OPEN\020\001\022\017\n\013IN_RECOVERY\020\002\022\n\n\006CLOSED\020\003\"!\n\n"
+
-      "DigestType\022\t\n\005CRC32\020\001\022\010\n\004HMAC\020\002\"@\n\037Ledge" +
-      "rRereplicationLayoutFormat\022\014\n\004type\030\001 \002(\t" +
-      "\022\017\n\007version\030\002 \002(\005\".\n\033UnderreplicatedLedg" +
-      "erFormat\022\017\n\007replica\030\001 \003(\t\"^\n\014CookieForma" +
-      "t\022\022\n\nbookieHost\030\001 \002(\t\022\022\n\njournalDir\030\002 \002(" +
-      "\t\022\022\n\nledgerDirs\030\003 \002(\t\022\022\n\ninstanceId\030\004 \001(" +
-      "\t\"\"\n\016LockDataFormat\022\020\n\010bookieId\030\001 \001(\t\"%\n" +
-      "\021AuditorVoteFormat\022\020\n\010bookieId\030\001 \001(\tB\037\n\033" +
-      "org.apache.bookkeeper.protoH\001"
+      "mSize\030\t \001(\005\022\r\n\005ctime\030\n \001(\003\0327\n\007Segment\022\026\n"
+
+      "\016ensembleMember\030\001 \003(\t\022\024\n\014firstEntryId\030\002 ",
+      "\002(\003\".\n\005State\022\010\n\004OPEN\020\001\022\017\n\013IN_RECOVERY\020\002\022"
+
+      "\n\n\006CLOSED\020\003\"!\n\nDigestType\022\t\n\005CRC32\020\001\022\010\n\004" +
+      "HMAC\020\002\"@\n\037LedgerRereplicationLayoutForma" +
+      "t\022\014\n\004type\030\001 \002(\t\022\017\n\007version\030\002 \002(\005\".\n\033Unde"
+
+      "rreplicatedLedgerFormat\022\017\n\007replica\030\001 \003(\t" +
+      "\"^\n\014CookieFormat\022\022\n\nbookieHost\030\001 \002(\t\022\022\n\n" +
+      "journalDir\030\002 \002(\t\022\022\n\nledgerDirs\030\003 \002(\t\022\022\n\n" +
+      "instanceId\030\004 \001(\t\"\"\n\016LockDataFormat\022\020\n\010bo" +
+      "okieId\030\001 \001(\t\"%\n\021AuditorVoteFormat\022\020\n\010boo" +
+      "kieId\030\001 \001(\tB\037\n\033org.apache.bookkeeper.pro",
+      "toH\001"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
       new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -4024,7 +4082,7 @@ public final class DataFormats {
           internal_static_LedgerMetadataFormat_fieldAccessorTable = new
             com.google.protobuf.GeneratedMessage.FieldAccessorTable(
               internal_static_LedgerMetadataFormat_descriptor,
-              new java.lang.String[] { "QuorumSize", "EnsembleSize", "Length", "LastEntryId",
"State", "Segment", "DigestType", "Password", "AckQuorumSize", },
+              new java.lang.String[] { "QuorumSize", "EnsembleSize", "Length", "LastEntryId",
"State", "Segment", "DigestType", "Password", "AckQuorumSize", "Ctime", },
               org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.class,
               org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.Builder.class);
           internal_static_LedgerMetadataFormat_Segment_descriptor =

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/proto/DataFormats.proto
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/proto/DataFormats.proto b/bookkeeper-server/src/main/proto/DataFormats.proto
index 67623ac..6d97c3a 100644
--- a/bookkeeper-server/src/main/proto/DataFormats.proto
+++ b/bookkeeper-server/src/main/proto/DataFormats.proto
@@ -48,6 +48,8 @@ message LedgerMetadataFormat {
     optional bytes password = 8;
 
     optional int32 ackQuorumSize = 9;
+    
+    optional int64 ctime = 10;
 }
 
 message LedgerRereplicationLayoutFormat {

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java
index d34c064..70ff6e0 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java
@@ -108,4 +108,29 @@ public class ListLedgersTest extends BaseTestCase {
         Assert.fail("Remove is not supported, we shouln't have reached this point");
 
     }
+    
+    @Test(timeout = 60000)
+    public void testCtimeRecorded()
+            throws Exception {
+
+        ClientConfiguration conf = new ClientConfiguration()
+                .setZkServers(zkUtil.getZooKeeperConnectString());
+
+        BookKeeper bkc = new BookKeeper(conf);
+
+        bkc.createLedger(digestType, "testPasswd".
+                getBytes()).close();
+
+        BookKeeperAdmin admin = new BookKeeperAdmin(zkUtil.
+                getZooKeeperConnectString());
+        Iterable<Long> iterable = admin.listLedgers();
+
+        for (Long lId : iterable) {
+            LedgerHandle ledger = bkc.openLedger(lId, digestType, "testPasswd".getBytes());
+            LedgerMetadata metaData = ledger.getLedgerMetadata();
+            Assert.assertTrue("ctime was not recorded", metaData.getCtime() > 0);
+        }
+
+    }
+    
 }


Mime
View raw message