pulsar-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] nkurihar edited a comment on issue #3535: Broker stopped with a lot of LedgerFencedException
Date Fri, 08 Feb 2019 02:18:53 GMT
nkurihar edited a comment on issue #3535: Broker stopped with a lot of LedgerFencedException
URL: https://github.com/apache/pulsar/issues/3535#issuecomment-461667425
 
 
   We found that producing from a particular topic causes Broker's frequent GCs, and finally
ZK session will be disconnected.
   
   Configuration of the problematic namespace:
   ```
   $ pulsar-admin namespaces policies tenant/global/ns
   {
     "auth_policies" : {
       "namespace_auth" : {
         “role1" : [ "consume", "produce" ],
         “role2" : [ "produce" ]
       },
       "destination_auth" : { }
     },
     "replication_clusters" : [ “cluster1" ],
     "bundles" : {
       "boundaries" : [ "0x00000000", "0x08000000", "0x0c000000", "0x10000000", "0x20000000",
"0x40000000", "0x60000000", "0x70000000", "0x78000000", "0x7c000000", "0x80000000", "0x90000000",
"0x94000000", "0x98000000", "0xa0000000", "0xc0000000", "0xdfffffff", "0xe3ffffff", "0xe7ffffff",
"0xefffffff", "0xffffffff" ],
       "numBundles" : 20
     },
     "backlog_quota_map" : {
       "destination_storage" : {
         "limit" : 2147483648,
         "policy" : "consumer_backlog_eviction"
       }
     },
     "clusterDispatchRate" : { },
     "deduplicationEnabled" : true,
     "latency_stats_sample_rate" : { },
     "message_ttl_in_seconds" : 86400,
     "retention_policies" : {
       "retentionTimeInMinutes" : 0,
       "retentionSizeInMB" : 0
     },
     "deleted" : false,
     "encryption_required" : false,
     "subscription_auth_mode" : "None"
   }
   ```
   
   Stats of the topic:
   ```
   $ pulsar-admin persistent partitioned-stats persistent://tenant/global/ns/topicname
   {
     "msgRateIn" : 13749.405145692821,
     "msgThroughputIn" : 2624357.272642204,
     "msgRateOut" : 0.0,
     "msgThroughputOut" : 0.0,
     "averageMsgSize" : 1526.9644796817477,
     "storageSize" : 16578083978,
     "publishers" : [ {
       "msgRateIn" : 916.2981245138371,
       "msgThroughputIn" : 174944.0752240065,
       "averageMsgSize" : 1524.0,
       "producerId" : 0
     }, {
       "msgRateIn" : 915.2981268948027,
       "msgThroughputIn" : 174631.82621486855,
       "averageMsgSize" : 1523.0,
       "producerId" : 0
     }, {
       "msgRateIn" : 916.7314553536784,
       "msgThroughputIn" : 174855.3587149024,
       "averageMsgSize" : 1523.0,
       "producerId" : 0
     }, {
       "msgRateIn" : 917.2147881797507,
       "msgThroughputIn" : 175129.79147638264,
       "averageMsgSize" : 1523.0,
       "producerId" : 0
     }, {
       "msgRateIn" : 916.1981230730042,
       "msgThroughputIn" : 175036.92487912314,
       "averageMsgSize" : 1525.0,
       "producerId" : 0
     }, {
       "msgRateIn" : 915.0814587846285,
       "msgThroughputIn" : 174646.9091628056,
       "averageMsgSize" : 1523.0,
       "producerId" : 0
     }, {
       "msgRateIn" : 918.5981173215638,
       "msgThroughputIn" : 175195.25776490584,
       "averageMsgSize" : 1521.0,
       "producerId" : 0
     }, {
       "msgRateIn" : 916.2481204650497,
       "msgThroughputIn" : 174752.4751328054,
       "averageMsgSize" : 1521.0,
       "producerId" : 0
     }, {
       "msgRateIn" : 917.6814516742087,
       "msgThroughputIn" : 175117.62447102348,
       "averageMsgSize" : 1522.0,
       "producerId" : 0
     }, {
       "msgRateIn" : 917.7647840932245,
       "msgThroughputIn" : 175091.40764492872,
       "averageMsgSize" : 1522.0,
       "producerId" : 0
     }, {
       "msgRateIn" : 915.7481213786745,
       "msgThroughputIn" : 174828.69157140085,
       "averageMsgSize" : 1523.0,
       "producerId" : 0
     }, {
       "msgRateIn" : 915.8814529493387,
       "msgThroughputIn" : 174970.22430954664,
       "averageMsgSize" : 1525.0,
       "producerId" : 0
     }, {
       "msgRateIn" : 916.4647860260764,
       "msgThroughputIn" : 174818.74155270297,
       "averageMsgSize" : 1522.0,
       "producerId" : 0
     }, {
       "msgRateIn" : 916.8647852416252,
       "msgThroughputIn" : 175094.0242297361,
       "averageMsgSize" : 1524.0,
       "producerId" : 0
     }, {
       "msgRateIn" : 917.3314497433578,
       "msgThroughputIn" : 175243.94029306492,
       "averageMsgSize" : 1524.0,
       "producerId" : 0
     } ],
     "subscriptions" : {
       “sub1" : {
         "msgRateOut" : 0.0,
         "msgThroughputOut" : 0.0,
         "msgRateRedeliver" : 0.0,
         "msgBacklog" : 86915729,
         "blockedSubscriptionOnUnackedMsgs" : false,
         "unackedMessages" : 0,
         "msgRateExpired" : 0.0,
         "consumers" : [ ]
       }
     },
     "replication" : { },
     "metadata" : {
       "partitions" : 8
     },
     "partitions" : { }
   } 
   ```
   Stats internal of an internal-topic
   ```
   $ sudo pulsar-admin persistent stats-internal persistent://tenant/global/ns/topicname-partition-0
   {
     "entriesAddedCounter" : 12087378,
     "numberOfEntries" : 11110595,
     "totalSize" : 2107489222,
     "currentLedgerEntries" : 669812,
     "currentLedgerSize" : 126853597,
     "lastLedgerCreatedTimestamp" : "2019-02-08T11:01:42.848+09:00",
     "waitingCursorsCount" : 0,
     "pendingAddEntriesCount" : 0,
     "lastConfirmedEntry" : "9287653:669811",
     "state" : "LedgerOpened",
     "ledgers" : [ {
       "ledgerId" : 9282667,
       "entries" : 935065,
       "size" : 177983818
     }, {
       "ledgerId" : 9282876,
       "entries" : 959774,
       "size" : 182570651
     }, {
       "ledgerId" : 9283069,
       "entries" : 980610,
       "size" : 186309626
     }, {
       "ledgerId" : 9283292,
       "entries" : 996760,
       "size" : 189755206
     }, {
       "ledgerId" : 9283585,
       "entries" : 1122265,
       "size" : 212600168
     }, {
       "ledgerId" : 9283938,
       "entries" : 1142511,
       "size" : 216121699
     }, {
       "ledgerId" : 9284223,
       "entries" : 1130209,
       "size" : 213851655
     }, {
       "ledgerId" : 9286911,
       "entries" : 1074060,
       "size" : 203609877
     }, {
       "ledgerId" : 9287187,
       "entries" : 1054774,
       "size" : 199907614
     }, {
       "ledgerId" : 9287401,
       "entries" : 1044755,
       "size" : 197925311
     }, {
       "ledgerId" : 9287653,
       "entries" : 0,
       "size" : 0
     } ],
     "cursors" : {
       "sub1" : {
         "markDeletePosition" : "9282667:265388",
         "readPosition" : "9282667:265389",
         "waitingReadOp" : false,
         "pendingReadOps" : 0,
         "messagesConsumedCounter" : 1242172,
         "cursorLedger" : 9282647,
         "cursorLedgerLastEntry" : 10,
         "individuallyDeletedMessages" : "[]",
         "lastLedgerSwitchTimestamp" : "2019-02-08T09:11:42.684+09:00",
         "state" : "Open",
         "numberOfEntriesSinceFirstNotAckedMessage" : 1,
         "totalNonContiguousDeletedMessagesRange" : 0,
         "properties" : { }
       },
       "pulsar.dedup" : {
         "markDeletePosition" : "9287653:669433",
         "readPosition" : "9287653:669434",
         "waitingReadOp" : false,
         "pendingReadOps" : 0,
         "messagesConsumedCounter" : 12087000,
         "cursorLedger" : 9282463,
         "cursorLedgerLastEntry" : 699,
         "individuallyDeletedMessages" : "[]",
         "lastLedgerSwitchTimestamp" : "2019-02-08T09:11:42.683+09:00",
         "state" : "Open",
         "numberOfEntriesSinceFirstNotAckedMessage" : 1,
         "totalNonContiguousDeletedMessagesRange" : 0,
         "properties" : {
           "cluster1-449-127492" : 60260034,
           "cluster1-449-127493" : 60262711,
           "cluster1-449-127495" : 60264335,
           "cluster1-449-127497" : 60261591,
           "cluster1-449-127499" : 60263472,
           "cluster1-449-127501" : 60262324,
           "cluster1-449-127503" : 60259581,
           "cluster1-449-127505" : 60265239,
           "cluster1-449-127507" : 60263064,
           "cluster1-449-127509" : 60263894,
           "cluster1-449-127511" : 60264359,
           "cluster1-449-127513" : 60263808,
           "cluster1-449-127515" : 60259626,
           "cluster1-449-127517" : 60263209,
           "cluster1-449-127519" : 60260679
         }
       }
     }
   }
   ```
   * the number of partition: 8
   *  deduplicationEnabled : true
   * message_ttl_in_seconds : 86400
   * There is no consumer. Thus the backlog is always full and messages are discarded 
    every time by the policy: "consumer_backlog_eviction"
   
   If you find any suspicious setting, please let us know.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message