Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 2D10A200CAE for ; Wed, 21 Jun 2017 17:00:55 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 2BD1C160BF9; Wed, 21 Jun 2017 15:00:55 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D826A160BF6 for ; Wed, 21 Jun 2017 17:00:52 +0200 (CEST) Received: (qmail 74855 invoked by uid 500); 21 Jun 2017 15:00:48 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 74646 invoked by uid 99); 21 Jun 2017 15:00:48 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 21 Jun 2017 15:00:48 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 943B7E93E1; Wed, 21 Jun 2017 15:00:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Wed, 21 Jun 2017 15:00:59 -0000 Message-Id: <19698fccc10348cfbbd8333a758ed04f@git.apache.org> In-Reply-To: <6bc4839171624c328aa0079232bd5edc@git.apache.org> References: <6bc4839171624c328aa0079232bd5edc@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [13/26] hbase-site git commit: Published site at 82d554e3783372cc6b05489452c815b57c06f6cd. archived-at: Wed, 21 Jun 2017 15:00:55 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/fdcfc8d5/devapidocs/src-html/org/apache/hadoop/hbase/quotas/QuotaTableUtil.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/quotas/QuotaTableUtil.html b/devapidocs/src-html/org/apache/hadoop/hbase/quotas/QuotaTableUtil.html index 3c6f9b8..7d3deb8 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/quotas/QuotaTableUtil.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/quotas/QuotaTableUtil.html @@ -39,731 +39,772 @@ 031import org.apache.commons.logging.Log; 032import org.apache.commons.logging.LogFactory; 033import org.apache.hadoop.hbase.Cell; -034import org.apache.hadoop.hbase.NamespaceDescriptor; -035import org.apache.hadoop.hbase.ServerName; -036import org.apache.hadoop.hbase.TableName; -037import org.apache.hadoop.hbase.classification.InterfaceAudience; -038import org.apache.hadoop.hbase.classification.InterfaceStability; -039import org.apache.hadoop.hbase.client.ClusterConnection; -040import org.apache.hadoop.hbase.client.Connection; -041import org.apache.hadoop.hbase.client.Get; -042import org.apache.hadoop.hbase.client.Put; -043import org.apache.hadoop.hbase.client.QuotaStatusCalls; -044import org.apache.hadoop.hbase.client.Result; -045import org.apache.hadoop.hbase.client.ResultScanner; -046import org.apache.hadoop.hbase.client.Scan; -047import org.apache.hadoop.hbase.client.Table; -048import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; -049import org.apache.hadoop.hbase.filter.CompareFilter; -050import org.apache.hadoop.hbase.filter.Filter; -051import org.apache.hadoop.hbase.filter.FilterList; -052import org.apache.hadoop.hbase.filter.QualifierFilter; -053import org.apache.hadoop.hbase.filter.RegexStringComparator; -054import org.apache.hadoop.hbase.filter.RowFilter; -055import org.apache.hadoop.hbase.protobuf.ProtobufMagic; -056import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString; -057import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException; -058import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations; -059import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -060import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; -061import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; -062import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse; -063import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse; -064import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse; -065import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse.TableQuotaSnapshot; -066import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.RegionSizes; -067import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; -068import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota; -069import org.apache.hadoop.hbase.util.Bytes; -070import org.apache.hadoop.hbase.util.Strings; -071 -072/** -073 * Helper class to interact with the quota table. -074 * <table> -075 * <tr><th>ROW-KEY</th><th>FAM/QUAL</th><th>DATA</th></tr> -076 * <tr><td>n.<namespace></td><td>q:s</td><td><global-quotas></td></tr> -077 * <tr><td>n.<namespace></td><td>u:p</td><td><namespace-quota policy></td></tr> -078 * <tr><td>n.<namespace></td><td>u:s</td><td><SpaceQuotaSnapshot></td></tr> -079 * <tr><td>t.<table></td><td>q:s</td><td><global-quotas></td></tr> -080 * <tr><td>t.<table></td><td>u:p</td><td><table-quota policy></td></tr> -081 * <tr><td>t.<table></td><td>u:ss.<snapshot name></td><td><SpaceQuotaSnapshot></td></tr> -082 * <tr><td>u.<user></td><td>q:s</td><td><global-quotas></td></tr> -083 * <tr><td>u.<user></td><td>q:s.<table></td><td><table-quotas></td></tr> -084 * <tr><td>u.<user></td><td>q:s.<ns></td><td><namespace-quotas></td></tr> -085 * </table -086 */ -087@InterfaceAudience.Private -088@InterfaceStability.Evolving -089public class QuotaTableUtil { -090 private static final Log LOG = LogFactory.getLog(QuotaTableUtil.class); -091 -092 /** System table for quotas */ -093 public static final TableName QUOTA_TABLE_NAME = -094 TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "quota"); -095 -096 protected static final byte[] QUOTA_FAMILY_INFO = Bytes.toBytes("q"); -097 protected static final byte[] QUOTA_FAMILY_USAGE = Bytes.toBytes("u"); -098 protected static final byte[] QUOTA_QUALIFIER_SETTINGS = Bytes.toBytes("s"); -099 protected static final byte[] QUOTA_QUALIFIER_SETTINGS_PREFIX = Bytes.toBytes("s."); -100 protected static final byte[] QUOTA_QUALIFIER_POLICY = Bytes.toBytes("p"); -101 protected static final byte[] QUOTA_SNAPSHOT_SIZE_QUALIFIER = Bytes.toBytes("ss"); -102 protected static final String QUOTA_POLICY_COLUMN = -103 Bytes.toString(QUOTA_FAMILY_USAGE) + ":" + Bytes.toString(QUOTA_QUALIFIER_POLICY); -104 protected static final byte[] QUOTA_USER_ROW_KEY_PREFIX = Bytes.toBytes("u."); -105 protected static final byte[] QUOTA_TABLE_ROW_KEY_PREFIX = Bytes.toBytes("t."); -106 protected static final byte[] QUOTA_NAMESPACE_ROW_KEY_PREFIX = Bytes.toBytes("n."); -107 -108 /* ========================================================================= -109 * Quota "settings" helpers -110 */ -111 public static Quotas getTableQuota(final Connection connection, final TableName table) -112 throws IOException { -113 return getQuotas(connection, getTableRowKey(table)); -114 } -115 -116 public static Quotas getNamespaceQuota(final Connection connection, final String namespace) -117 throws IOException { -118 return getQuotas(connection, getNamespaceRowKey(namespace)); -119 } -120 -121 public static Quotas getUserQuota(final Connection connection, final String user) -122 throws IOException { -123 return getQuotas(connection, getUserRowKey(user)); -124 } -125 -126 public static Quotas getUserQuota(final Connection connection, final String user, -127 final TableName table) throws IOException { -128 return getQuotas(connection, getUserRowKey(user), getSettingsQualifierForUserTable(table)); -129 } -130 -131 public static Quotas getUserQuota(final Connection connection, final String user, -132 final String namespace) throws IOException { -133 return getQuotas(connection, getUserRowKey(user), -134 getSettingsQualifierForUserNamespace(namespace)); -135 } -136 -137 private static Quotas getQuotas(final Connection connection, final byte[] rowKey) -138 throws IOException { -139 return getQuotas(connection, rowKey, QUOTA_QUALIFIER_SETTINGS); -140 } -141 -142 private static Quotas getQuotas(final Connection connection, final byte[] rowKey, -143 final byte[] qualifier) throws IOException { -144 Get get = new Get(rowKey); -145 get.addColumn(QUOTA_FAMILY_INFO, qualifier); -146 Result result = doGet(connection, get); -147 if (result.isEmpty()) { -148 return null; -149 } -150 return quotasFromData(result.getValue(QUOTA_FAMILY_INFO, qualifier)); -151 } -152 -153 public static Get makeGetForTableQuotas(final TableName table) { -154 Get get = new Get(getTableRowKey(table)); -155 get.addFamily(QUOTA_FAMILY_INFO); -156 return get; -157 } -158 -159 public static Get makeGetForNamespaceQuotas(final String namespace) { -160 Get get = new Get(getNamespaceRowKey(namespace)); -161 get.addFamily(QUOTA_FAMILY_INFO); -162 return get; -163 } -164 -165 public static Get makeGetForUserQuotas(final String user, final Iterable<TableName> tables, -166 final Iterable<String> namespaces) { -167 Get get = new Get(getUserRowKey(user)); -168 get.addColumn(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS); -169 for (final TableName table: tables) { -170 get.addColumn(QUOTA_FAMILY_INFO, getSettingsQualifierForUserTable(table)); -171 } -172 for (final String ns: namespaces) { -173 get.addColumn(QUOTA_FAMILY_INFO, getSettingsQualifierForUserNamespace(ns)); -174 } -175 return get; -176 } -177 -178 public static Scan makeScan(final QuotaFilter filter) { -179 Scan scan = new Scan(); -180 scan.addFamily(QUOTA_FAMILY_INFO); -181 if (filter != null && !filter.isNull()) { -182 scan.setFilter(makeFilter(filter)); -183 } -184 return scan; -185 } -186 -187 /** -188 * converts quotafilter to serializeable filterlists. -189 */ -190 public static Filter makeFilter(final QuotaFilter filter) { -191 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); -192 if (!Strings.isEmpty(filter.getUserFilter())) { -193 FilterList userFilters = new FilterList(FilterList.Operator.MUST_PASS_ONE); -194 boolean hasFilter = false; -195 -196 if (!Strings.isEmpty(filter.getNamespaceFilter())) { -197 FilterList nsFilters = new FilterList(FilterList.Operator.MUST_PASS_ALL); -198 nsFilters.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, -199 new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0))); -200 nsFilters.addFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, -201 new RegexStringComparator( -202 getSettingsQualifierRegexForUserNamespace(filter.getNamespaceFilter()), 0))); -203 userFilters.addFilter(nsFilters); -204 hasFilter = true; -205 } -206 if (!Strings.isEmpty(filter.getTableFilter())) { -207 FilterList tableFilters = new FilterList(FilterList.Operator.MUST_PASS_ALL); -208 tableFilters.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, -209 new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0))); -210 tableFilters.addFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, -211 new RegexStringComparator( -212 getSettingsQualifierRegexForUserTable(filter.getTableFilter()), 0))); -213 userFilters.addFilter(tableFilters); -214 hasFilter = true; -215 } -216 if (!hasFilter) { -217 userFilters.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, -218 new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0))); -219 } -220 -221 filterList.addFilter(userFilters); -222 } else if (!Strings.isEmpty(filter.getTableFilter())) { -223 filterList.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, -224 new RegexStringComparator(getTableRowKeyRegex(filter.getTableFilter()), 0))); -225 } else if (!Strings.isEmpty(filter.getNamespaceFilter())) { -226 filterList.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, -227 new RegexStringComparator(getNamespaceRowKeyRegex(filter.getNamespaceFilter()), 0))); -228 } -229 return filterList; -230 } -231 -232 /** -233 * Creates a {@link Scan} which returns only quota snapshots from the quota table. -234 */ -235 public static Scan makeQuotaSnapshotScan() { -236 return makeQuotaSnapshotScanForTable(null); -237 } -238 -239 /** -240 * Fetches all {@link SpaceQuotaSnapshot} objects from the {@code hbase:quota} table. -241 * -242 * @param conn The HBase connection -243 * @return A map of table names and their computed snapshot. -244 */ -245 public static Map<TableName,SpaceQuotaSnapshot> getSnapshots(Connection conn) throws IOException { -246 Map<TableName,SpaceQuotaSnapshot> snapshots = new HashMap<>(); -247 try (Table quotaTable = conn.getTable(QUOTA_TABLE_NAME); -248 ResultScanner rs = quotaTable.getScanner(makeQuotaSnapshotScan())) { -249 for (Result r : rs) { -250 extractQuotaSnapshot(r, snapshots); -251 } -252 } -253 return snapshots; -254 } -255 -256 /** -257 * Creates a {@link Scan} which returns only {@link SpaceQuotaSnapshot} from the quota table for a -258 * specific table. -259 * @param tn Optionally, a table name to limit the scan's rowkey space. Can be null. -260 */ -261 public static Scan makeQuotaSnapshotScanForTable(TableName tn) { -262 Scan s = new Scan(); -263 // Limit to "u:v" column -264 s.addColumn(QUOTA_FAMILY_USAGE, QUOTA_QUALIFIER_POLICY); -265 if (null == tn) { -266 s.setRowPrefixFilter(QUOTA_TABLE_ROW_KEY_PREFIX); -267 } else { -268 byte[] row = getTableRowKey(tn); -269 // Limit rowspace to the "t:" prefix -270 s.withStartRow(row, true).withStopRow(row, true); -271 } -272 return s; -273 } -274 -275 /** -276 * Extracts the {@link SpaceViolationPolicy} and {@link TableName} from the provided -277 * {@link Result} and adds them to the given {@link Map}. If the result does not contain -278 * the expected information or the serialized policy in the value is invalid, this method -279 * will throw an {@link IllegalArgumentException}. -280 * -281 * @param result A row from the quota table. -282 * @param snapshots A map of snapshots to add the result of this method into. -283 */ -284 public static void extractQuotaSnapshot( -285 Result result, Map<TableName,SpaceQuotaSnapshot> snapshots) { -286 byte[] row = Objects.requireNonNull(result).getRow(); -287 if (row == null) { -288 throw new IllegalArgumentException("Provided result had a null row"); -289 } -290 final TableName targetTableName = getTableFromRowKey(row); -291 Cell c = result.getColumnLatestCell(QUOTA_FAMILY_USAGE, QUOTA_QUALIFIER_POLICY); -292 if (c == null) { -293 throw new IllegalArgumentException("Result did not contain the expected column " -294 + QUOTA_POLICY_COLUMN + ", " + result.toString()); -295 } -296 ByteString buffer = UnsafeByteOperations.unsafeWrap( -297 c.getValueArray(), c.getValueOffset(), c.getValueLength()); -298 try { -299 QuotaProtos.SpaceQuotaSnapshot snapshot = QuotaProtos.SpaceQuotaSnapshot.parseFrom(buffer); -300 snapshots.put(targetTableName, SpaceQuotaSnapshot.toSpaceQuotaSnapshot(snapshot)); -301 } catch (InvalidProtocolBufferException e) { -302 throw new IllegalArgumentException( -303 "Result did not contain a valid SpaceQuota protocol buffer message", e); -304 } -305 } -306 -307 public static interface UserQuotasVisitor { -308 void visitUserQuotas(final String userName, final Quotas quotas) -309 throws IOException; -310 void visitUserQuotas(final String userName, final TableName table, final Quotas quotas) -311 throws IOException; -312 void visitUserQuotas(final String userName, final String namespace, final Quotas quotas) -313 throws IOException; -314 } -315 -316 public static interface TableQuotasVisitor { -317 void visitTableQuotas(final TableName tableName, final Quotas quotas) -318 throws IOException; -319 } -320 -321 public static interface NamespaceQuotasVisitor { -322 void visitNamespaceQuotas(final String namespace, final Quotas quotas) -323 throws IOException; -324 } -325 -326 public static interface QuotasVisitor extends UserQuotasVisitor, -327 TableQuotasVisitor, NamespaceQuotasVisitor { -328 } -329 -330 public static void parseResult(final Result result, final QuotasVisitor visitor) -331 throws IOException { -332 byte[] row = result.getRow(); -333 if (isNamespaceRowKey(row)) { -334 parseNamespaceResult(result, visitor); -335 } else if (isTableRowKey(row)) { -336 parseTableResult(result, visitor); -337 } else if (isUserRowKey(row)) { -338 parseUserResult(result, visitor); -339 } else { -340 LOG.warn("unexpected row-key: " + Bytes.toString(row)); -341 } -342 } -343 -344 public static void parseResultToCollection(final Result result, -345 Collection<QuotaSettings> quotaSettings) throws IOException { -346 -347 QuotaTableUtil.parseResult(result, new QuotaTableUtil.QuotasVisitor() { -348 @Override -349 public void visitUserQuotas(String userName, Quotas quotas) { -350 quotaSettings.addAll(QuotaSettingsFactory.fromUserQuotas(userName, quotas)); -351 } -352 -353 @Override -354 public void visitUserQuotas(String userName, TableName table, Quotas quotas) { -355 quotaSettings.addAll(QuotaSettingsFactory.fromUserQuotas(userName, table, quotas)); -356 } -357 -358 @Override -359 public void visitUserQuotas(String userName, String namespace, Quotas quotas) { -360 quotaSettings.addAll(QuotaSettingsFactory.fromUserQuotas(userName, namespace, quotas)); -361 } -362 -363 @Override -364 public void visitTableQuotas(TableName tableName, Quotas quotas) { -365 quotaSettings.addAll(QuotaSettingsFactory.fromTableQuotas(tableName, quotas)); -366 } -367 -368 @Override -369 public void visitNamespaceQuotas(String namespace, Quotas quotas) { -370 quotaSettings.addAll(QuotaSettingsFactory.fromNamespaceQuotas(namespace, quotas)); -371 } -372 }); -373 } -374 -375 public static void parseNamespaceResult(final Result result, -376 final NamespaceQuotasVisitor visitor) throws IOException { -377 String namespace = getNamespaceFromRowKey(result.getRow()); -378 parseNamespaceResult(namespace, result, visitor); -379 } -380 -381 protected static void parseNamespaceResult(final String namespace, final Result result, -382 final NamespaceQuotasVisitor visitor) throws IOException { -383 byte[] data = result.getValue(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS); -384 if (data != null) { -385 Quotas quotas = quotasFromData(data); -386 visitor.visitNamespaceQuotas(namespace, quotas); -387 } -388 } -389 -390 public static void parseTableResult(final Result result, final TableQuotasVisitor visitor) -391 throws IOException { -392 TableName table = getTableFromRowKey(result.getRow()); -393 parseTableResult(table, result, visitor); -394 } -395 -396 protected static void parseTableResult(final TableName table, final Result result, -397 final TableQuotasVisitor visitor) throws IOException { -398 byte[] data = result.getValue(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS); -399 if (data != null) { -400 Quotas quotas = quotasFromData(data); -401 visitor.visitTableQuotas(table, quotas); -402 } -403 } -404 -405 public static void parseUserResult(final Result result, final UserQuotasVisitor visitor) -406 throws IOException { -407 String userName = getUserFromRowKey(result.getRow()); -408 parseUserResult(userName, result, visitor); -409 } -410 -411 protected static void parseUserResult(final String userName, final Result result, -412 final UserQuotasVisitor visitor) throws IOException { -413 Map<byte[], byte[]> familyMap = result.getFamilyMap(QUOTA_FAMILY_INFO); -414 if (familyMap == null || familyMap.isEmpty()) return; -415 -416 for (Map.Entry<byte[], byte[]> entry: familyMap.entrySet()) { -417 Quotas quotas = quotasFromData(entry.getValue()); -418 if (Bytes.startsWith(entry.getKey(), QUOTA_QUALIFIER_SETTINGS_PREFIX)) { -419 String name = Bytes.toString(entry.getKey(), QUOTA_QUALIFIER_SETTINGS_PREFIX.length); -420 if (name.charAt(name.length() - 1) == TableName.NAMESPACE_DELIM) { -421 String namespace = name.substring(0, name.length() - 1); -422 visitor.visitUserQuotas(userName, namespace, quotas); -423 } else { -424 TableName table = TableName.valueOf(name); -425 visitor.visitUserQuotas(userName, table, quotas); -426 } -427 } else if (Bytes.equals(entry.getKey(), QUOTA_QUALIFIER_SETTINGS)) { -428 visitor.visitUserQuotas(userName, quotas); -429 } -430 } -431 } -432 -433 /** -434 * Creates a {@link Put} to store the given {@code snapshot} for the given {@code tableName} in -435 * the quota table. -436 */ -437 static Put createPutForSpaceSnapshot(TableName tableName, SpaceQuotaSnapshot snapshot) { -438 Put p = new Put(getTableRowKey(tableName)); -439 p.addColumn( -440 QUOTA_FAMILY_USAGE, QUOTA_QUALIFIER_POLICY, -441 SpaceQuotaSnapshot.toProtoSnapshot(snapshot).toByteArray()); -442 return p; -443 } -444 -445 /** -446 * Creates a {@link Get} for the HBase snapshot's size against the given table. -447 */ -448 static Get makeGetForSnapshotSize(TableName tn, String snapshot) { -449 Get g = new Get(Bytes.add(QUOTA_TABLE_ROW_KEY_PREFIX, Bytes.toBytes(tn.toString()))); -450 g.addColumn( -451 QUOTA_FAMILY_USAGE, -452 Bytes.add(QUOTA_SNAPSHOT_SIZE_QUALIFIER, Bytes.toBytes(snapshot))); -453 return g; -454 } -455 -456 /** -457 * Creates a {@link Put} to persist the current size of the {@code snapshot} with respect to -458 * the given {@code table}. -459 */ -460 static Put createPutForSnapshotSize(TableName tableName, String snapshot, long size) { -461 // We just need a pb message with some `long usage`, so we can just reuse the -462 // SpaceQuotaSnapshot message instead of creating a new one. -463 Put p = new Put(getTableRowKey(tableName)); -464 p.addColumn(QUOTA_FAMILY_USAGE, getSnapshotSizeQualifier(snapshot), -465 org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaSnapshot -466 .newBuilder().setQuotaUsage(size).build().toByteArray()); -467 return p; -468 } -469 -470 /** -471 * Creates a {@code Put} for the namespace's total snapshot size. -472 */ -473 static Put createPutForNamespaceSnapshotSize(String namespace, long size) { -474 Put p = new Put(getNamespaceRowKey(namespace)); -475 p.addColumn(QUOTA_FAMILY_USAGE, QUOTA_SNAPSHOT_SIZE_QUALIFIER, -476 org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaSnapshot -477 .newBuilder().setQuotaUsage(size).build().toByteArray()); -478 return p; -479 } -480 -481 /** -482 * Fetches the computed size of all snapshots against tables in a namespace for space quotas. -483 */ -484 static long getNamespaceSnapshotSize( -485 Connection conn, String namespace) throws IOException { -486 try (Table quotaTable = conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) { -487 Result r = quotaTable.get(createGetNamespaceSnapshotSize(namespace)); -488 if (r.isEmpty()) { -489 return 0L; -490 } -491 r.advance(); -492 return parseSnapshotSize(r.current()); -493 } catch (InvalidProtocolBufferException e) { -494 throw new IOException("Could not parse snapshot size value for namespace " + namespace, e); -495 } -496 } -497 -498 /** -499 * Creates a {@code Get} to fetch the namespace's total snapshot size. -500 */ -501 static Get createGetNamespaceSnapshotSize(String namespace) { -502 Get g = new Get(getNamespaceRowKey(namespace)); -503 g.addColumn(QUOTA_FAMILY_USAGE, QUOTA_SNAPSHOT_SIZE_QUALIFIER); -504 return g; -505 } -506 -507 /** -508 * Parses the snapshot size from the given Cell's value. -509 */ -510 static long parseSnapshotSize(Cell c) throws InvalidProtocolBufferException { -511 ByteString bs = UnsafeByteOperations.unsafeWrap( -512 c.getValueArray(), c.getValueOffset(), c.getValueLength()); -513 return QuotaProtos.SpaceQuotaSnapshot.parseFrom(bs).getQuotaUsage(); -514 } -515 -516 static Scan createScanForSnapshotSizes(TableName table) { -517 byte[] rowkey = getTableRowKey(table); -518 return new Scan() -519 // Fetch just this one row -520 .withStartRow(rowkey) -521 .withStopRow(rowkey, true) -522 // Just the usage family -523 .addFamily(QUOTA_FAMILY_USAGE) -524 // Only the snapshot size qualifiers -525 .setFilter(new ColumnPrefixFilter(QUOTA_SNAPSHOT_SIZE_QUALIFIER)); -526 } -527 -528 /* ========================================================================= -529 * Space quota status RPC helpers -530 */ -531 /** -532 * Fetches the table sizes on the filesystem as tracked by the HBase Master. -533 */ -534 public static Map<TableName,Long> getMasterReportedTableSizes( -535 Connection conn) throws IOException { -536 if (!(conn instanceof ClusterConnection)) { -537 throw new IllegalArgumentException("Expected a ClusterConnection"); -538 } -539 ClusterConnection clusterConn = (ClusterConnection) conn; -540 GetSpaceQuotaRegionSizesResponse response = QuotaStatusCalls.getMasterRegionSizes( -541 clusterConn, 0); -542 Map<TableName,Long> tableSizes = new HashMap<>(); -543 for (RegionSizes sizes : response.getSizesList()) { -544 TableName tn = ProtobufUtil.toTableName(sizes.getTableName()); -545 tableSizes.put(tn, sizes.getSize()); -546 } -547 return tableSizes; -548 } -549 -550 /** -551 * Fetches the observed {@link SpaceQuotaSnapshot}s observed by a RegionServer. -552 */ -553 public static Map<TableName,SpaceQuotaSnapshot> getRegionServerQuotaSnapshots( -554 Connection conn, ServerName regionServer) throws IOException { -555 if (!(conn instanceof ClusterConnection)) { -556 throw new IllegalArgumentException("Expected a ClusterConnection"); -557 } -558 ClusterConnection clusterConn = (ClusterConnection) conn; -559 GetSpaceQuotaSnapshotsResponse response = QuotaStatusCalls.getRegionServerQuotaSnapshot( -560 clusterConn, 0, regionServer); -561 Map<TableName,SpaceQuotaSnapshot> snapshots = new HashMap<>(); -562 for (TableQuotaSnapshot snapshot : response.getSnapshotsList()) { -563 snapshots.put( -564 ProtobufUtil.toTableName(snapshot.getTableName()), -565 SpaceQuotaSnapshot.toSpaceQuotaSnapshot(snapshot.getSnapshot())); -566 } -567 return snapshots; -568 } -569 -570 /** -571 * Returns the Master's view of a quota on the given {@code tableName} or null if the -572 * Master has no quota information on that table. -573 */ -574 public static SpaceQuotaSnapshot getCurrentSnapshot( -575 Connection conn, TableName tn) throws IOException { -576 if (!(conn instanceof ClusterConnection)) { -577 throw new IllegalArgumentException("Expected a ClusterConnection"); -578 } -579 ClusterConnection clusterConn = (ClusterConnection) conn; -580 GetQuotaStatesResponse resp = QuotaStatusCalls.getMasterQuotaStates(clusterConn, 0); -581 HBaseProtos.TableName protoTableName = ProtobufUtil.toProtoTableName(tn); -582 for (GetQuotaStatesResponse.TableQuotaSnapshot tableSnapshot : resp.getTableSnapshotsList()) { -583 if (protoTableName.equals(tableSnapshot.getTableName())) { -584 return SpaceQuotaSnapshot.toSpaceQuotaSnapshot(tableSnapshot.getSnapshot()); -585 } -586 } -587 return null; -588 } -589 -590 /** -591 * Returns the Master's view of a quota on the given {@code namespace} or null if the -592 * Master has no quota information on that namespace. -593 */ -594 public static SpaceQuotaSnapshot getCurrentSnapshot( -595 Connection conn, String namespace) throws IOException { -596 if (!(conn instanceof ClusterConnection)) { -597 throw new IllegalArgumentException("Expected a ClusterConnection"); -598 } -599 ClusterConnection clusterConn = (ClusterConnection) conn; -600 GetQuotaStatesResponse resp = QuotaStatusCalls.getMasterQuotaStates(clusterConn, 0); -601 for (GetQuotaStatesResponse.NamespaceQuotaSnapshot nsSnapshot : resp.getNsSnapshotsList()) { -602 if (namespace.equals(nsSnapshot.getNamespace())) { -603 return SpaceQuotaSnapshot.toSpaceQuotaSnapshot(nsSnapshot.getSnapshot()); -604 } -605 } -606 return null; -607 } -608 -609 /* ========================================================================= -610 * Quotas protobuf helpers -611 */ -612 protected static Quotas quotasFromData(final byte[] data) throws IOException { -613 return quotasFromData(data, 0, data.length); -614 } -615 -616 protected static Quotas quotasFromData( -617 final byte[] data, int offset, int length) throws IOException { -618 int magicLen = ProtobufMagic.lengthOfPBMagic(); -619 if (!ProtobufMagic.isPBMagicPrefix(data, offset, magicLen)) { -620 throw new IOException("Missing pb magic prefix"); +034import org.apache.hadoop.hbase.CellScanner; +035import org.apache.hadoop.hbase.NamespaceDescriptor; +036import org.apache.hadoop.hbase.ServerName; +037import org.apache.hadoop.hbase.TableName; +038import org.apache.hadoop.hbase.classification.InterfaceAudience; +039import org.apache.hadoop.hbase.classification.InterfaceStability; +040import org.apache.hadoop.hbase.client.ClusterConnection; +041import org.apache.hadoop.hbase.client.Connection; +042import org.apache.hadoop.hbase.client.Get; +043import org.apache.hadoop.hbase.client.Put; +044import org.apache.hadoop.hbase.client.QuotaStatusCalls; +045import org.apache.hadoop.hbase.client.Result; +046import org.apache.hadoop.hbase.client.ResultScanner; +047import org.apache.hadoop.hbase.client.Scan; +048import org.apache.hadoop.hbase.client.Table; +049import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; +050import org.apache.hadoop.hbase.filter.CompareFilter; +051import org.apache.hadoop.hbase.filter.Filter; +052import org.apache.hadoop.hbase.filter.FilterList; +053import org.apache.hadoop.hbase.filter.QualifierFilter; +054import org.apache.hadoop.hbase.filter.RegexStringComparator; +055import org.apache.hadoop.hbase.filter.RowFilter; +056import org.apache.hadoop.hbase.protobuf.ProtobufMagic; +057import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString; +058import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException; +059import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations; +060import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +061import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; +062import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; +063import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse; +064import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse; +065import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse; +066import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse.TableQuotaSnapshot; +067import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.RegionSizes; +068import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; +069import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota; +070import org.apache.hadoop.hbase.util.Bytes; +071import org.apache.hadoop.hbase.util.Strings; +072 +073/** +074 * Helper class to interact with the quota table. +075 * <table> +076 * <tr><th>ROW-KEY</th><th>FAM/QUAL</th><th>DATA</th></tr> +077 * <tr><td>n.<namespace></td><td>q:s</td><td><global-quotas></td></tr> +078 * <tr><td>n.<namespace></td><td>u:p</td><td><namespace-quota policy></td></tr> +079 * <tr><td>n.<namespace></td><td>u:s</td><td><SpaceQuotaSnapshot></td></tr> +080 * <tr><td>t.<table></td><td>q:s</td><td><global-quotas></td></tr> +081 * <tr><td>t.<table></td><td>u:p</td><td><table-quota policy></td></tr> +082 * <tr><td>t.<table></td><td>u:ss.<snapshot name></td><td><SpaceQuotaSnapshot></td></tr> +083 * <tr><td>u.<user></td><td>q:s</td><td><global-quotas></td></tr> +084 * <tr><td>u.<user></td><td>q:s.<table></td><td><table-quotas></td></tr> +085 * <tr><td>u.<user></td><td>q:s.<ns></td><td><namespace-quotas></td></tr> +086 * </table +087 */ +088@InterfaceAudience.Private +089@InterfaceStability.Evolving +090public class QuotaTableUtil { +091 private static final Log LOG = LogFactory.getLog(QuotaTableUtil.class); +092 +093 /** System table for quotas */ +094 public static final TableName QUOTA_TABLE_NAME = +095 TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "quota"); +096 +097 protected static final byte[] QUOTA_FAMILY_INFO = Bytes.toBytes("q"); +098 protected static final byte[] QUOTA_FAMILY_USAGE = Bytes.toBytes("u"); +099 protected static final byte[] QUOTA_QUALIFIER_SETTINGS = Bytes.toBytes("s"); +100 protected static final byte[] QUOTA_QUALIFIER_SETTINGS_PREFIX = Bytes.toBytes("s."); +101 protected static final byte[] QUOTA_QUALIFIER_POLICY = Bytes.toBytes("p"); +102 protected static final byte[] QUOTA_SNAPSHOT_SIZE_QUALIFIER = Bytes.toBytes("ss"); +103 protected static final String QUOTA_POLICY_COLUMN = +104 Bytes.toString(QUOTA_FAMILY_USAGE) + ":" + Bytes.toString(QUOTA_QUALIFIER_POLICY); +105 protected static final byte[] QUOTA_USER_ROW_KEY_PREFIX = Bytes.toBytes("u."); +106 protected static final byte[] QUOTA_TABLE_ROW_KEY_PREFIX = Bytes.toBytes("t."); +107 protected static final byte[] QUOTA_NAMESPACE_ROW_KEY_PREFIX = Bytes.toBytes("n."); +108 +109 /* ========================================================================= +110 * Quota "settings" helpers +111 */ +112 public static Quotas getTableQuota(final Connection connection, final TableName table) +113 throws IOException { +114 return getQuotas(connection, getTableRowKey(table)); +115 } +116 +117 public static Quotas getNamespaceQuota(final Connection connection, final String namespace) +118 throws IOException { +119 return getQuotas(connection, getNamespaceRowKey(namespace)); +120 } +121 +122 public static Quotas getUserQuota(final Connection connection, final String user) +123 throws IOException { +124 return getQuotas(connection, getUserRowKey(user)); +125 } +126 +127 public static Quotas getUserQuota(final Connection connection, final String user, +128 final TableName table) throws IOException { +129 return getQuotas(connection, getUserRowKey(user), getSettingsQualifierForUserTable(table)); +130 } +131 +132 public static Quotas getUserQuota(final Connection connection, final String user, +133 final String namespace) throws IOException { +134 return getQuotas(connection, getUserRowKey(user), +135 getSettingsQualifierForUserNamespace(namespace)); +136 } +137 +138 private static Quotas getQuotas(final Connection connection, final byte[] rowKey) +139 throws IOException { +140 return getQuotas(connection, rowKey, QUOTA_QUALIFIER_SETTINGS); +141 } +142 +143 private static Quotas getQuotas(final Connection connection, final byte[] rowKey, +144 final byte[] qualifier) throws IOException { +145 Get get = new Get(rowKey); +146 get.addColumn(QUOTA_FAMILY_INFO, qualifier); +147 Result result = doGet(connection, get); +148 if (result.isEmpty()) { +149 return null; +150 } +151 return quotasFromData(result.getValue(QUOTA_FAMILY_INFO, qualifier)); +152 } +153 +154 public static Get makeGetForTableQuotas(final TableName table) { +155 Get get = new Get(getTableRowKey(table)); +156 get.addFamily(QUOTA_FAMILY_INFO); +157 return get; +158 } +159 +160 public static Get makeGetForNamespaceQuotas(final String namespace) { +161 Get get = new Get(getNamespaceRowKey(namespace)); +162 get.addFamily(QUOTA_FAMILY_INFO); +163 return get; +164 } +165 +166 public static Get makeGetForUserQuotas(final String user, final Iterable<TableName> tables, +167 final Iterable<String> namespaces) { +168 Get get = new Get(getUserRowKey(user)); +169 get.addColumn(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS); +170 for (final TableName table: tables) { +171 get.addColumn(QUOTA_FAMILY_INFO, getSettingsQualifierForUserTable(table)); +172 } +173 for (final String ns: namespaces) { +174 get.addColumn(QUOTA_FAMILY_INFO, getSettingsQualifierForUserNamespace(ns)); +175 } +176 return get; +177 } +178 +179 public static Scan makeScan(final QuotaFilter filter) { +180 Scan scan = new Scan(); +181 scan.addFamily(QUOTA_FAMILY_INFO); +182 if (filter != null && !filter.isNull()) { +183 scan.setFilter(makeFilter(filter)); +184 } +185 return scan; +186 } +187 +188 /** +189 * converts quotafilter to serializeable filterlists. +190 */ +191 public static Filter makeFilter(final QuotaFilter filter) { +192 FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); +193 if (!Strings.isEmpty(filter.getUserFilter())) { +194 FilterList userFilters = new FilterList(FilterList.Operator.MUST_PASS_ONE); +195 boolean hasFilter = false; +196 +197 if (!Strings.isEmpty(filter.getNamespaceFilter())) { +198 FilterList nsFilters = new FilterList(FilterList.Operator.MUST_PASS_ALL); +199 nsFilters.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, +200 new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0))); +201 nsFilters.addFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, +202 new RegexStringComparator( +203 getSettingsQualifierRegexForUserNamespace(filter.getNamespaceFilter()), 0))); +204 userFilters.addFilter(nsFilters); +205 hasFilter = true; +206 } +207 if (!Strings.isEmpty(filter.getTableFilter())) { +208 FilterList tableFilters = new FilterList(FilterList.Operator.MUST_PASS_ALL); +209 tableFilters.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, +210 new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0))); +211 tableFilters.addFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, +212 new RegexStringComparator( +213 getSettingsQualifierRegexForUserTable(filter.getTableFilter()), 0))); +214 userFilters.addFilter(tableFilters); +215 hasFilter = true; +216 } +217 if (!hasFilter) { +218 userFilters.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, +219 new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0))); +220 } +221 +222 filterList.addFilter(userFilters); +223 } else if (!Strings.isEmpty(filter.getTableFilter())) { +224 filterList.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, +225 new RegexStringComparator(getTableRowKeyRegex(filter.getTableFilter()), 0))); +226 } else if (!Strings.isEmpty(filter.getNamespaceFilter())) { +227 filterList.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, +228 new RegexStringComparator(getNamespaceRowKeyRegex(filter.getNamespaceFilter()), 0))); +229 } +230 return filterList; +231 } +232 +233 /** +234 * Creates a {@link Scan} which returns only quota snapshots from the quota table. +235 */ +236 public static Scan makeQuotaSnapshotScan() { +237 return makeQuotaSnapshotScanForTable(null); +238 } +239 +240 /** +241 * Fetches all {@link SpaceQuotaSnapshot} objects from the {@code hbase:quota} table. +242 * +243 * @param conn The HBase connection +244 * @return A map of table names and their computed snapshot. +245 */ +246 public static Map<TableName,SpaceQuotaSnapshot> getSnapshots(Connection conn) throws IOException { +247 Map<TableName,SpaceQuotaSnapshot> snapshots = new HashMap<>(); +248 try (Table quotaTable = conn.getTable(QUOTA_TABLE_NAME); +249 ResultScanner rs = quotaTable.getScanner(makeQuotaSnapshotScan())) { +250 for (Result r : rs) { +251 extractQuotaSnapshot(r, snapshots); +252 } +253 } +254 return snapshots; +255 } +256 +257 /** +258 * Creates a {@link Scan} which returns only {@link SpaceQuotaSnapshot} from the quota table for a +259 * specific table. +260 * @param tn Optionally, a table name to limit the scan's rowkey space. Can be null. +261 */ +262 public static Scan makeQuotaSnapshotScanForTable(TableName tn) { +263 Scan s = new Scan(); +264 // Limit to "u:v" column +265 s.addColumn(QUOTA_FAMILY_USAGE, QUOTA_QUALIFIER_POLICY); +266 if (null == tn) { +267 s.setRowPrefixFilter(QUOTA_TABLE_ROW_KEY_PREFIX); +268 } else { +269 byte[] row = getTableRowKey(tn); +270 // Limit rowspace to the "t:" prefix +271 s.withStartRow(row, true).withStopRow(row, true); +272 } +273 return s; +274 } +275 +276 /** +277 * Extracts the {@link SpaceViolationPolicy} and {@link TableName} from the provided +278 * {@link Result} and adds them to