Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id B626A200C10 for ; Fri, 20 Jan 2017 06:06:47 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id B4AB0160B5D; Fri, 20 Jan 2017 05:06:47 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 967AF160B5A for ; Fri, 20 Jan 2017 06:06:44 +0100 (CET) Received: (qmail 77767 invoked by uid 500); 20 Jan 2017 05:06:39 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 75241 invoked by uid 99); 20 Jan 2017 05:06:37 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 20 Jan 2017 05:06:37 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 09A18F4035; Fri, 20 Jan 2017 05:06:37 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sjlee@apache.org To: common-commits@hadoop.apache.org Date: Fri, 20 Jan 2017 05:06:54 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [19/28] hadoop git commit: YARN-5928. Move ATSv2 HBase backend code into a new module that is only dependent at runtime by yarn servers. Contributed by Haibo Chen. archived-at: Fri, 20 Jan 2017 05:06:47 -0000 http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java deleted file mode 100644 index e93b470..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java +++ /dev/null @@ -1,243 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.yarn.server.timelineservice.storage.common; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.Tag; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension; -import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation; -import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute; -import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -/** - * A bunch of utility functions used in HBase TimelineService backend. - */ -public final class HBaseTimelineStorageUtils { - /** milliseconds in one day. */ - public static final long MILLIS_ONE_DAY = 86400000L; - private static final Log LOG = - LogFactory.getLog(HBaseTimelineStorageUtils.class); - - private HBaseTimelineStorageUtils() { - } - - - /** - * Combines the input array of attributes and the input aggregation operation - * into a new array of attributes. - * - * @param attributes Attributes to be combined. - * @param aggOp Aggregation operation. - * @return array of combined attributes. - */ - public static Attribute[] combineAttributes(Attribute[] attributes, - AggregationOperation aggOp) { - int newLength = getNewLengthCombinedAttributes(attributes, aggOp); - Attribute[] combinedAttributes = new Attribute[newLength]; - - if (attributes != null) { - System.arraycopy(attributes, 0, combinedAttributes, 0, attributes.length); - } - - if (aggOp != null) { - Attribute a2 = aggOp.getAttribute(); - combinedAttributes[newLength - 1] = a2; - } - return combinedAttributes; - } - - /** - * Returns a number for the new array size. The new array is the combination - * of input array of attributes and the input aggregation operation. - * - * @param attributes Attributes. - * @param aggOp Aggregation operation. - * @return the size for the new array - */ - private static int getNewLengthCombinedAttributes(Attribute[] attributes, - AggregationOperation aggOp) { - int oldLength = getAttributesLength(attributes); - int aggLength = getAppOpLength(aggOp); - return oldLength + aggLength; - } - - private static int getAppOpLength(AggregationOperation aggOp) { - if (aggOp != null) { - return 1; - } - return 0; - } - - private static int getAttributesLength(Attribute[] attributes) { - if (attributes != null) { - return attributes.length; - } - return 0; - } - - /** - * Returns the first seen aggregation operation as seen in the list of input - * tags or null otherwise. - * - * @param tags list of HBase tags. - * @return AggregationOperation - */ - public static AggregationOperation getAggregationOperationFromTagsList( - List tags) { - for (AggregationOperation aggOp : AggregationOperation.values()) { - for (Tag tag : tags) { - if (tag.getType() == aggOp.getTagType()) { - return aggOp; - } - } - } - return null; - } - - /** - * Creates a {@link Tag} from the input attribute. - * - * @param attribute Attribute from which tag has to be fetched. - * @return a HBase Tag. - */ - public static Tag getTagFromAttribute(Map.Entry attribute) { - // attribute could be either an Aggregation Operation or - // an Aggregation Dimension - // Get the Tag type from either - AggregationOperation aggOp = AggregationOperation - .getAggregationOperation(attribute.getKey()); - if (aggOp != null) { - Tag t = new Tag(aggOp.getTagType(), attribute.getValue()); - return t; - } - - AggregationCompactionDimension aggCompactDim = - AggregationCompactionDimension.getAggregationCompactionDimension( - attribute.getKey()); - if (aggCompactDim != null) { - Tag t = new Tag(aggCompactDim.getTagType(), attribute.getValue()); - return t; - } - return null; - } - - /** - * creates a new cell based on the input cell but with the new value. - * - * @param origCell Original cell - * @param newValue new cell value - * @return cell - * @throws IOException while creating new cell. - */ - public static Cell createNewCell(Cell origCell, byte[] newValue) - throws IOException { - return CellUtil.createCell(CellUtil.cloneRow(origCell), - CellUtil.cloneFamily(origCell), CellUtil.cloneQualifier(origCell), - origCell.getTimestamp(), KeyValue.Type.Put.getCode(), newValue); - } - - /** - * creates a cell with the given inputs. - * - * @param row row of the cell to be created - * @param family column family name of the new cell - * @param qualifier qualifier for the new cell - * @param ts timestamp of the new cell - * @param newValue value of the new cell - * @param tags tags in the new cell - * @return cell - * @throws IOException while creating the cell. - */ - public static Cell createNewCell(byte[] row, byte[] family, byte[] qualifier, - long ts, byte[] newValue, byte[] tags) throws IOException { - return CellUtil.createCell(row, family, qualifier, ts, KeyValue.Type.Put, - newValue, tags); - } - - /** - * returns app id from the list of tags. - * - * @param tags cell tags to be looked into - * @return App Id as the AggregationCompactionDimension - */ - public static String getAggregationCompactionDimension(List tags) { - String appId = null; - for (Tag t : tags) { - if (AggregationCompactionDimension.APPLICATION_ID.getTagType() == t - .getType()) { - appId = Bytes.toString(t.getValue()); - return appId; - } - } - return appId; - } - - public static boolean isFlowRunTable(HRegionInfo hRegionInfo, - Configuration conf) { - String regionTableName = hRegionInfo.getTable().getNameAsString(); - String flowRunTableName = conf.get(FlowRunTable.TABLE_NAME_CONF_NAME, - FlowRunTable.DEFAULT_TABLE_NAME); - if (HBaseTimelineStorageUtils.LOG.isDebugEnabled()) { - HBaseTimelineStorageUtils.LOG.debug("regionTableName=" + regionTableName); - } - if (flowRunTableName.equalsIgnoreCase(regionTableName)) { - if (HBaseTimelineStorageUtils.LOG.isDebugEnabled()) { - HBaseTimelineStorageUtils.LOG.debug( - "table is the flow run table!! " + flowRunTableName); - } - return true; - } - return false; - } - - /** - * Converts an int into it's inverse int to be used in (row) keys - * where we want to have the largest int value in the top of the table - * (scans start at the largest int first). - * - * @param key value to be inverted so that the latest version will be first in - * a scan. - * @return inverted int - */ - public static int invertInt(int key) { - return Integer.MAX_VALUE - key; - } - - /** - * returns the timestamp of that day's start (which is midnight 00:00:00 AM) - * for a given input timestamp. - * - * @param ts Timestamp. - * @return timestamp of that day's beginning (midnight) - */ - public static long getTopOfTheDayTimestamp(long ts) { - long dayTimestamp = ts - (ts % MILLIS_ONE_DAY); - return dayTimestamp; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverter.java deleted file mode 100644 index 4229e81..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverter.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.timelineservice.storage.common; - -/** - * Interface which has to be implemented for encoding and decoding row keys and - * columns. - */ -public interface KeyConverter { - /** - * Encodes a key as a byte array. - * - * @param key key to be encoded. - * @return a byte array. - */ - byte[] encode(T key); - - /** - * Decodes a byte array and returns a key of type T. - * - * @param bytes byte representation - * @return an object(key) of type T which has been constructed after decoding - * the bytes. - */ - T decode(byte[] bytes); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java deleted file mode 100644 index 600601a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.timelineservice.storage.common; - -import java.io.IOException; -import java.io.Serializable; - -import org.apache.hadoop.hbase.util.Bytes; - -/** - * Encodes a value by interpreting it as a Long and converting it to bytes and - * decodes a set of bytes as a Long. - */ -public final class LongConverter implements NumericValueConverter, - Serializable { - - /** - * Added because we implement Comparator. - */ - private static final long serialVersionUID = 1L; - - public LongConverter() { - } - - @Override - public byte[] encodeValue(Object value) throws IOException { - if (!TimelineStorageUtils.isIntegralValue(value)) { - throw new IOException("Expected integral value"); - } - return Bytes.toBytes(((Number)value).longValue()); - } - - @Override - public Object decodeValue(byte[] bytes) throws IOException { - if (bytes == null) { - return null; - } - return Bytes.toLong(bytes); - } - - /** - * Compares two numbers as longs. If either number is null, it will be taken - * as 0. - * - * @param num1 the first {@code Long} to compare. - * @param num2 the second {@code Long} to compare. - * @return -1 if num1 is less than num2, 0 if num1 is equal to num2 and 1 if - * num1 is greater than num2. - */ - @Override - public int compare(Number num1, Number num2) { - return Long.compare((num1 == null) ? 0L : num1.longValue(), - (num2 == null) ? 0L : num2.longValue()); - } - - @Override - public Number add(Number num1, Number num2, Number...numbers) { - long sum = ((num1 == null) ? 0L : num1.longValue()) + - ((num2 == null) ? 0L : num2.longValue()); - for (Number num : numbers) { - sum = sum + ((num == null) ? 0L : num.longValue()); - } - return sum; - } - - /** - * Converts a timestamp into it's inverse timestamp to be used in (row) keys - * where we want to have the most recent timestamp in the top of the table - * (scans start at the most recent timestamp first). - * - * @param key value to be inverted so that the latest version will be first in - * a scan. - * @return inverted long - */ - public static long invertLong(long key) { - return Long.MAX_VALUE - key; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongKeyConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongKeyConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongKeyConverter.java deleted file mode 100644 index 4a724d6..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongKeyConverter.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.timelineservice.storage.common; - -import java.io.IOException; -/** - * Encodes and decodes column names / row keys which are long. - */ -public final class LongKeyConverter implements KeyConverter { - - /** - * To delegate the actual work to. - */ - private final LongConverter longConverter = new LongConverter(); - - public LongKeyConverter() { - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter - * #encode(java.lang.Object) - */ - @Override - public byte[] encode(Long key) { - try { - // IOException will not be thrown here as we are explicitly passing - // Long. - return longConverter.encodeValue(key); - } catch (IOException e) { - return null; - } - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter - * #decode(byte[]) - */ - @Override - public Long decode(byte[] bytes) { - try { - return (Long) longConverter.decodeValue(bytes); - } catch (IOException e) { - return null; - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java deleted file mode 100644 index 8fb6536..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.timelineservice.storage.common; - -import java.util.Comparator; - -/** - * Extends ValueConverter interface for numeric converters to support numerical - * operations such as comparison, addition, etc. - */ -public interface NumericValueConverter extends ValueConverter, - Comparator { - /** - * Adds two or more numbers. If either of the numbers are null, it is taken as - * 0. - * - * @param num1 the first number to add. - * @param num2 the second number to add. - * @param numbers Rest of the numbers to be added. - * @return result after adding up the numbers. - */ - Number add(Number num1, Number num2, Number...numbers); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java deleted file mode 100644 index 8a2e01a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.timelineservice.storage.common; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -/** - * Encapsulates a range with start and end indices. - */ -@InterfaceAudience.Private -@InterfaceStability.Unstable -public class Range { - private final int startIdx; - private final int endIdx; - - /** - * Defines a range from start index (inclusive) to end index (exclusive). - * - * @param start - * Starting index position - * @param end - * Ending index position (exclusive) - */ - public Range(int start, int end) { - if (start < 0 || end < start) { - throw new IllegalArgumentException( - "Invalid range, required that: 0 <= start <= end; start=" + start - + ", end=" + end); - } - - this.startIdx = start; - this.endIdx = end; - } - - public int start() { - return startIdx; - } - - public int end() { - return endIdx; - } - - public int length() { - return endIdx - startIdx; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/RowKeyPrefix.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/RowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/RowKeyPrefix.java deleted file mode 100644 index 6159dc7..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/RowKeyPrefix.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.timelineservice.storage.common; - -/** - * In queries where a single result is needed, an exact rowkey can be used - * through the corresponding rowkey#getRowKey() method. For queries that need to - * scan over a range of rowkeys, a partial (the initial part) of rowkeys are - * used. Classes implementing RowKeyPrefix indicate that they are the initial - * part of rowkeys, with different constructors with fewer number of argument to - * form a partial rowkey, a prefix. - * - * @param indicating the type of rowkey that a particular implementation is - * a prefix for. - */ -public interface RowKeyPrefix { - - /** - * Create a row key prefix, meaning a partial rowkey that can be used in range - * scans. Which fields are included in the prefix will depend on the - * constructor of the specific instance that was used. Output depends on which - * constructor was used. - * @return a prefix of the following form {@code fist!second!...!last!} - */ - byte[] getRowKeyPrefix(); - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java deleted file mode 100644 index 5090b4d..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java +++ /dev/null @@ -1,575 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.yarn.server.timelineservice.storage.common; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.hadoop.hbase.util.Bytes; - -/** - * Used to separate row qualifiers, column qualifiers and compound fields. - */ -public enum Separator { - - /** - * separator in key or column qualifier fields. - */ - QUALIFIERS("!", "%0$"), - - /** - * separator in values, and/or compound key/column qualifier fields. - */ - VALUES("=", "%1$"), - - /** - * separator in values, often used to avoid having these in qualifiers and - * names. Note that if we use HTML form encoding through URLEncoder, we end up - * getting a + for a space, which may already occur in strings, so we don't - * want that. - */ - SPACE(" ", "%2$"), - - /** - * separator in values, often used to avoid having these in qualifiers and - * names. - */ - TAB("\t", "%3$"); - - // a reserved character that starts each of the encoded values and is encoded - // first in order to escape naturally occurring instances of encoded values - // although it can be expressed as an enum instance, we define them as private - // variables to hide it from callers - private static final String PERCENT = "%"; - private static final String PERCENT_ENCODED = "%9$"; - - private static final Pattern PERCENT_PATTERN = - Pattern.compile(PERCENT, Pattern.LITERAL); - private static final String PERCENT_REPLACEMENT = - Matcher.quoteReplacement(PERCENT); - - private static final Pattern PERCENT_ENCODED_PATTERN = - Pattern.compile(PERCENT_ENCODED, Pattern.LITERAL); - private static final String PERCENT_ENCODED_REPLACEMENT = - Matcher.quoteReplacement(PERCENT_ENCODED); - - /** - * The string value of this separator. - */ - private final String value; - - /** - * The bye representation of value. - */ - private final byte[] bytes; - - // pre-compiled patterns and quoted replacements for optimization - private final Pattern valuePattern; - private final String valueReplacement; - - private final Pattern encodedValuePattern; - private final String encodedValueReplacement; - - /** - * Indicator for variable size of an individual segment in a split. The - * segment ends wherever separator is encountered. - * Typically used for string. - * Also used to indicate that there is no fixed number of splits which need to - * be returned. If split limit is specified as this, all possible splits are - * returned. - */ - public static final int VARIABLE_SIZE = 0; - - - /** empty string. */ - public static final String EMPTY_STRING = ""; - - /** empty bytes. */ - public static final byte[] EMPTY_BYTES = new byte[0]; - - /** - * @param value of the separator to use. Cannot be null or empty string. - * @param encodedValue choose something that isn't likely to occur in the data - * itself. Cannot be null or empty string. - */ - private Separator(String value, String encodedValue) { - this.value = value; - - // validation - if (value == null || value.length() == 0 || encodedValue == null - || encodedValue.length() == 0) { - throw new IllegalArgumentException( - "Cannot create separator from null or empty string."); - } - - this.bytes = Bytes.toBytes(value); - this.valuePattern = Pattern.compile(value, Pattern.LITERAL); - this.valueReplacement = Matcher.quoteReplacement(value); - - this.encodedValuePattern = Pattern.compile(encodedValue, Pattern.LITERAL); - this.encodedValueReplacement = Matcher.quoteReplacement(encodedValue); - } - - /** - * @return the original value of the separator - */ - public String getValue() { - return value; - } - - /** - * Used to make token safe to be used with this separator without collisions. - * It must be paired with {@link #decode(String)} for it to be - * decoded correctly. - *

- * If you need to encode a given string for multiple separators, - * {@link #encode(String, Separator...)} should be used over successive - * invocations of this method. It will result in a more compact version of the - * encoded value. - * - * @param token Token to be encoded. - * @return the token with any occurrences of this separator URLEncoded. - */ - public String encode(String token) { - if (token == null || token.length() == 0) { - // Nothing to replace - return token; - } - // first encode the percent to escape naturally occurring encoded values - String escaped = encodePercent(token); - return encodeSingle(escaped, this); - } - - private static String replace(String token, Pattern pattern, - String replacement) { - return pattern.matcher(token).replaceAll(replacement); - } - - private static String encodeSingle(String token, Separator separator) { - return replace(token, separator.valuePattern, - separator.encodedValueReplacement); - } - - private static String encodePercent(String token) { - return replace(token, PERCENT_PATTERN, PERCENT_ENCODED_REPLACEMENT); - } - - /** - * Decode the token encoded using {@link #encode(String)}. It must be - * used for the result encoded with {@link #encode(String)} to be able to - * recover the original. - * - * @param token Token to be decoded. - * @return the token with any occurrences of the encoded separator replaced by - * the separator itself. - */ - public String decode(String token) { - if (token == null || token.length() == 0) { - // Nothing to replace - return token; - } - String escaped = decodeSingle(token, this); - // decode percent to de-escape - return decodePercent(escaped); - } - - private static String decodeSingle(String token, Separator separator) { - return replace(token, separator.encodedValuePattern, - separator.valueReplacement); - } - - private static String decodePercent(String token) { - return replace(token, PERCENT_ENCODED_PATTERN, PERCENT_REPLACEMENT); - } - - /** - * Encode the given separators in the token with their encoding equivalents. - * It must be paired with {@link #decode(byte[], Separator...)} or - * {@link #decode(String, Separator...)} with the same separators for it to be - * decoded correctly. - *

- * If you need to encode a given string for multiple separators, this form of - * encoding should be used over successive invocations of - * {@link #encode(String)}. It will result in a more compact version of the - * encoded value. - * - * @param token containing possible separators that need to be encoded. - * @param separators to be encoded in the token with their URLEncoding - * equivalent. - * @return non-null byte representation of the token with occurrences of the - * separators encoded. - */ - public static byte[] encode(String token, Separator... separators) { - if (token == null || token.length() == 0) { - return EMPTY_BYTES; - } - String result = token; - // first encode the percent to escape naturally occurring encoded values - result = encodePercent(token); - for (Separator separator : separators) { - if (separator != null) { - result = encodeSingle(result, separator); - } - } - return Bytes.toBytes(result); - } - - /** - * Decode the given separators in the token with their decoding equivalents. - * It must be used for the result encoded with - * {@link #encode(String, Separator...)} with the same separators to be able - * to recover the original. - * - * @param token containing possible separators that need to be encoded. - * @param separators to be encoded in the token with their URLEncoding - * equivalent. - * @return String representation of the token with occurrences of the URL - * encoded separators decoded. - */ - public static String decode(byte[] token, Separator... separators) { - if (token == null) { - return null; - } - return decode(Bytes.toString(token), separators); - } - - /** - * Decode the given separators in the token with their decoding equivalents. - * It must be used for the result encoded with - * {@link #encode(String, Separator...)} with the same separators to be able - * to recover the original. - * - * @param token containing possible separators that need to be encoded. - * @param separators to be encoded in the token with their URLEncoding - * equivalent. - * @return String representation of the token with occurrences of the URL - * encoded separators decoded. - */ - public static String decode(String token, Separator... separators) { - if (token == null) { - return null; - } - String result = token; - for (Separator separator : separators) { - if (separator != null) { - result = decodeSingle(result, separator); - } - } - // decode percent to de-escape - return decodePercent(result); - } - - /** - * Returns a single byte array containing all of the individual arrays - * components separated by this separator. - * - * @param components Byte array components to be joined together. - * @return byte array after joining the components - */ - public byte[] join(byte[]... components) { - if (components == null || components.length == 0) { - return EMPTY_BYTES; - } - - int finalSize = 0; - finalSize = this.value.length() * (components.length - 1); - for (byte[] comp : components) { - if (comp != null) { - finalSize += comp.length; - } - } - - byte[] buf = new byte[finalSize]; - int offset = 0; - for (int i = 0; i < components.length; i++) { - if (components[i] != null) { - System.arraycopy(components[i], 0, buf, offset, components[i].length); - offset += components[i].length; - } - if (i < (components.length - 1)) { - System.arraycopy(this.bytes, 0, buf, offset, this.value.length()); - offset += this.value.length(); - } - } - return buf; - } - - /** - * Concatenates items (as String), using this separator. - * - * @param items Items join, {@code toString()} will be called in each item. - * Any occurrence of the separator in the individual strings will be - * first encoded. Cannot be null. - * @return non-null joined result. Note that when separator is {@literal null} - * the result is simply all items concatenated and the process is not - * reversible through {@link #splitEncoded(String)} - */ - public String joinEncoded(String... items) { - if (items == null || items.length == 0) { - return ""; - } - - StringBuilder sb = new StringBuilder(encode(items[0].toString())); - // Start at 1, we've already grabbed the first value at index 0 - for (int i = 1; i < items.length; i++) { - sb.append(this.value); - sb.append(encode(items[i].toString())); - } - - return sb.toString(); - } - - /** - * Concatenates items (as String), using this separator. - * - * @param items Items join, {@code toString()} will be called in each item. - * Any occurrence of the separator in the individual strings will be - * first encoded. Cannot be null. - * @return non-null joined result. Note that when separator is {@literal null} - * the result is simply all items concatenated and the process is not - * reversible through {@link #splitEncoded(String)} - */ - public String joinEncoded(Iterable items) { - if (items == null) { - return ""; - } - Iterator i = items.iterator(); - if (!i.hasNext()) { - return ""; - } - - StringBuilder sb = new StringBuilder(encode(i.next().toString())); - while (i.hasNext()) { - sb.append(this.value); - sb.append(encode(i.next().toString())); - } - - return sb.toString(); - } - - /** - * @param compoundValue containing individual values separated by this - * separator, which have that separator encoded. - * @return non-null set of values from the compoundValue with the separator - * decoded. - */ - public Collection splitEncoded(String compoundValue) { - List result = new ArrayList(); - if (compoundValue != null) { - for (String val : valuePattern.split(compoundValue)) { - result.add(decode(val)); - } - } - return result; - } - - /** - * Splits the source array into multiple array segments using this separator, - * up to a maximum of count items. This will naturally produce copied byte - * arrays for each of the split segments. - * - * @param source to be split - * @param limit on how many segments are supposed to be returned. A - * non-positive value indicates no limit on number of segments. - * @return source split by this separator. - */ - public byte[][] split(byte[] source, int limit) { - return split(source, this.bytes, limit); - } - - /** - * Splits the source array into multiple array segments using this separator. - * The sizes indicate the sizes of the relative components/segments. - * In case one of the segments contains this separator before the specified - * size is reached, the separator will be considered part of that segment and - * we will continue till size is reached. - * Variable length strings cannot contain this separator and are indiced with - * a size of {@value #VARIABLE_SIZE}. Such strings are encoded for this - * separator and decoded after the results from split is returned. - * - * @param source byte array to be split. - * @param sizes sizes of relative components/segments. - * @return source split by this separator as per the sizes specified.. - */ - public byte[][] split(byte[] source, int[] sizes) { - return split(source, this.bytes, sizes); - } - - /** - * Splits the source array into multiple array segments using this separator, - * as many times as splits are found. This will naturally produce copied byte - * arrays for each of the split segments. - * - * @param source byte array to be split - * @return source split by this separator. - */ - public byte[][] split(byte[] source) { - return split(source, this.bytes); - } - - /** - * Returns a list of ranges identifying [start, end) -- closed, open -- - * positions within the source byte array that would be split using the - * separator byte array. - * The sizes indicate the sizes of the relative components/segments. - * In case one of the segments contains this separator before the specified - * size is reached, the separator will be considered part of that segment and - * we will continue till size is reached. - * Variable length strings cannot contain this separator and are indiced with - * a size of {@value #VARIABLE_SIZE}. Such strings are encoded for this - * separator and decoded after the results from split is returned. - * - * @param source the source data - * @param separator the separator pattern to look for - * @param sizes indicate the sizes of the relative components/segments. - * @return a list of ranges. - */ - private static List splitRanges(byte[] source, byte[] separator, - int[] sizes) { - List segments = new ArrayList(); - if (source == null || separator == null) { - return segments; - } - // VARIABLE_SIZE here indicates that there is no limit to number of segments - // to return. - int limit = VARIABLE_SIZE; - if (sizes != null && sizes.length > 0) { - limit = sizes.length; - } - int start = 0; - int currentSegment = 0; - itersource: for (int i = 0; i < source.length; i++) { - for (int j = 0; j < separator.length; j++) { - if (source[i + j] != separator[j]) { - continue itersource; - } - } - // all separator elements matched - if (limit > VARIABLE_SIZE) { - if (segments.size() >= (limit - 1)) { - // everything else goes in one final segment - break; - } - if (sizes != null) { - int currentSegExpectedSize = sizes[currentSegment]; - if (currentSegExpectedSize > VARIABLE_SIZE) { - int currentSegSize = i - start; - if (currentSegSize < currentSegExpectedSize) { - // Segment not yet complete. More bytes to parse. - continue itersource; - } else if (currentSegSize > currentSegExpectedSize) { - // Segment is not as per size. - throw new IllegalArgumentException( - "Segments not separated as per expected sizes"); - } - } - } - } - segments.add(new Range(start, i)); - start = i + separator.length; - // i will be incremented again in outer for loop - i += separator.length - 1; - currentSegment++; - } - // add in remaining to a final range - if (start <= source.length) { - if (sizes != null) { - // Check if final segment is as per size specified. - if (sizes[currentSegment] > VARIABLE_SIZE && - source.length - start > sizes[currentSegment]) { - // Segment is not as per size. - throw new IllegalArgumentException( - "Segments not separated as per expected sizes"); - } - } - segments.add(new Range(start, source.length)); - } - return segments; - } - - /** - * Splits based on segments calculated based on limit/sizes specified for the - * separator. - * - * @param source byte array to be split. - * @param segments specifies the range for each segment. - * @return a byte[][] split as per the segment ranges. - */ - private static byte[][] split(byte[] source, List segments) { - byte[][] splits = new byte[segments.size()][]; - for (int i = 0; i < segments.size(); i++) { - Range r = segments.get(i); - byte[] tmp = new byte[r.length()]; - if (tmp.length > 0) { - System.arraycopy(source, r.start(), tmp, 0, r.length()); - } - splits[i] = tmp; - } - return splits; - } - - /** - * Splits the source array into multiple array segments using the given - * separator based on the sizes. This will naturally produce copied byte - * arrays for each of the split segments. - * - * @param source source array. - * @param separator separator represented as a byte array. - * @param sizes sizes of relative components/segments. - * @return byte[][] after splitting the source. - */ - private static byte[][] split(byte[] source, byte[] separator, int[] sizes) { - List segments = splitRanges(source, separator, sizes); - return split(source, segments); - } - - /** - * Splits the source array into multiple array segments using the given - * separator. This will naturally produce copied byte arrays for each of the - * split segments. - * - * @param source Source array. - * @param separator Separator represented as a byte array. - * @return byte[][] after splitting the source. - */ - private static byte[][] split(byte[] source, byte[] separator) { - return split(source, separator, (int[]) null); - } - - /** - * Splits the source array into multiple array segments using the given - * separator, up to a maximum of count items. This will naturally produce - * copied byte arrays for each of the split segments. - * - * @param source Source array. - * @param separator Separator represented as a byte array. - * @param limit a non-positive value indicates no limit on number of segments. - * @return byte[][] after splitting the input source. - */ - private static byte[][] split(byte[] source, byte[] separator, int limit) { - int[] sizes = null; - if (limit > VARIABLE_SIZE) { - sizes = new int[limit]; - } - List segments = splitRanges(source, separator, sizes); - return split(source, segments); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/StringKeyConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/StringKeyConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/StringKeyConverter.java deleted file mode 100644 index 282848e..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/StringKeyConverter.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.timelineservice.storage.common; - -/** - * Encodes and decodes column names / row keys which are merely strings. - * Column prefixes are not part of the column name passed for encoding. It is - * added later, if required in the associated ColumnPrefix implementations. - */ -public final class StringKeyConverter implements KeyConverter { - - public StringKeyConverter() { - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter - * #encode(java.lang.Object) - */ - @Override - public byte[] encode(String key) { - return Separator.encode(key, Separator.SPACE, Separator.TAB); - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter - * #decode(byte[]) - */ - @Override - public String decode(byte[] bytes) { - return Separator.decode(bytes, Separator.TAB, Separator.SPACE); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java deleted file mode 100644 index 8e6c259..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.timelineservice.storage.common; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.hbase.util.Bytes; - -/** - * contains the constants used in the context of schema accesses for - * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity} - * information. - */ -@InterfaceAudience.Private -@InterfaceStability.Unstable -public final class TimelineHBaseSchemaConstants { - private TimelineHBaseSchemaConstants() { - } - - /** - * Used to create a pre-split for tables starting with a username in the - * prefix. TODO: this may have to become a config variable (string with - * separators) so that different installations can presplit based on their own - * commonly occurring names. - */ - private final static byte[][] USERNAME_SPLITS = { - Bytes.toBytes("a"), Bytes.toBytes("ad"), Bytes.toBytes("an"), - Bytes.toBytes("b"), Bytes.toBytes("ca"), Bytes.toBytes("cl"), - Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"), - Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"), - Bytes.toBytes("j"), Bytes.toBytes("k"), Bytes.toBytes("l"), - Bytes.toBytes("m"), Bytes.toBytes("n"), Bytes.toBytes("o"), - Bytes.toBytes("q"), Bytes.toBytes("r"), Bytes.toBytes("s"), - Bytes.toBytes("se"), Bytes.toBytes("t"), Bytes.toBytes("u"), - Bytes.toBytes("v"), Bytes.toBytes("w"), Bytes.toBytes("x"), - Bytes.toBytes("y"), Bytes.toBytes("z") - }; - - /** - * The length at which keys auto-split. - */ - public static final String USERNAME_SPLIT_KEY_PREFIX_LENGTH = "4"; - - /** - * @return splits for splits where a user is a prefix. - */ - public static byte[][] getUsernameSplits() { - byte[][] kloon = USERNAME_SPLITS.clone(); - // Deep copy. - for (int row = 0; row < USERNAME_SPLITS.length; row++) { - kloon[row] = Bytes.copy(USERNAME_SPLITS[row]); - } - return kloon; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java deleted file mode 100644 index d03b37d..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.timelineservice.storage.common; - -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.hadoop.yarn.api.records.ApplicationId; - -/** - * Utility class that allows HBase coprocessors to interact with unique - * timestamps. - */ -public class TimestampGenerator { - - /* - * if this is changed, then reading cell timestamps written with older - * multiplier value will not work - */ - public static final long TS_MULTIPLIER = 1000000L; - - private final AtomicLong lastTimestamp = new AtomicLong(); - - /** - * Returns the current wall clock time in milliseconds, multiplied by the - * required precision. - * - * @return current timestamp. - */ - public long currentTime() { - // We want to align cell timestamps with current time. - // cell timestamps are not be less than - // System.currentTimeMillis() * TS_MULTIPLIER. - return System.currentTimeMillis() * TS_MULTIPLIER; - } - - /** - * Returns a timestamp value unique within the scope of this - * {@code TimestampGenerator} instance. For usage by HBase - * {@code RegionObserver} coprocessors, this normally means unique within a - * given region. - * - * Unlikely scenario of generating a non-unique timestamp: if there is a - * sustained rate of more than 1M hbase writes per second AND if region fails - * over within that time range of timestamps being generated then there may be - * collisions writing to a cell version of the same column. - * - * @return unique timestamp. - */ - public long getUniqueTimestamp() { - long lastTs; - long nextTs; - do { - lastTs = lastTimestamp.get(); - nextTs = Math.max(lastTs + 1, currentTime()); - } while (!lastTimestamp.compareAndSet(lastTs, nextTs)); - return nextTs; - } - - /** - * Returns a timestamp multiplied with TS_MULTIPLIER and last few digits of - * application id. - * - * Unlikely scenario of generating a timestamp that is a duplicate: If more - * than a 1M concurrent apps are running in one flow run AND write to same - * column at the same time, then say appId of 1M and 1 will overlap - * with appId of 001 and there may be collisions for that flow run's - * specific column. - * - * @param incomingTS Timestamp to be converted. - * @param appId Application Id. - * @return a timestamp multiplied with TS_MULTIPLIER and last few digits of - * application id - */ - public static long getSupplementedTimestamp(long incomingTS, String appId) { - long suffix = getAppIdSuffix(appId); - long outgoingTS = incomingTS * TS_MULTIPLIER + suffix; - return outgoingTS; - - } - - private static long getAppIdSuffix(String appIdStr) { - if (appIdStr == null) { - return 0L; - } - ApplicationId appId = ApplicationId.fromString(appIdStr); - long id = appId.getId() % TS_MULTIPLIER; - return id; - } - - /** - * truncates the last few digits of the timestamp which were supplemented by - * the TimestampGenerator#getSupplementedTimestamp function. - * - * @param incomingTS Timestamp to be truncated. - * @return a truncated timestamp value - */ - public static long getTruncatedTimestamp(long incomingTS) { - return incomingTS / TS_MULTIPLIER; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java deleted file mode 100644 index 64a11f8..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.timelineservice.storage.common; - -import org.apache.hadoop.hbase.client.BufferedMutator; - -/** - * Just a typed wrapper around {@link BufferedMutator} used to ensure that - * columns can write only to the table mutator for the right table. - */ -public interface TypedBufferedMutator extends BufferedMutator { - // This class is intentionally left (almost) blank -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java deleted file mode 100644 index 757a6d3..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.timelineservice.storage.common; - -import java.io.IOException; - -/** - * Converter used to encode/decode value associated with a column prefix or a - * column. - */ -public interface ValueConverter { - - /** - * Encode an object as a byte array depending on the converter implementation. - * - * @param value Value to be encoded. - * @return a byte array - * @throws IOException if any problem is encountered while encoding. - */ - byte[] encodeValue(Object value) throws IOException; - - /** - * Decode a byte array and convert it into an object depending on the - * converter implementation. - * - * @param bytes Byte array to be decoded. - * @return an object - * @throws IOException if any problem is encountered while decoding. - */ - Object decodeValue(byte[] bytes) throws IOException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java deleted file mode 100644 index 0df5b8a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Package org.apache.hadoop.yarn.server.timelineservice.storage.common contains - * a set of utility classes used across backend storage reader and writer. - */ -@InterfaceAudience.Private -@InterfaceStability.Unstable -package org.apache.hadoop.yarn.server.timelineservice.storage.common; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java deleted file mode 100644 index 93b4b36..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.timelineservice.storage.entity; - -import java.io.IOException; - -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter; -import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute; - -/** - * Identifies fully qualified columns for the {@link EntityTable}. - */ -public enum EntityColumn implements Column { - - /** - * Identifier for the entity. - */ - ID(EntityColumnFamily.INFO, "id"), - - /** - * The type of entity. - */ - TYPE(EntityColumnFamily.INFO, "type"), - - /** - * When the entity was created. - */ - CREATED_TIME(EntityColumnFamily.INFO, "created_time", new LongConverter()), - - /** - * The version of the flow that this entity belongs to. - */ - FLOW_VERSION(EntityColumnFamily.INFO, "flow_version"); - - private final ColumnHelper column; - private final ColumnFamily columnFamily; - private final String columnQualifier; - private final byte[] columnQualifierBytes; - - EntityColumn(ColumnFamily columnFamily, - String columnQualifier) { - this(columnFamily, columnQualifier, GenericConverter.getInstance()); - } - - EntityColumn(ColumnFamily columnFamily, - String columnQualifier, ValueConverter converter) { - this.columnFamily = columnFamily; - this.columnQualifier = columnQualifier; - // Future-proof by ensuring the right column prefix hygiene. - this.columnQualifierBytes = - Bytes.toBytes(Separator.SPACE.encode(columnQualifier)); - this.column = new ColumnHelper(columnFamily, converter); - } - - /** - * @return the column name value - */ - private String getColumnQualifier() { - return columnQualifier; - } - - public void store(byte[] rowKey, - TypedBufferedMutator tableMutator, Long timestamp, - Object inputValue, Attribute... attributes) throws IOException { - column.store(rowKey, tableMutator, columnQualifierBytes, timestamp, - inputValue, attributes); - } - - public Object readResult(Result result) throws IOException { - return column.readResult(result, columnQualifierBytes); - } - - /** - * Retrieve an {@link EntityColumn} given a name, or null if there is no - * match. The following holds true: {@code columnFor(x) == columnFor(y)} if - * and only if {@code x.equals(y)} or {@code (x == y == null)} - * - * @param columnQualifier Name of the column to retrieve - * @return the corresponding {@link EntityColumn} or null - */ - public static final EntityColumn columnFor(String columnQualifier) { - - // Match column based on value, assume column family matches. - for (EntityColumn ec : EntityColumn.values()) { - // Find a match based only on name. - if (ec.getColumnQualifier().equals(columnQualifier)) { - return ec; - } - } - - // Default to null - return null; - } - - @Override - public byte[] getColumnQualifierBytes() { - return columnQualifierBytes.clone(); - } - - @Override - public byte[] getColumnFamilyBytes() { - return columnFamily.getBytes(); - } - - @Override - public ValueConverter getValueConverter() { - return column.getValueConverter(); - } - - /** - * Retrieve an {@link EntityColumn} given a name, or null if there is no - * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)} - * if and only if {@code a.equals(b) & x.equals(y)} or - * {@code (x == y == null)} - * - * @param columnFamily The columnFamily for which to retrieve the column. - * @param name Name of the column to retrieve - * @return the corresponding {@link EntityColumn} or null if both arguments - * don't match. - */ - public static final EntityColumn columnFor(EntityColumnFamily columnFamily, - String name) { - - for (EntityColumn ec : EntityColumn.values()) { - // Find a match based column family and on name. - if (ec.columnFamily.equals(columnFamily) - && ec.getColumnQualifier().equals(name)) { - return ec; - } - } - - // Default to null - return null; - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java deleted file mode 100644 index 7c63727..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.timelineservice.storage.entity; - -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; - -/** - * Represents the entity table column families. - */ -public enum EntityColumnFamily implements ColumnFamily { - - /** - * Info column family houses known columns, specifically ones included in - * columnfamily filters. - */ - INFO("i"), - - /** - * Configurations are in a separate column family for two reasons: a) the size - * of the config values can be very large and b) we expect that config values - * are often separately accessed from other metrics and info columns. - */ - CONFIGS("c"), - - /** - * Metrics have a separate column family, because they have a separate TTL. - */ - METRICS("m"); - - /** - * Byte representation of this column family. - */ - private final byte[] bytes; - - /** - * @param value create a column family with this name. Must be lower case and - * without spaces. - */ - EntityColumnFamily(String value) { - // column families should be lower case and not contain any spaces. - this.bytes = Bytes.toBytes(Separator.SPACE.encode(value)); - } - - public byte[] getBytes() { - return Bytes.copy(bytes); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a925cb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java deleted file mode 100644 index e410549..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java +++ /dev/null @@ -1,300 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.timelineservice.storage.entity; - -import java.io.IOException; -import java.util.Map; -import java.util.NavigableMap; - -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter; -import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute; - -/** - * Identifies partially qualified columns for the entity table. - */ -public enum EntityColumnPrefix implements ColumnPrefix { - - /** - * To store TimelineEntity getIsRelatedToEntities values. - */ - IS_RELATED_TO(EntityColumnFamily.INFO, "s"), - - /** - * To store TimelineEntity getRelatesToEntities values. - */ - RELATES_TO(EntityColumnFamily.INFO, "r"), - - /** - * To store TimelineEntity info values. - */ - INFO(EntityColumnFamily.INFO, "i"), - - /** - * Lifecycle events for an entity. - */ - EVENT(EntityColumnFamily.INFO, "e", true), - - /** - * Config column stores configuration with config key as the column name. - */ - CONFIG(EntityColumnFamily.CONFIGS, null), - - /** - * Metrics are stored with the metric name as the column name. - */ - METRIC(EntityColumnFamily.METRICS, null, new LongConverter()); - - private final ColumnHelper column; - private final ColumnFamily columnFamily; - - /** - * Can be null for those cases where the provided column qualifier is the - * entire column name. - */ - private final String columnPrefix; - private final byte[] columnPrefixBytes; - - /** - * Private constructor, meant to be used by the enum definition. - * - * @param columnFamily that this column is stored in. - * @param columnPrefix for this column. - */ - EntityColumnPrefix(ColumnFamily columnFamily, - String columnPrefix) { - this(columnFamily, columnPrefix, false, GenericConverter.getInstance()); - } - - EntityColumnPrefix(ColumnFamily columnFamily, - String columnPrefix, boolean compondColQual) { - this(columnFamily, columnPrefix, compondColQual, - GenericConverter.getInstance()); - } - - EntityColumnPrefix(ColumnFamily columnFamily, - String columnPrefix, ValueConverter converter) { - this(columnFamily, columnPrefix, false, converter); - } - - /** - * Private constructor, meant to be used by the enum definition. - * - * @param columnFamily that this column is stored in. - * @param columnPrefix for this column. - * @param converter used to encode/decode values to be stored in HBase for - * this column prefix. - */ - EntityColumnPrefix(ColumnFamily columnFamily, - String columnPrefix, boolean compondColQual, ValueConverter converter) { - column = new ColumnHelper(columnFamily, converter); - this.columnFamily = columnFamily; - this.columnPrefix = columnPrefix; - if (columnPrefix == null) { - this.columnPrefixBytes = null; - } else { - // Future-proof by ensuring the right column prefix hygiene. - this.columnPrefixBytes = - Bytes.toBytes(Separator.SPACE.encode(columnPrefix)); - } - } - - /** - * @return the column name value - */ - public String getColumnPrefix() { - return columnPrefix; - } - - @Override - public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) { - return ColumnHelper.getColumnQualifier( - this.columnPrefixBytes, qualifierPrefix); - } - - @Override - public byte[] getColumnPrefixBytes(String qualifierPrefix) { - return ColumnHelper.getColumnQualifier( - this.columnPrefixBytes, qualifierPrefix); - } - - @Override - public byte[] getColumnFamilyBytes() { - return columnFamily.getBytes(); - } - - @Override - public ValueConverter getValueConverter() { - return column.getValueConverter(); - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix - * #store(byte[], - * org.apache.hadoop.yarn.server.timelineservice.storage.common. - * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object, - * org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute[]) - */ - public void store(byte[] rowKey, - TypedBufferedMutator tableMutator, String qualifier, - Long timestamp, Object inputValue, Attribute... attributes) - throws IOException { - - // Null check - if (qualifier == null) { - throw new IOException("Cannot store column with null qualifier in " - + tableMutator.getName().getNameAsString()); - } - - byte[] columnQualifier = getColumnPrefixBytes(qualifier); - - column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue, - attributes); - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix - * #store(byte[], - * org.apache.hadoop.yarn.server.timelineservice.storage.common. - * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object) - */ - public void store(byte[] rowKey, - TypedBufferedMutator tableMutator, byte[] qualifier, - Long timestamp, Object inputValue, Attribute... attributes) - throws IOException { - - // Null check - if (qualifier == null) { - throw new IOException("Cannot store column with null qualifier in " - + tableMutator.getName().getNameAsString()); - } - - byte[] columnQualifier = getColumnPrefixBytes(qualifier); - - column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue, - attributes); - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix - * #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String) - */ - public Object readResult(Result result, String qualifier) throws IOException { - byte[] columnQualifier = - ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier); - return column.readResult(result, columnQualifier); - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix - * #readResults(org.apache.hadoop.hbase.client.Result, - * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter) - */ - public Map readResults(Result result, - KeyConverter keyConverter) throws IOException { - return column.readResults(result, columnPrefixBytes, keyConverter); - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix - * #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result, - * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter) - */ - public NavigableMap> - readResultsWithTimestamps(Result result, KeyConverter keyConverter) - throws IOException { - return column.readResultsWithTimestamps(result, columnPrefixBytes, - keyConverter); - } - - /** - * Retrieve an {@link EntityColumnPrefix} given a name, or null if there is no - * match. The following holds true: {@code columnFor(x) == columnFor(y)} if - * and only if {@code x.equals(y)} or {@code (x == y == null)} - * - * @param columnPrefix Name of the column to retrieve - * @return the corresponding {@link EntityColumnPrefix} or null - */ - public static final EntityColumnPrefix columnFor(String columnPrefix) { - - // Match column based on value, assume column family matches. - for (EntityColumnPrefix ecp : EntityColumnPrefix.values()) { - // Find a match based only on name. - if (ecp.getColumnPrefix().equals(columnPrefix)) { - return ecp; - } - } - - // Default to null - return null; - } - - /** - * Retrieve an {@link EntityColumnPrefix} given a name, or null if there is no - * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)} - * if and only if {@code (x == y == null)} or - * {@code a.equals(b) & x.equals(y)} - * - * @param columnFamily The columnFamily for which to retrieve the column. - * @param columnPrefix Name of the column to retrieve - * @return the corresponding {@link EntityColumnPrefix} or null if both - * arguments don't match. - */ - public static final EntityColumnPrefix columnFor( - EntityColumnFamily columnFamily, String columnPrefix) { - - // TODO: needs unit test to confirm and need to update javadoc to explain - // null prefix case. - - for (EntityColumnPrefix ecp : EntityColumnPrefix.values()) { - // Find a match based column family and on name. - if (ecp.columnFamily.equals(columnFamily) - && (((columnPrefix == null) && (ecp.getColumnPrefix() == null)) || - (ecp.getColumnPrefix().equals(columnPrefix)))) { - return ecp; - } - } - - // Default to null - return null; - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org