Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 46FCC18FC8 for ; Mon, 29 Feb 2016 22:29:35 +0000 (UTC) Received: (qmail 32311 invoked by uid 500); 29 Feb 2016 22:29:35 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 32224 invoked by uid 500); 29 Feb 2016 22:29:34 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 32215 invoked by uid 99); 29 Feb 2016 22:29:34 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 29 Feb 2016 22:29:34 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8A922E788B; Mon, 29 Feb 2016 22:29:34 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Message-Id: <0818247e763c48028f43ccdd7aa6c7e6@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-9853. Ozone: Add container definitions. Contributed by Anu Engineer. Date: Mon, 29 Feb 2016 22:29:34 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/HDFS-7240 1b7df69d3 -> f3fbae8b8 HDFS-9853. Ozone: Add container definitions. Contributed by Anu Engineer. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3fbae8b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3fbae8b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3fbae8b Branch: refs/heads/HDFS-7240 Commit: f3fbae8b84d4a9dcd9713d379e46fd2348c745df Parents: 1b7df69 Author: Anu Engineer Authored: Mon Feb 29 14:26:49 2016 -0800 Committer: Anu Engineer Committed: Mon Feb 29 14:26:49 2016 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 + .../org/apache/hadoop/ozone/package-info.java | 35 +++ .../main/proto/DatanodeContainerProtocol.proto | 314 +++++++++++++++++++ 3 files changed, 350 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3fbae8b/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index f1d5af9..4fc1588 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -351,6 +351,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> QJournalProtocol.proto editlog.proto fsimage.proto + DatanodeContainerProtocol.proto ${project.build.directory}/generated-sources/java http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3fbae8b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java new file mode 100644 index 0000000..db399db --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone; + +/** + This package contains class that support ozone implementation on the datanode + side. + + Main parts of ozone on datanode are: + + 1. REST Interface - This code lives under the web directory and listens to the + WebHDFS port. + + 2. Datanode container classes: This support persistence of ozone objects on + datanode. These classes live under container directory. + + 3. Client and Shell: We also support a ozone REST client lib, they are under + web/client and web/ozShell. + + */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3fbae8b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeContainerProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeContainerProtocol.proto new file mode 100644 index 0000000..0fba636 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeContainerProtocol.proto @@ -0,0 +1,314 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * These .proto interfaces are private and Unstable. + * Please see http://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/InterfaceClassification.html + * for what changes are allowed for a *Unstable* .proto interface. + */ + +// This file contains protocol buffers that are used to transfer data +// to and from the datanode. +option java_package = "org.apache.hadoop.hdfs.ozone.protocol.proto"; +option java_outer_classname = "ContainerProtos"; +option java_generate_equals_and_hash = true; +package hadoop.hdfs.ozone; +import "hdfs.proto"; + +/** + * Commands that are used to manipulate the state of containers on a datanode. + * + * These commands allow us to work against the datanode - from + * StorageContainer Manager as well as clients. + * + * 1. CreateContainer - This call is usually made by Storage Container + * manager, when we need to create a new container on a given datanode. + * + * 2. ReadContainer - Allows end user to stat a container. For example + * this allows us to return the metadata of a container. + * + * 3. UpdateContainer - Updates a container metadata. + + * 4. DeleteContainer - This call is made to delete a container. + * + * 5. ListContainer - Returns the list of containers on this + * datanode. This will be used by tests and tools. + * + * 6. CreateKey - Given a valid container, creates a key. + * + * 7. ReadKey - Allows user to read the metadata of a Key. + * + * 8. UpdateKey - Updates the metadata of a Key. + * + * 9. DeleteKey - Deletes a given key. + * + * 10. ListKey - Returns a list of keys that are present inside + * a given container. + * + * 11. ReadChunk - Allows us to read a chunk. + * + * 12. DeleteChunk - Delete an unused chunk. + * + * 13. WriteChunk - Allows us to write a chunk + * + * 14. ListChunk - Given a Container/Key returns the list of Chunks. + * + */ + +enum Type { + CreateContainer = 1; + ReadContainer = 2; + UpdateContainer = 3; + DeleteContainer = 4; + ListContainer = 5; + + CreateKey = 6; + Readkey = 7; + UpdateKey = 8; + DeleteKey = 9; + ListKey = 10; + + ReadChunk = 11; + DeleteChunk = 12; + WriteChunk = 13; + ListChunk = 14; +} + + +message ContainerCommandRequestProto { + required Type cmdType = 1; // Type of the command + + // A string that identifies this command, we generate Trace ID in Ozone + // frontend and this allows us to trace that command all over ozone. + optional string traceID = 2; + + // One of the following command is available when the corresponding + // cmdType is set. At the protocol level we allow only + // one command in each packet. + // TODO : Upgrade to Protobuf 2.6 or later. + optional CreateContainerRequestProto createContainer = 3; + optional ReadContainerRequestProto readContainer = 4; + optional UpdateContainerRequestProto updateContainer = 5; + optional DeleteContainerRequestProto deleteContainer = 6; + optional ListContainerRequestProto listContainer = 7; + + optional CreateKeyRequestProto createKey = 8; + optional ReadKeyRequestProto readKey = 9; + optional UpdateKeyRequestProto updateKey = 10; + optional DeleteKeyRequestProto deleteKey = 11; + optional ListKeyRequestProto listKey = 12; + + optional ReadChunkRequestProto readChunk = 13; + optional WriteChunkRequestProto writeChunk = 14; + optional DeleteChunkRequestProto deleteChunk = 15; + optional ListChunkRequestProto listChunk = 16; +} + +message ContainerCommandResponseProto { + required Type cmdType = 1; + optional string traceID = 2; + + optional CreateContainerResponseProto createContainer = 3; + optional ReadContainerResponseProto readContainer = 4; + optional UpdateContainerResponseProto updateContainer = 5; + optional DeleteContainerResponseProto deleteContainer = 6; + optional ListContainerResponseProto listContainer = 7; + + optional CreateKeyResponseProto createKey = 8; + optional ReadKeyResponeProto readKey = 9; + optional UpdateKeyResponseProto updateKey = 10; + optional DeleteKeyResponeProto deleteKey = 11; + optional ListKeyResponeProto listKey = 12; + + optional WriteChunkReponseProto writeChunk = 13; + optional ReadChunkReponseProto readChunk = 14; + optional DeleteChunkResponseProto deleteChunk = 15; + optional ListChunkResponseProto listChunk = 16; + +} + +// A pipeline is composed of one or more datanodes that back a container. +message Pipeline { + required string leaderID = 1; + repeated DatanodeIDProto members = 2; + optional string containerName = 3; +} + +message KeyValue { + required string key = 1; + optional string value = 2; +} + +message ContainerData { + required string name = 1; + repeated KeyValue metadata = 2; + optional string containerPath = 3; +} + +// Container Messages. +message CreateContainerRequestProto { + required Pipeline pipeline = 1; + required ContainerData containerData = 2; +} + +message CreateContainerResponseProto { +} + +message ReadContainerRequestProto { + required Pipeline pipeline = 1; + required string name = 2; +} + +message ReadContainerResponseProto { + optional ContainerData containerData = 2; +} + +message UpdateContainerRequestProto { + required Pipeline pipeline = 1; + required ContainerData containerData = 2; +} + +message UpdateContainerResponseProto { +} + +message DeleteContainerRequestProto { + required Pipeline pipeline = 1; + required string name = 2; +} + +message DeleteContainerResponseProto { +} + +message ListContainerRequestProto { + required Pipeline pipeline = 1; + optional string prevKey = 2; // if this is not set query from start. + required uint32 count = 3; // Max Results to return +} + +message ListContainerResponseProto { + repeated ContainerData containerData = 1; +} + + +message ContainerKeyData { + optional string containerName = 1; + required string name = 2; + repeated KeyValue metadata = 3; +} + +// Key Messages. +message CreateKeyRequestProto { + required Pipeline pipeline = 1; + required ContainerKeyData containerKeyData = 2; +} + +message CreateKeyResponseProto { +} + +message ReadKeyRequestProto { + required Pipeline pipeline = 1; + required ContainerKeyData containerKeyData = 2; +} + +message ReadKeyResponeProto { + repeated KeyValue metadata = 1; + repeated chunkInfo chunkData = 2; +} + +message UpdateKeyRequestProto { + required Pipeline pipeline = 1; + required ContainerKeyData containerKeyData = 2; +} + +message UpdateKeyResponseProto { +} + + +message DeleteKeyRequestProto { + required Pipeline pipeline = 1; + required string name = 2; +} + +message DeleteKeyResponeProto { +} + +message ListKeyRequestProto { + required Pipeline pipeline = 1; + required string prevKey = 2; + required uint32 count = 3; +} + +message ListKeyResponeProto { + repeated ContainerKeyData containerKeyData = 1; +} + +// Chunk Operations + +message chunkInfo { + required uint64 offset = 1; + required uint64 len = 2; + optional uint64 checksum = 3; + repeated KeyValue metadata = 4; +} + +message WriteChunkRequestProto { + required Pipeline pipeline = 1; + required string containerName = 2; + required string keyName = 3; + required chunkInfo chunkData = 4; + repeated bytes data = 5; +} + +message WriteChunkReponseProto { +} + +message ReadChunkRequestProto { + required Pipeline pipeline = 1; + required string containerName = 2; + required string keyName = 3; + required chunkInfo chunkData = 4; +} + +message ReadChunkReponseProto { + required Pipeline pipeline = 1; + required chunkInfo chunkData = 2; + repeated bytes data = 3; +} + +message DeleteChunkRequestProto { + required Pipeline pipeline = 1; + required string containerName = 2; + required string keyName = 3; + required chunkInfo chunkData = 4; +} + +message DeleteChunkResponseProto { +} + +message ListChunkRequestProto { + required Pipeline pipeline = 1; + required string containerName = 2; + required string keyName = 3; + required uint64 startOffset = 4; + required uint32 count = 5; +} + +message ListChunkResponseProto { + repeated chunkInfo chunkData = 1; +} +