Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id A53EF200C61 for ; Mon, 10 Apr 2017 16:15:58 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id A3BE8160BAA; Mon, 10 Apr 2017 14:15:58 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id AC7FC160B99 for ; Mon, 10 Apr 2017 16:15:56 +0200 (CEST) Received: (qmail 24266 invoked by uid 500); 10 Apr 2017 14:15:55 -0000 Mailing-List: contact commits-help@kylin.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kylin.apache.org Delivered-To: mailing list commits@kylin.apache.org Received: (qmail 24183 invoked by uid 99); 10 Apr 2017 14:15:55 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 10 Apr 2017 14:15:55 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A8C27E69B1; Mon, 10 Apr 2017 14:15:55 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: lidong@apache.org To: commits@kylin.apache.org Date: Mon, 10 Apr 2017 14:15:59 -0000 Message-Id: <3146cfbd0c464125a626a110aa52a78f@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [5/5] kylin git commit: KYLIN-1672 support kylin on cdh 5.7 archived-at: Mon, 10 Apr 2017 14:15:58 -0000 KYLIN-1672 support kylin on cdh 5.7 Signed-off-by: Li Yang Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f2450cb6 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f2450cb6 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f2450cb6 Branch: refs/heads/yang22-cdh5.7 Commit: f2450cb6b3d09f51c9c3215e5323a9b7d91e00cb Parents: de40194 Author: Lynne Jiang Authored: Mon May 16 03:33:27 2016 -0700 Committer: lidongsjtu Committed: Mon Apr 10 22:13:54 2017 +0800 ---------------------------------------------------------------------- dev-support/test_all_against_hdp_2_2_4_2_2.sh | 0 .../kylin/engine/mr/steps/MockupMapContext.java | 15 +- examples/test_case_data/sandbox/core-site.xml | 146 +++--- examples/test_case_data/sandbox/hbase-site.xml | 162 ++---- examples/test_case_data/sandbox/hdfs-site.xml | 259 ++-------- examples/test_case_data/sandbox/mapred-site.xml | 398 ++++++--------- examples/test_case_data/sandbox/yarn-site.xml | 496 ++----------------- pom.xml | 16 +- server/pom.xml | 36 ++ .../storage/hbase/steps/MockupMapContext.java | 19 +- tool/pom.xml | 12 + 11 files changed, 428 insertions(+), 1131 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f2450cb6/dev-support/test_all_against_hdp_2_2_4_2_2.sh ---------------------------------------------------------------------- diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh old mode 100644 new mode 100755 http://git-wip-us.apache.org/repos/asf/kylin/blob/f2450cb6/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java index 847071d..9900465 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java @@ -77,6 +77,7 @@ public class MockupMapContext { outKV[0] = key; outKV[1] = value; } + } @Override @@ -99,6 +100,7 @@ public class MockupMapContext { throw new NotImplementedException(); } + @Override public float getProgress() { throw new NotImplementedException(); @@ -195,17 +197,17 @@ public class MockupMapContext { } @Override - public RawComparator getSortComparator() { + public boolean userClassesTakesPrecedence() { throw new NotImplementedException(); } @Override - public String getJar() { + public RawComparator getSortComparator() { throw new NotImplementedException(); } @Override - public RawComparator getGroupingComparator() { + public String getJar() { throw new NotImplementedException(); } @@ -221,7 +223,7 @@ public class MockupMapContext { @Override public boolean getProfileEnabled() { - throw new NotImplementedException(); + return false; } @Override @@ -308,6 +310,11 @@ public class MockupMapContext { public RawComparator getCombinerKeyGroupingComparator() { throw new NotImplementedException(); } + + @Override + public RawComparator getGroupingComparator() { + return null; + } }); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/f2450cb6/examples/test_case_data/sandbox/core-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml index 9aa588c..6162406 100644 --- a/examples/test_case_data/sandbox/core-site.xml +++ b/examples/test_case_data/sandbox/core-site.xml @@ -14,152 +14,146 @@ See the License for the specific language governing permissions and limitations under the License. --> + - fs.defaultFS - hdfs://sandbox.hortonworks.com:8020 - true + hdfs://quickstart.cloudera:8020 - fs.trash.interval - 360 + 1 - - ha.failover-controller.active-standby-elector.zk.op.retries - 120 + io.compression.codecs + org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec - - hadoop.http.authentication.simple.anonymous.allowed - true + hadoop.security.authentication + simple - - hadoop.proxyuser.falcon.groups - users + hadoop.security.authorization + false + + + hadoop.rpc.protection + authentication + + + hadoop.security.auth_to_local + DEFAULT - - hadoop.proxyuser.falcon.hosts + hadoop.proxyuser.oozie.hosts * - - hadoop.proxyuser.hbase.groups - users + hadoop.proxyuser.oozie.groups + * - - hadoop.proxyuser.hbase.hosts + hadoop.proxyuser.mapred.hosts * - - hadoop.proxyuser.hcat.groups + hadoop.proxyuser.mapred.groups * - - hadoop.proxyuser.hcat.hosts - sandbox.hortonworks.com + hadoop.proxyuser.flume.hosts + * - - hadoop.proxyuser.hive.groups - users + hadoop.proxyuser.flume.groups + * + + + hadoop.proxyuser.HTTP.hosts + * + + + hadoop.proxyuser.HTTP.groups + * - hadoop.proxyuser.hive.hosts * - - hadoop.proxyuser.hue.groups + hadoop.proxyuser.hive.groups * - hadoop.proxyuser.hue.hosts * - - hadoop.proxyuser.oozie.groups + hadoop.proxyuser.hue.groups * - - hadoop.proxyuser.oozie.hosts - sandbox.hortonworks.com + hadoop.proxyuser.httpfs.hosts + * - - hadoop.proxyuser.root.groups + hadoop.proxyuser.httpfs.groups * - - hadoop.proxyuser.root.hosts + hadoop.proxyuser.hdfs.groups * - - hadoop.security.auth_to_local - DEFAULT + hadoop.proxyuser.hdfs.hosts + * - - hadoop.security.authentication - simple + hadoop.proxyuser.yarn.hosts + * - - hadoop.security.authorization - false + hadoop.proxyuser.yarn.groups + * - - io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec + hadoop.security.group.mapping + org.apache.hadoop.security.ShellBasedUnixGroupsMapping - - io.file.buffer.size - 131072 + hadoop.security.instrumentation.requires.admin + false - - io.serializations - org.apache.hadoop.io.serializer.WritableSerialization + net.topology.script.file.name + /etc/hadoop/conf.cloudera.yarn/topology.py - - ipc.client.connect.max.retries - 50 + io.file.buffer.size + 65536 - - ipc.client.connection.maxidletime - 30000 + hadoop.ssl.enabled + false + + + hadoop.ssl.require.client.cert + false + true - - ipc.client.idlethreshold - 8000 + hadoop.ssl.keystores.factory.class + org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory + true - - ipc.server.tcpnodelay - true + hadoop.ssl.server.conf + ssl-server.xml + true - - mapreduce.jobtracker.webinterface.trusted - false + hadoop.ssl.client.conf + ssl-client.xml + true - http://git-wip-us.apache.org/repos/asf/kylin/blob/f2450cb6/examples/test_case_data/sandbox/hbase-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml index 734908e..58c6223 100644 --- a/examples/test_case_data/sandbox/hbase-site.xml +++ b/examples/test_case_data/sandbox/hbase-site.xml @@ -15,180 +15,104 @@ limitations under the License. --> - - - dfs.domain.socket.path - /var/lib/hadoop-hdfs/dn_socket - - - hbase.client.keyvalue.maxsize - 10485760 - - - - hbase.client.scanner.caching - 100 + hbase.rootdir + hdfs://quickstart.cloudera:8020/hbase - - hbase.cluster.distributed + hbase.replication true - - - hbase.coprocessor.master.classes - com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor - - - hbase.coprocessor.region.classes - com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor + hbase.client.write.buffer + 2097152 - - - hbase.defaults.for.version.skip - true - - - hbase.hregion.majorcompaction - 604800000 + hbase.client.pause + 100 - - hbase.hregion.majorcompaction.jitter - 0.50 + hbase.client.retries.number + 35 - - hbase.hregion.max.filesize - 10737418240 + hbase.client.scanner.caching + 100 - - hbase.hregion.memstore.block.multiplier - 4 + hbase.client.keyvalue.maxsize + 10485760 - - hbase.hregion.memstore.flush.size - 134217728 - - - - hbase.hregion.memstore.mslab.enabled + hbase.ipc.client.allowsInterrupt true - - hbase.hstore.blockingStoreFiles + hbase.client.primaryCallTimeout.get 10 - - - hbase.hstore.compactionThreshold - 3 - - - hbase.local.dir - ${hbase.tmp.dir}/local + hbase.client.primaryCallTimeout.multiget + 10 - - hbase.master.info.bindAddress - 0.0.0.0 + hbase.regionserver.thrift.http + false - - hbase.master.info.port - 60010 + hbase.thrift.support.proxyuser + false - - hbase.master.port + hbase.rpc.timeout 60000 - - - hbase.regionserver.global.memstore.lowerLimit - 0.38 - - - hbase.regionserver.global.memstore.upperLimit - 0.4 - - - - hbase.regionserver.handler.count - 60 + hbase.snapshot.enabled + true - - hbase.regionserver.info.port - 60030 + hbase.snapshot.master.timeoutMillis + 60000 - - hbase.rootdir - hdfs://sandbox.hortonworks.com:8020/apps/hbase/data + hbase.snapshot.region.timeout + 60000 - - hbase.rpc.protection - PRIVACY + hbase.snapshot.master.timeout.millis + 60000 - hbase.security.authentication simple - - hbase.security.authorization - true + hbase.rpc.protection + authentication - - hbase.superuser - hbase + zookeeper.session.timeout + 60000 - - hbase.tmp.dir - /hadoop/hbase + zookeeper.znode.parent + /hbase - - hbase.zookeeper.property.clientPort - 2181 + zookeeper.znode.rootserver + root-region-server - hbase.zookeeper.quorum - sandbox.hortonworks.com + quickstart.cloudera - - hbase.zookeeper.useMulti - true - - - - hfile.block.cache.size - 0.40 - - - - zookeeper.session.timeout - 30000 + hbase.zookeeper.property.clientPort + 2181 - - zookeeper.znode.parent - /hbase-unsecure + hbase.rest.ssl.enabled + false - http://git-wip-us.apache.org/repos/asf/kylin/blob/f2450cb6/examples/test_case_data/sandbox/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml index 1175fff..05854bd 100644 --- a/examples/test_case_data/sandbox/hdfs-site.xml +++ b/examples/test_case_data/sandbox/hdfs-site.xml @@ -15,271 +15,68 @@ limitations under the License. --> - - - dfs.block.access.token.enable - false - - - - dfs.block.size - 34217472 - - - - dfs.blockreport.initialDelay - 120 - - - - dfs.blocksize - 134217728 - - - - dfs.client.read.shortcircuit - true - - - - dfs.client.read.shortcircuit.streams.cache.size - 4096 - - - - dfs.cluster.administrators - hdfs - - - - dfs.datanode.address - 0.0.0.0:50010 - - - - dfs.datanode.balance.bandwidthPerSec - 6250000 - - - - dfs.datanode.data.dir - /hadoop/hdfs/data - true - - - - dfs.datanode.data.dir.perm - 750 - - - dfs.datanode.du.reserved - 1073741824 - - - - dfs.datanode.failed.volumes.tolerated - 0 - true - - - - dfs.datanode.http.address - 0.0.0.0:50075 - - - - dfs.datanode.https.address - 0.0.0.0:50475 - - - - dfs.datanode.ipc.address - 0.0.0.0:8010 - - - - dfs.datanode.max.transfer.threads - 1024 - - - - dfs.datanode.max.xcievers - 1024 - - - - dfs.domain.socket.path - /var/lib/hadoop-hdfs/dn_socket - - - - dfs.heartbeat.interval - 3 + dfs.namenode.name.dir + file:///var/lib/hadoop-hdfs/cache/hdfs/dfs/name - - dfs.hosts.exclude - /etc/hadoop/conf/dfs.exclude + dfs.namenode.servicerpc-address + quickstart.cloudera:8022 - - dfs.http.policy - HTTP_ONLY + dfs.https.address + quickstart.cloudera:50470 - dfs.https.port 50470 - - - dfs.journalnode.edits.dir - /hadoop/hdfs/journalnode - - - - dfs.journalnode.http-address - 0.0.0.0:8480 - - - - dfs.journalnode.https-address - 0.0.0.0:8481 - - - - dfs.namenode.accesstime.precision - 3600000 - - - - dfs.namenode.avoid.read.stale.datanode - true - - - - dfs.namenode.avoid.write.stale.datanode - true - - - - dfs.namenode.checkpoint.dir - /hadoop/hdfs/namesecondary - - - - dfs.namenode.checkpoint.edits.dir - ${dfs.namenode.checkpoint.dir} - - - - dfs.namenode.checkpoint.period - 21600 - - - - dfs.namenode.checkpoint.txns - 1000000 - - - - dfs.namenode.handler.count - 100 - - dfs.namenode.http-address - sandbox.hortonworks.com:50070 - true - - - - dfs.namenode.https-address - sandbox.hortonworks.com:50470 - - - - dfs.namenode.name.dir - /hadoop/hdfs/namenode - true - - - - dfs.namenode.name.dir.restore - true - - - - dfs.namenode.safemode.threshold-pct - 1.0f + quickstart.cloudera:50070 - - dfs.namenode.secondary.http-address - sandbox.hortonworks.com:50090 + dfs.replication + 1 - - dfs.namenode.stale.datanode.interval - 30000 + dfs.blocksize + 134217728 - - dfs.namenode.startup.delay.block.deletion.sec - 3600 + dfs.client.use.datanode.hostname + false - - dfs.namenode.write.stale.datanode.ratio - 1.0f + fs.permissions.umask-mode + 022 - - dfs.nfs.exports.allowed.hosts - * rw + dfs.namenode.acls.enabled + false - - dfs.nfs3.dump.dir - /tmp/.hdfs-nfs + dfs.client.use.legacy.blockreader + false - - dfs.permissions.enabled - true - - - - dfs.permissions.superusergroup - hdfs + dfs.client.read.shortcircuit + false - - dfs.replication - 1 + dfs.domain.socket.path + /var/run/hdfs-sockets/dn - - dfs.replication.max - 50 + dfs.client.read.shortcircuit.skip.checksum + false - - dfs.support.append - true - true + dfs.client.domain.socket.data.traffic + false - - dfs.webhdfs.enabled + dfs.datanode.hdfs-blocks-metadata.enabled true - true - - - fs.permissions.umask-mode - 022 - - http://git-wip-us.apache.org/repos/asf/kylin/blob/f2450cb6/examples/test_case_data/sandbox/mapred-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml index e90f594..c9b1ca4 100644 --- a/examples/test_case_data/sandbox/mapred-site.xml +++ b/examples/test_case_data/sandbox/mapred-site.xml @@ -15,241 +15,165 @@ limitations under the License. --> - - - io.sort.mb - 128 - - - - mapred.child.java.opts - -Xmx200m - - - - mapreduce.map.memory.mb - 512 - - - - mapreduce.reduce.memory.mb - 512 - - - - mapreduce.admin.map.child.java.opts - -server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version} - - - - mapreduce.admin.reduce.child.java.opts - -server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version} - - - - mapreduce.admin.user.env - LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64 - - - - mapreduce.am.max-attempts - 2 - - - - mapreduce.application.classpath - /tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure - - - - mapreduce.application.framework.path - /hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework - - - - mapreduce.cluster.administrators - hadoop - - - - mapreduce.framework.name - yarn - - - - mapreduce.job.emit-timeline-data - false - - - - - mapreduce.job.reduce.slowstart.completedmaps - 1 - - - - mapreduce.jobhistory.address - sandbox.hortonworks.com:10020 - - - - mapreduce.jobhistory.bind-host - 0.0.0.0 - - - - mapreduce.jobhistory.done-dir - /mr-history/done - - - - mapreduce.jobhistory.intermediate-done-dir - /mr-history/tmp - - - - mapreduce.jobhistory.webapp.address - sandbox.hortonworks.com:19888 - - - - mapreduce.map.java.opts - -Xmx512m - - - - mapreduce.map.log.level - INFO - - - - mapreduce.map.memory.mb - 512 - - - - mapreduce.map.output.compress - false - - - - mapreduce.map.sort.spill.percent - 0.7 - - - - mapreduce.map.speculative - false - - - - mapreduce.output.fileoutputformat.compress - false - - - - mapreduce.output.fileoutputformat.compress.type - BLOCK - - - - mapreduce.reduce.input.buffer.percent - 0.0 - - - - mapreduce.reduce.java.opts - -Xmx200m - - - - mapreduce.reduce.log.level - INFO - - - - mapreduce.reduce.memory.mb - 512 - - - - mapreduce.reduce.shuffle.fetch.retry.enabled - 1 - - - - mapreduce.reduce.shuffle.fetch.retry.interval-ms - 1000 - - - - mapreduce.reduce.shuffle.fetch.retry.timeout-ms - 30000 - - - - mapreduce.reduce.shuffle.input.buffer.percent - 0.7 - - - - mapreduce.reduce.shuffle.merge.percent - 0.66 - - - - mapreduce.reduce.shuffle.parallelcopies - 30 - - - - mapreduce.reduce.speculative - false - - - - mapreduce.shuffle.port - 13562 - - - - mapreduce.task.io.sort.factor - 100 - - - - mapreduce.task.io.sort.mb - 128 - - - - mapreduce.task.timeout - 300000 - - - - yarn.app.mapreduce.am.admin-command-opts - -Dhdp.version=${hdp.version} - - - - yarn.app.mapreduce.am.command-opts - -Xmx512m - - - - yarn.app.mapreduce.am.log.level - INFO - - - - yarn.app.mapreduce.am.resource.mb - 512 - - - - yarn.app.mapreduce.am.staging-dir - /user - - + + mapreduce.job.split.metainfo.maxsize + 10000000 + + + mapreduce.job.counters.max + 120 + + + mapreduce.output.fileoutputformat.compress + false + + + mapreduce.output.fileoutputformat.compress.type + BLOCK + + + mapreduce.output.fileoutputformat.compress.codec + org.apache.hadoop.io.compress.DefaultCodec + + + mapreduce.map.output.compress.codec + org.apache.hadoop.io.compress.SnappyCodec + + + mapreduce.map.output.compress + true + + + zlib.compress.level + DEFAULT_COMPRESSION + + + mapreduce.task.io.sort.factor + 64 + + + mapreduce.map.sort.spill.percent + 0.8 + + + mapreduce.reduce.shuffle.parallelcopies + 10 + + + mapreduce.task.timeout + 600000 + + + mapreduce.client.submit.file.replication + 1 + + + mapreduce.job.reduces + 1 + + + mapreduce.task.io.sort.mb + 16 + + + mapreduce.map.speculative + false + + + mapreduce.reduce.speculative + false + + + mapreduce.job.reduce.slowstart.completedmaps + 0.8 + + + mapreduce.jobhistory.address + quickstart.cloudera:10020 + + + mapreduce.jobhistory.webapp.address + quickstart.cloudera:19888 + + + mapreduce.jobhistory.webapp.https.address + quickstart.cloudera:19890 + + + mapreduce.jobhistory.admin.address + quickstart.cloudera:10033 + + + mapreduce.framework.name + yarn + + + yarn.app.mapreduce.am.staging-dir + /user + + + mapreduce.am.max-attempts + 2 + + + yarn.app.mapreduce.am.resource.mb + 128 + + + yarn.app.mapreduce.am.resource.cpu-vcores + 1 + + + mapreduce.job.ubertask.enable + false + + + yarn.app.mapreduce.am.command-opts + -Djava.net.preferIPv4Stack=true -Xmx52428800 + + + mapreduce.map.java.opts + -Djava.net.preferIPv4Stack=true -Xmx52428800 + + + mapreduce.reduce.java.opts + -Djava.net.preferIPv4Stack=true -Xmx52428800 + + + yarn.app.mapreduce.am.admin.user.env + LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH + + + mapreduce.map.memory.mb + 128 + + + mapreduce.map.cpu.vcores + 1 + + + mapreduce.reduce.memory.mb + 128 + + + mapreduce.reduce.cpu.vcores + 1 + + + mapreduce.job.heap.memory-mb.ratio + 0.8 + + + mapreduce.application.classpath + /tmp/kylin/*,/usr/lib/hadoop-mapreduce/lib/*,/etc/hadoop/conf:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-mapreduce/.//*,/usr/lib/hbase/hbase-common.jar,/etc/hive/conf + + + mapreduce.admin.user.env + LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH + + + mapreduce.shuffle.max.connections + 80 + + http://git-wip-us.apache.org/repos/asf/kylin/blob/f2450cb6/examples/test_case_data/sandbox/yarn-site.xml ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml index 8256158..8988d4a 100644 --- a/examples/test_case_data/sandbox/yarn-site.xml +++ b/examples/test_case_data/sandbox/yarn-site.xml @@ -15,520 +15,128 @@ limitations under the License. --> - - - hadoop.registry.rm.enabled - false - - - - hadoop.registry.zk.quorum - sandbox.hortonworks.com:2181 - - yarn.acl.enable - false - - - - yarn.admin.acl - - - - - yarn.application.classpath - $HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/* - - - - yarn.client.nodemanager-connect.max-wait-ms - 60000 - - - - yarn.client.nodemanager-connect.retry-interval-ms - 10000 - - - - yarn.http.policy - HTTP_ONLY - - - - yarn.log-aggregation-enable - true - - - - yarn.log-aggregation.retain-seconds - 2592000 - - - - yarn.log.server.url - http://sandbox.hortonworks.com:19888/jobhistory/logs - - - - yarn.node-labels.fs-store.retry-policy-spec - 2000, 500 - - - - yarn.node-labels.fs-store.root-dir - /system/yarn/node-labels - - - - yarn.node-labels.manager-class - org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager - - - - yarn.nodemanager.address - 0.0.0.0:45454 - - - - yarn.nodemanager.admin-env - MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX - - - - yarn.nodemanager.aux-services - mapreduce_shuffle - - - - yarn.nodemanager.aux-services.mapreduce_shuffle.class - org.apache.hadoop.mapred.ShuffleHandler - - - - yarn.nodemanager.bind-host - 0.0.0.0 - - - - yarn.nodemanager.container-executor.class - org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor - - - - yarn.nodemanager.container-monitor.interval-ms - 3000 - - - - yarn.nodemanager.delete.debug-delay-sec - 0 - - - - yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage - 90 - - - - yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb - 1000 - - - - yarn.nodemanager.disk-health-checker.min-healthy-disks - 0.25 - - - - yarn.nodemanager.health-checker.interval-ms - 135000 - - - - yarn.nodemanager.health-checker.script.timeout-ms - 60000 - - - - yarn.nodemanager.linux-container-executor.cgroups.hierarchy - hadoop-yarn - - - - yarn.nodemanager.linux-container-executor.cgroups.mount - false - - - - yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage - false - - - - yarn.nodemanager.linux-container-executor.group - hadoop - - - - yarn.nodemanager.linux-container-executor.resources-handler.class - org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler - - - - yarn.nodemanager.local-dirs - /hadoop/yarn/local - - - - yarn.nodemanager.log-aggregation.compression-type - gz - - - - yarn.nodemanager.log-aggregation.debug-enabled - false - - - - yarn.nodemanager.log-aggregation.num-log-files-per-app - 30 - - - - yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds - -1 - - - - yarn.nodemanager.log-dirs - /hadoop/yarn/log - - - - yarn.nodemanager.log.retain-second - 604800 - - - - yarn.nodemanager.pmem-check-enabled - false - - - - yarn.nodemanager.recovery.dir - /var/log/hadoop-yarn/nodemanager/recovery-state - - - - yarn.nodemanager.recovery.enabled true - - - yarn.nodemanager.remote-app-log-dir - /app-logs - - - - yarn.nodemanager.remote-app-log-dir-suffix - logs - - - - yarn.nodemanager.resource.cpu-vcores - 8 - - - - yarn.nodemanager.resource.memory-mb - 9216 - - - - yarn.nodemanager.resource.percentage-physical-cpu-limit - 100 - - - - yarn.nodemanager.vmem-check-enabled - false - - - yarn.nodemanager.vmem-pmem-ratio - 10 + yarn.admin.acl + * - yarn.resourcemanager.address - sandbox.hortonworks.com:8050 + quickstart.cloudera:8032 - yarn.resourcemanager.admin.address - sandbox.hortonworks.com:8141 - - - - yarn.resourcemanager.am.max-attempts - 2 - - - - yarn.resourcemanager.bind-host - 0.0.0.0 - - - - yarn.resourcemanager.connect.max-wait.ms - 900000 - - - - yarn.resourcemanager.connect.retry-interval.ms - 30000 - - - - yarn.resourcemanager.fs.state-store.retry-policy-spec - 2000, 500 - - - - yarn.resourcemanager.fs.state-store.uri - - - - - yarn.resourcemanager.ha.enabled - false - - - - yarn.resourcemanager.hostname - sandbox.hortonworks.com - - - - yarn.resourcemanager.nodes.exclude-path - /etc/hadoop/conf/yarn.exclude - - - - yarn.resourcemanager.recovery.enabled - true + quickstart.cloudera:8033 - - - yarn.resourcemanager.resource-tracker.address - sandbox.hortonworks.com:8025 - - yarn.resourcemanager.scheduler.address - sandbox.hortonworks.com:8030 + quickstart.cloudera:8030 - - yarn.resourcemanager.scheduler.class - org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler - - - - yarn.resourcemanager.state-store.max-completed-applications - ${yarn.resourcemanager.max-completed-applications} - - - - yarn.resourcemanager.store.class - org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore - - - - yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size - 10 - - - - yarn.resourcemanager.system-metrics-publisher.enabled - true + yarn.resourcemanager.resource-tracker.address + quickstart.cloudera:8031 - yarn.resourcemanager.webapp.address - sandbox.hortonworks.com:8088 + quickstart.cloudera:8088 - - - yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled - false - - yarn.resourcemanager.webapp.https.address - localhost:8090 + quickstart.cloudera:8090 - - yarn.resourcemanager.webapp.proxyuser.hcat.groups - * + yarn.resourcemanager.client.thread-count + 50 - - yarn.resourcemanager.webapp.proxyuser.hcat.hosts - * + yarn.resourcemanager.scheduler.client.thread-count + 50 - - yarn.resourcemanager.webapp.proxyuser.oozie.groups - * + yarn.resourcemanager.admin.client.thread-count + 1 - - yarn.resourcemanager.webapp.proxyuser.oozie.hosts - * + yarn.scheduler.minimum-allocation-mb + 1 - - yarn.resourcemanager.work-preserving-recovery.enabled - true + yarn.scheduler.increment-allocation-mb + 512 - - yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms - 10000 + yarn.scheduler.maximum-allocation-mb + 2816 - - yarn.resourcemanager.zk-acl - world:anyone:rwcda + yarn.scheduler.minimum-allocation-vcores + 1 - - yarn.resourcemanager.zk-address - localhost:2181 + yarn.scheduler.increment-allocation-vcores + 1 - - yarn.resourcemanager.zk-num-retries - 1000 + yarn.scheduler.maximum-allocation-vcores + 2 - - yarn.resourcemanager.zk-retry-interval-ms + yarn.resourcemanager.amliveliness-monitor.interval-ms 1000 - - yarn.resourcemanager.zk-state-store.parent-path - /rmstore + yarn.am.liveness-monitor.expiry-interval-ms + 600000 - - - yarn.resourcemanager.zk-timeout-ms - 10000 - - - - yarn.scheduler.maximum-allocation-mb - 9216 - - - yarn.scheduler.minimum-allocation-mb - 1536 + yarn.resourcemanager.am.max-attempts + 2 - - yarn.timeline-service.address - sandbox.hortonworks.com:10200 + yarn.resourcemanager.container.liveness-monitor.interval-ms + 600000 - - yarn.timeline-service.bind-host - 0.0.0.0 + yarn.resourcemanager.nm.liveness-monitor.interval-ms + 1000 - - yarn.timeline-service.client.max-retries - 30 + yarn.nm.liveness-monitor.expiry-interval-ms + 600000 - - yarn.timeline-service.client.retry-interval-ms - 1000 + yarn.resourcemanager.resource-tracker.client.thread-count + 50 - - yarn.timeline-service.enabled - true + yarn.application.classpath + $HADOOP_CLIENT_CONF_DIR,$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_YARN_HOME/*,$HADOOP_YARN_HOME/lib/* - - yarn.timeline-service.generic-application-history.store-class - org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore + yarn.resourcemanager.scheduler.class + org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler - - yarn.timeline-service.http-authentication.simple.anonymous.allowed + yarn.scheduler.fair.user-as-default-queue true - - - yarn.timeline-service.http-authentication.type - simple - - - yarn.timeline-service.leveldb-timeline-store.path - /hadoop/yarn/timeline + yarn.scheduler.fair.preemption + false - - yarn.timeline-service.leveldb-timeline-store.read-cache-size - 104857600 + yarn.scheduler.fair.sizebasedweight + false - - yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size - 10000 + yarn.scheduler.fair.assignmultiple + false - - yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size + yarn.resourcemanager.max-completed-applications 10000 - - - yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms - 300000 - - - - yarn.timeline-service.store-class - org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore - - - - yarn.timeline-service.ttl-enable - true - - - - yarn.timeline-service.ttl-ms - 2678400000 - - - - yarn.timeline-service.webapp.address - sandbox.hortonworks.com:8188 - - - - yarn.timeline-service.webapp.https.address - sandbox.hortonworks.com:8190 - - http://git-wip-us.apache.org/repos/asf/kylin/blob/f2450cb6/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index dc23a2f..1d87bc7 100644 --- a/pom.xml +++ b/pom.xml @@ -46,19 +46,19 @@ UTF-8 - 2.7.1 - 2.7.1 + 2.6.0-cdh5.7.0 + 2.6.0-cdh5.7.0 - 1.2.1 - 1.2.1 + 1.1.0-cdh5.7.0 + 1.1.0-cdh5.7.0 - 1.1.1 + 1.2.0-cdh5.7.0 0.10.0.0 - 3.4.6 + 3.4.5-cdh5.7.0 2.7.1 2.2.4 3.0.1 @@ -814,6 +814,10 @@ conjars http://conjars.org/repo/ + + cloudera + https://repository.cloudera.com/artifactory/cloudera-repos/ + http://git-wip-us.apache.org/repos/asf/kylin/blob/f2450cb6/server/pom.xml ---------------------------------------------------------------------- diff --git a/server/pom.xml b/server/pom.xml index cf92fb1..20f4483 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -112,6 +112,10 @@ javax.servlet servlet-api + + com.google.protobuf + protobuf-java + javax.servlet.jsp jsp-api @@ -131,6 +135,10 @@ javax.servlet.jsp jsp-api + + com.google.protobuf + protobuf-java + @@ -146,6 +154,10 @@ javax.servlet.jsp jsp-api + + com.google.protobuf + protobuf-java + @@ -161,6 +173,10 @@ javax.servlet.jsp jsp-api + + com.google.protobuf + protobuf-java + @@ -176,6 +192,10 @@ javax.servlet.jsp jsp-api + + com.google.protobuf + protobuf-java + @@ -199,6 +219,10 @@ javax.servlet.jsp jsp-api + + com.google.protobuf + protobuf-java + @@ -214,6 +238,10 @@ javax.servlet.jsp jsp-api + + com.google.protobuf + protobuf-java + @@ -229,6 +257,10 @@ javax.servlet.jsp jsp-api + + com.google.protobuf + protobuf-java + @@ -262,6 +294,10 @@ javax.servlet.jsp jsp-api + + com.google.protobuf + protobuf-java + http://git-wip-us.apache.org/repos/asf/kylin/blob/f2450cb6/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java index d5c3f60..5adf327 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java @@ -100,11 +100,6 @@ public class MockupMapContext { } @Override - public float getProgress() { - throw new NotImplementedException(); - } - - @Override public Counter getCounter(Enum counterName) { throw new NotImplementedException(); } @@ -165,6 +160,11 @@ public class MockupMapContext { } @Override + public boolean userClassesTakesPrecedence() { + return false; + } + + @Override public Class> getInputFormatClass() throws ClassNotFoundException { throw new NotImplementedException(); } @@ -214,10 +214,6 @@ public class MockupMapContext { throw new NotImplementedException(); } - @Override - public boolean getTaskCleanupNeeded() { - throw new NotImplementedException(); - } @Override public boolean getProfileEnabled() { @@ -230,11 +226,6 @@ public class MockupMapContext { } @Override - public IntegerRanges getProfileTaskRange(boolean isMap) { - throw new NotImplementedException(); - } - - @Override public String getUser() { throw new NotImplementedException(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/f2450cb6/tool/pom.xml ---------------------------------------------------------------------- diff --git a/tool/pom.xml b/tool/pom.xml index f4afdf5..8629b25 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -49,6 +49,18 @@ + org.apache.hadoop + hadoop-yarn-api + ${yarn.version} + provided + + + org.apache.hadoop + hadoop-yarn-common + ${yarn.version} + provided + + org.apache.hbase hbase-client provided