eagle-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (EAGLE-820) add unit test for eagle-jpm-mr-history
Date Thu, 15 Dec 2016 02:54:00 GMT

    [ https://issues.apache.org/jira/browse/EAGLE-820?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15750200#comment-15750200
] 

ASF GitHub Bot commented on EAGLE-820:
--------------------------------------

Github user wujinhu commented on a diff in the pull request:

    https://github.com/apache/incubator-eagle/pull/710#discussion_r92535638
  
    --- Diff: eagle-jpm/eagle-jpm-mr-history/src/test/resources/job_1479206441898_508949_conf.xml
---
    @@ -0,0 +1,1337 @@
    +<!--
    +   Licensed to the Apache Software Foundation (ASF) under one or more
    +   contributor license agreements.  See the NOTICE file distributed with
    +   this work for additional information regarding copyright ownership.
    +   The ASF licenses this file to You under the Apache License, Version 2.0
    +   (the "License"); you may not use this file except in compliance with
    +   the License.  You may obtain a copy of the License at
    +
    +       http://www.apache.org/licenses/LICENSE-2.0
    +
    +   Unless required by applicable law or agreed to in writing, software
    +   distributed under the License is distributed on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +   See the License for the specific language governing permissions and
    +   limitations under the License.
    +-->
    +<?xml version="1.0" encoding="UTF-8" standalone="no"?><configuration>
    +<property><name>hadoop.proxyuser.hive.groups</name><value>*</value><source>core-site.xml</source><source>job.xml</source></property>
    +<property><name>dfs.block.access.token.lifetime</name><value>600</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>dfs.namenode.rpc-address.nameservice2.namenode309</name><value>yhd-jqhadoop180.int.yihaodian.com:8020</value><source>hdfs-site.xml</source><source>job.xml</source></property>
    +<property><name>hive.skewjoin.key</name><value>100000</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>mapreduce.job.heap.memory-mb.ratio</name><value>0.8</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>dfs.namenode.rpc-address.nameservice2.namenode307</name><value>yhd-jqhadoop175.int.yihaodian.com:8020</value><source>hdfs-site.xml</source><source>job.xml</source></property>
    +<property><name>hive.index.compact.binary.search</name><value>true</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>mapreduce.map.log.level</name><value>INFO</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>dfs.namenode.lazypersist.file.scrub.interval.sec</name><value>300</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>mapreduce.admin.user.env</name><value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value><source>mapred-site.xml</source><source>job.xml</source></property>
    +<property><name>file.bytes-per-checksum</name><value>512</value><source>core-default.xml</source><source>job.xml</source></property>
    +<property><name>mapreduce.client.completion.pollinterval</name><value>5000</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name><value>false</value><source>yarn-default.xml</source><source>job.xml</source></property>
    +<property><name>yarn.log-aggregation-enable</name><value>false</value><source>yarn-default.xml</source><source>job.xml</source></property>
    +<property><name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name><value>org.apache.hadoop.mapred.ShuffleHandler</value><source>yarn-default.xml</source><source>job.xml</source></property>
    +<property><name>dfs.namenode.edit.log.autoroll.check.interval.ms</name><value>300000</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>mapreduce.job.speculative.retry-after-speculate</name><value>15000</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>ipc.client.fallback-to-simple-auth-allowed</name><value>false</value><source>core-default.xml</source><source>job.xml</source></property>
    +<property><name>dfs.client.failover.connection.retries</name><value>0</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>mapreduce.jobtracker.system.dir</name><value>${hadoop.tmp.dir}/mapred/system</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>hive.metastore.event.db.listener.timetolive</name><value>86400s</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>yarn.scheduler.minimum-allocation-mb</name><value>1024</value><source>yarn-site.xml</source><source>job.xml</source></property>
    +<property><name>mapreduce.task.profile.map.params</name><value>${mapreduce.task.profile.params}</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>mapreduce.map.memory.mb</name><value>2048</value><source>mapred-site.xml</source><source>job.xml</source></property>
    +<property><name>mapreduce.tasktracker.dns.interface</name><value>default</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>dfs.datanode.failed.volumes.tolerated</name><value>0</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>hive.server2.authentication</name><value>NONE</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>_hive.tmp_table_space</name><value>/tmp/warehouse/bi_etl/9cbc3c63-c8f9-48cc-94a3-43ae6191ddf1/_tmp_space.db</value><source>programatically</source><source>job.xml</source></property>
    +<property><name>stream.stderr.reporter.prefix</name><value>reporter:</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>dfs.client.slow.io.warning.threshold.ms</name><value>30000</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>hadoop.security.groups.cache.secs</name><value>300</value><source>core-default.xml</source><source>job.xml</source></property>
    +<property><name>yarn.nodemanager.env-whitelist</name><value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME</value><source>yarn-default.xml</source><source>job.xml</source></property>
    +<property><name>dfs.namenode.top.window.num.buckets</name><value>10</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>hive.metastore.authorization.storage.checks</name><value>false</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>map.sort.class</name><value>org.apache.hadoop.util.QuickSort</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>dfs.namenode.safemode.threshold-pct</name><value>0.999f</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>mapreduce.jobtracker.jobhistory.task.numberprogresssplits</name><value>12</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>datanucleus.storeManagerType</name><value>rdbms</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>dfs.short.circuit.shared.memory.watcher.interrupt.check.ms</name><value>60000</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>hive.metastore.aggregate.stats.cache.max.writer.wait</name><value>5000ms</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>fs.s3n.block.size</name><value>67108864</value><source>core-default.xml</source><source>job.xml</source></property>
    +<property><name>yarn.resourcemanager.client.thread-count</name><value>50</value><source>yarn-site.xml</source><source>job.xml</source></property>
    +<property><name>dfs.client.read.shortcircuit</name><value>false</value><source>hdfs-site.xml</source><source>job.xml</source></property>
    +<property><name>mapreduce.job.end-notification.max.retry.interval</name><value>5000</value><source>mapred-default.xml</source></property>
    +<property><name>hadoop.security.authentication</name><value>simple</value><source>core-site.xml</source><source>job.xml</source></property>
    +<property><name>dfs.client.mmap.retry.timeout.ms</name><value>300000</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>dfs.datanode.readahead.bytes</name><value>4193404</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>mapreduce.jobhistory.max-age-ms</name><value>604800000</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>yarn.app.mapreduce.client-am.ipc.max-retries</name><value>3</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>hive.mapjoin.followby.map.aggr.hash.percentmemory</name><value>0.3</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>hive.script.recordwriter</name><value>org.apache.hadoop.hive.ql.exec.TextRecordWriter</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>hive.multigroupby.singlereducer</name><value>true</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name><value>250</value><source>yarn-default.xml</source><source>job.xml</source></property>
    +<property><name>hive.vectorized.execution.reduce.groupby.enabled</name><value>true</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>hadoop.shell.missing.defaultFs.warning</name><value>true</value><source>core-default.xml</source><source>job.xml</source></property>
    +<property><name>fs.trash.interval</name><value>1</value><source>core-site.xml</source><source>job.xml</source></property>
    +<property><name>dfs.datanode.max.locked.memory</name><value>0</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>hadoop.http.filter.initializers</name><value>org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer</value><source>programatically</source><source>job.xml</source></property>
    +<property><name>fs.har.impl</name><value>org.apache.hadoop.hive.shims.HiveHarFileSystem</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>dfs.datanode.ipc.address</name><value>0.0.0.0:50020</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>dfs.namenode.delegation.token.renew-interval</name><value>86400000</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>yarn.resourcemanager.webapp.address</name><value>${yarn.resourcemanager.hostname}:8088</value><source>yarn-default.xml</source><source>job.xml</source></property>
    +<property><name>hive.spark.client.rpc.sasl.mechanisms</name><value>DIGEST-MD5</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>hive.current.database</name><value>default</value><source>programatically</source><source>job.xml</source></property>
    +<property><name>hive.optimize.index.groupby</name><value>false</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>mapreduce.shuffle.max.connections</name><value>80</value><source>mapred-site.xml</source><source>job.xml</source></property>
    +<property><name>yarn.nodemanager.local-cache.max-files-per-directory</name><value>8192</value><source>yarn-default.xml</source><source>job.xml</source></property>
    +<property><name>dfs.namenode.replication.work.multiplier.per.iteration</name><value>2</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>mapred.mapper.new-api</name><value>false</value><source>programatically</source><source>job.xml</source></property>
    +<property><name>mapreduce.jobtracker.address</name><value>local</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>dfs.datanode.slow.io.warning.threshold.ms</name><value>300</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>yarn.client.max-cached-nodemanagers-proxies</name><value>0</value><source>yarn-default.xml</source><source>job.xml</source></property>
    +<property><name>mapreduce.job.working.dir</name><value>viewfs://cluster9/user/bi_etl</value><source>programatically</source><source>job.xml</source></property>
    +<property><name>mapreduce.tasktracker.instrumentation</name><value>org.apache.hadoop.mapred.TaskTrackerMetricsInst</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>hadoop.registry.zk.quorum</name><value>localhost:2181</value><source>yarn-default.xml</source><source>job.xml</source></property>
    +<property><name>yarn.app.mapreduce.am.job.committer.commit-window</name><value>10000</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>hadoop.registry.zk.session.timeout.ms</name><value>60000</value><source>yarn-default.xml</source><source>job.xml</source></property>
    +<property><name>hive.tez.exec.print.summary</name><value>false</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>nfs.exports.allowed.hosts</name><value>* rw</value><source>core-default.xml</source><source>job.xml</source></property>
    +<property><name>hive.metastore.filter.hook</name><value>org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>hive.metastore.integral.jdo.pushdown</name><value>false</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>hadoop.util.hash.type</name><value>murmur</value><source>core-default.xml</source><source>job.xml</source></property>
    +<property><name>dfs.namenode.replication.min</name><value>1</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>mapreduce.job.user.name</name><value>bi_etl</value><source>programatically</source></property>
    +<property><name>hive.cli.print.current.db</name><value>false</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>hive.debug.localtask</name><value>false</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>hive.security.authorization.enabled</name><value>false</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>hive.warehouse.subdir.inherit.perms</name><value>true</value><source>file:/data/opt/hive1.2.1/apache-hive-1.2.1-bin/conf/hive-site.xml</source><source>job.xml</source></property>
    +<property><name>mapreduce.jobtracker.retiredjobs.cache.size</name><value>1000</value><source>mapred-default.xml</source><source>job.xml</source></property>
    +<property><name>s3.client-write-packet-size</name><value>65536</value><source>core-default.xml</source><source>job.xml</source></property>
    +<property><name>yarn.resourcemanager.scheduler.address</name><value>${yarn.resourcemanager.hostname}:8030</value><source>yarn-default.xml</source><source>job.xml</source></property>
    +<property><name>hive.metastore.try.direct.sql</name><value>true</value><source>programatically</source><source>org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@78dfcf46</source><source>job.xml</source></property>
    +<property><name>fs.s3.block.size</name><value>67108864</value><source>core-default.xml</source><source>job.xml</source></property>
    +<property><name>hadoop.security.group.mapping.ldap.search.filter.user</name><value>(&amp;(objectClass=user)(sAMAccountName={0}))</value><source>core-default.xml</source><source>job.xml</source></property>
    +<property><name>dfs.permissions.superusergroup</name><value>supergroup</value><source>hdfs-default.xml</source><source>job.xml</source></property>
    +<property><name>yarn.resourcemanager.resource-tracker.address</name><value>${yarn.resourcemanager.hostname}:8031</value><source>yarn-default.xml</source><source>job.xml</source></property>
    +<property><name>yarn.resourcemanager.resource-tracker.address.rm405</name><value>yhd-jqhadoop168.int.yihaodian.com:8031</value><source>yarn-site.xml</source><source>job.xml</source></property>
    --- End diff --
    
    Pls remove sensitive information.


> add unit test for eagle-jpm-mr-history
> --------------------------------------
>
>                 Key: EAGLE-820
>                 URL: https://issues.apache.org/jira/browse/EAGLE-820
>             Project: Eagle
>          Issue Type: Improvement
>            Reporter: luokun
>            Assignee: luokun
>




--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message