falcon-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Raghav Kumar Gautam (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (FALCON-306) Feed late replication is not working
Date Fri, 28 Mar 2014 00:27:17 GMT

    [ https://issues.apache.org/jira/browse/FALCON-306?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13950201#comment-13950201
] 

Raghav Kumar Gautam commented on FALCON-306:
--------------------------------------------

{code:xml}
<?xml version="1.0" encoding="UTF-8"?>
<cluster xmlns="uri:falcon:cluster:0.1" name="corp-c920d665-4014-4cc9-91ab-d962988032f7"
description="" colo="default">
  <interfaces>
    <interface type="readonly" endpoint="webhdfs://rkg1-2:50070" version="0.20.2"/>
    <interface type="write" endpoint="hdfs://rkg1-2:8020" version="0.20.2"/>
    <interface type="execute" endpoint="rkg1-2:8032" version="0.20.2"/>
    <interface type="workflow" endpoint="http://rkg1-2:11000/oozie/" version="3.1"/>
    <interface type="messaging" endpoint="tcp://rkg1-2:61616?daemon=true" version="5.1.6"/>
  </interfaces>
  <locations>
    <location name="staging" path="/projects/ivory/staging"/>
    <location name="temp" path="/tmp"/>
    <location name="working" path="/projectsTest/ivory/working"/>
  </locations>
  <properties>
    <property name="field1" value="value1"/>
    <property name="field2" value="value2"/>
    <property name="dfs.namenode.kerberos.principal" value="nn/rkg1-2@EXAMPLE.COM"/>
    <property name="hive.metastore.kerberos.principal" value="hive/rkg1-2@EXAMPLE.COM"/>
  </properties>
</cluster>
{code}
{code:xml}
<?xml version="1.0" encoding="UTF-8"?>
<cluster xmlns="uri:falcon:cluster:0.1" name="corp-44c1d0e0-6880-4bcb-bef3-5c17823712a4"
description="" colo="default">
  <interfaces>
    <interface type="readonly" endpoint="webhdfs://rkg1-7:50070" version="0.20.2"/>
    <interface type="write" endpoint="hdfs://rkg1-7:8020" version="0.20.2"/>
    <interface type="execute" endpoint="rkg1-7:8032" version="0.20.2"/>
    <interface type="workflow" endpoint="http://rkg1-7:11000/oozie/" version="3.1"/>
    <interface type="messaging" endpoint="tcp://rkg1-2:61616?daemon=true" version="5.1.6"/>
  </interfaces>
  <locations>
    <location name="staging" path="/projects/ivory/staging"/>
    <location name="temp" path="/tmp"/>
    <location name="working" path="/projectsTest/ivory/working"/>
  </locations>
  <properties>
    <property name="field1" value="value1"/>
    <property name="field2" value="value2"/>
    <property name="dfs.namenode.kerberos.principal" value="nn/rkg1-7@EXAMPLE.COM"/>
    <property name="hive.metastore.kerberos.principal" value="hive/rkg1-7@EXAMPLE.COM"/>
  </properties>
</cluster>
{code}
{code:xml}
<?xml version="1.0" encoding="UTF-8"?>
<cluster xmlns="uri:falcon:cluster:0.1" name="corp-cab970c8-3ca4-446e-92a7-db750a98c8a2"
description="" colo="default">
  <interfaces>
    <interface type="readonly" endpoint="webhdfs://rkg1-1:50070" version="0.20.2"/>
    <interface type="write" endpoint="hdfs://rkg1-1:8020" version="0.20.2"/>
    <interface type="execute" endpoint="rkg1-1:8032" version="0.20.2"/>
    <interface type="workflow" endpoint="http://rkg1-1:11000/oozie/" version="3.1"/>
    <interface type="messaging" endpoint="tcp://rkg1-2:61616?daemon=true" version="5.1.6"/>
  </interfaces>
  <locations>
    <location name="staging" path="/projects/ivory/staging"/>
    <location name="temp" path="/tmp"/>
    <location name="working" path="/projectsTest/ivory/working"/>
  </locations>
  <properties>
    <property name="field1" value="value1"/>
    <property name="field2" value="value2"/>
    <property name="dfs.namenode.kerberos.principal" value="nn/rkg1-1@EXAMPLE.COM"/>
    <property name="hive.metastore.kerberos.principal" value="hive/rkg1-1@EXAMPLE.COM"/>
  </properties>
</cluster>
{code}
{code:xml}
<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="uri:falcon:feed:0.1" name="raaw-logs16-775fcf01-6fbb-4226-bbe3-d9fc80c13130"
description="clicks log">
  <partitions>
    <partition name="country"/>
    <partition name="colo"/>
  </partitions>
  <frequency>minutes(20)</frequency>
  <timezone>UTC</timezone>
  <late-arrival cut-off="hours(6)"/>
  <clusters>
    <cluster name="corp-44c1d0e0-6880-4bcb-bef3-5c17823712a4" type="source" partition="US/${cluster.colo}">
      <validity start="2014-03-27T23:19Z" end="2099-01-01T00:00Z"/>
      <retention limit="hours(10)" action="delete"/>
    </cluster>
    <cluster name="corp-c920d665-4014-4cc9-91ab-d962988032f7" type="target">
      <validity start="2014-03-27T23:19Z" end="2099-01-01T00:00Z"/>
      <retention limit="hours(10)" action="delete"/>
    </cluster>
    <cluster name="corp-cab970c8-3ca4-446e-92a7-db750a98c8a2" type="source" partition="UK/${cluster.colo}">
      <validity start="2014-03-27T23:19Z" end="2099-01-01T00:00Z"/>
      <retention limit="hours(10)" action="delete"/>
    </cluster>
  </clusters>
  <locations>
    <location type="data" path="/tmp/falcon-regression/input-data/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}/"/>
    <location type="stats" path="/projects/ivory/clicksStats"/>
    <location type="meta" path="/projects/ivory/clicksMetaData"/>
  </locations>
  <ACL owner="testuser" group="group" permission="0x755"/>
  <schema location="/schema/clicks" provider="protobuf"/>
  <properties>
    <property name="field1" value="value1"/>
    <property name="field2" value="value2"/>
  </properties>
</feed>
{code}

> Feed late replication is not working
> ------------------------------------
>
>                 Key: FALCON-306
>                 URL: https://issues.apache.org/jira/browse/FALCON-306
>             Project: Falcon
>          Issue Type: Bug
>            Reporter: Raghav Kumar Gautam
>              Labels: falcon-regression
>
> *STR*
> - Source clusters have following folders:
>   at colo=ua3, partition=UK/${cluster.colo} folder=input-data/2014/02/05/22/08/UK/ua3
>   at colo=ua2, partition=US/${cluster.colo} folder=input-data/2014/02/05/22/08/US/ua2
> - Target cluster
>   at colo=ua1: folder=input-data/2014/02/05/22/08/
> - Upload some data in source folders
> - Replication goes through
> - Upload some more data in source folders
> - Late replication does not happen
> *Test*
> PrismFeedLateReplicationTest#multipleSourceOneTarget_pastData
> PrismFeedLateReplicationTest#multipleSourceOneTarget_futureData



--
This message was sent by Atlassian JIRA
(v6.2#6252)

Mime
View raw message