falcon-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Pragya Mittal (JIRA)" <j...@apache.org>
Subject [jira] [Updated] (FALCON-1887) Process schedule fails when entity contains optional dataSet
Date Tue, 05 Apr 2016 14:18:25 GMT

     [ https://issues.apache.org/jira/browse/FALCON-1887?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Pragya Mittal updated FALCON-1887:
----------------------------------
    Description: 
Process schedule fails in dry run when entity contains optional dataSet.

Process definition :
{noformat}
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<process name="OptionalInputTest-agregator-3" xmlns="uri:falcon:process:0.1">
    <clusters>
        <cluster name="OptionalInputTest-corp-999c05510">
            <validity start="2010-01-02T01:00Z" end="2010-01-02T01:12Z"/>
        </cluster>
    </clusters>
    <parallel>2</parallel>
    <order>FIFO</order>
    <frequency>minutes(5)</frequency>
    <timezone>UTC</timezone>
    <inputs>
        <input name="inputData0" feed="OptionalInputTest-raaw-logs16-cfe063e3-input0" start="now(0,-10)"
end="now(0,0)" optional="true"/>
        <input name="inputData" feed="OptionalInputTest-raaw-logs16-cfe063e3-input1" start="now(0,-10)"
end="now(0,0)" optional="false"/>
    </inputs>
    <outputs>
        <output name="outputData" feed="OptionalInputTest-raaw-logs16-cfe063e3-output0"
instance="now(0,0)"/>
    </outputs>
    <properties>
        <property name="queueName" value="default"/>
        <property name="fileTime" value="${formatTime(dateOffset(instanceTime(), 1, 'DAY'),
'yyyy-MMM-dd')}"/>
        <property name="user" value="${user()}"/>
    </properties>
    <workflow path="/tmp/falcon-regression/OptionalInputTest/aggregator"/>
    <retry policy="periodic" delay="minutes(3)" attempts="3"/>
    <ACL owner="pragya" group="dataqa" permission="*"/>
</process>

{noformat}


While scheduling with/without skipDryRun option
{noformat}
falcon entity -type process -schedule -name OptionalInputTest-agregator-3
ERROR: Bad Request;ua1/E0803 : E0803: IO error, Unable to evaluate :${dataIn('inputData0',
'null')}#${dataIn('inputData', 'null')}:
CausedBy: E0803: IO error, Unable to evaluate :${dataIn('inputData0', 'null')}#${dataIn('inputData',
'null')}:

falcon entity -type process -schedule -name OptionalInputTest-agregator-3 -skipDryRun
ua1/OptionalInputTest-agregator-bcjk-1(process) scheduled successfully
{noformat}


Server logs :
{noformat}
2016-04-05 14:03:57,659 ERROR - [816508486@qtp-811301908-51 - 9f673069-d551-4e0c-8514-49bdf138cca3:dataqa:POST//entities/schedule/process/OptionalInputTest-agregator-3]
~ Entity schedule failed for process: OptionalInputTest-agregator-3 (AbstractSchedulableEntityManager:105)
org.apache.falcon.FalconException: E0803 : E0803: IO error, Unable to evaluate :${dataIn('inputData0',
'null')}#${dataIn('inputData', 'null')}:
	at org.apache.falcon.workflow.engine.OozieWorkflowEngine.dryRunInternal(OozieWorkflowEngine.java:240)
	at org.apache.falcon.workflow.engine.OozieWorkflowEngine.schedule(OozieWorkflowEngine.java:178)
	at org.apache.falcon.resource.AbstractSchedulableEntityManager.scheduleInternal(AbstractSchedulableEntityManager.java:103)
	at org.apache.falcon.resource.AbstractSchedulableEntityManager.schedule(AbstractSchedulableEntityManager.java:82)
	at org.apache.falcon.resource.SchedulableEntityManager.schedule(SchedulableEntityManager.java:262)
	at sun.reflect.GeneratedMethodAccessor101.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:497)
	at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)
	at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:185)
	at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75)
	at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:288)
	at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
	at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108)
	at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
	at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84)
	at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469)
	at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400)
	at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349)
	at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339)
	at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416)
	at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:537)
	at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:699)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
	at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
	at org.apache.falcon.security.FalconAuthorizationFilter.doFilter(FalconAuthorizationFilter.java:108)
	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
	at org.apache.falcon.security.FalconAuthenticationFilter$2.doFilter(FalconAuthenticationFilter.java:188)
	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:572)
	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:542)
	at org.apache.falcon.security.FalconAuthenticationFilter.doFilter(FalconAuthenticationFilter.java:197)
	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
	at org.apache.falcon.security.FalconAuditFilter.doFilter(FalconAuditFilter.java:64)
	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
	at org.apache.falcon.security.HostnameFilter.doFilter(HostnameFilter.java:82)
	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
	at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
	at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
	at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
	at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:767)
	at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
	at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
	at org.mortbay.jetty.Server.handle(Server.java:326)
	at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
	at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
	at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
	at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
	at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
	at org.mortbay.jetty.bio.SocketConnector$Connection.run(SocketConnector.java:228)
	at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
Caused by: E0803 : E0803: IO error, Unable to evaluate :${dataIn('inputData0', 'null')}#${dataIn('inputData',
'null')}:
	at org.apache.oozie.client.OozieClient.handleError(OozieClient.java:612)
	at org.apache.oozie.client.OozieClient$JobSubmit.call(OozieClient.java:695)
	at org.apache.oozie.client.OozieClient$JobSubmit.call(OozieClient.java:665)
	at org.apache.oozie.client.OozieClient$ClientCallable.call(OozieClient.java:560)
	at org.apache.oozie.client.OozieClient.dryrun(OozieClient.java:829)
	at org.apache.oozie.client.ProxyOozieClient.access$401(ProxyOozieClient.java:48)
	at org.apache.oozie.client.ProxyOozieClient$4.call(ProxyOozieClient.java:177)
	at org.apache.oozie.client.ProxyOozieClient$4.call(ProxyOozieClient.java:174)
	at org.apache.oozie.client.OozieClient.doAs(OozieClient.java:244)
	at org.apache.oozie.client.ProxyOozieClient.dryrun(ProxyOozieClient.java:174)
	at org.apache.falcon.workflow.engine.OozieWorkflowEngine.dryRunInternal(OozieWorkflowEngine.java:238)
	... 50 more
2016-04-05 14:03:57,659 INFO  - [816508486@qtp-811301908-51 - 9f673069-d551-4e0c-8514-49bdf138cca3:dataqa:POST//entities/schedule/process/OptionalInputTest-agregator-3]
~ Successfully released lock on (process) OptionalInputTest-agregator-3 by 816508486@qtp-811301908-51
- 9f673069-d551-4e0c-8514-49bdf138cca3 (MemoryLocks:70)
2016-04-05 14:03:57,659 INFO  - [816508486@qtp-811301908-51 - 9f673069-d551-4e0c-8514-49bdf138cca3:dataqa:POST//entities/schedule/process/OptionalInputTest-agregator-3]
~ Memory lock released for (process) OptionalInputTest-agregator-3 (AbstractSchedulableEntityManager:110)
2016-04-05 14:03:57,659 ERROR - [816508486@qtp-811301908-51 - 9f673069-d551-4e0c-8514-49bdf138cca3:dataqa:POST//entities/schedule/process/OptionalInputTest-agregator-3]
~ Unable to schedule workflow (AbstractSchedulableEntityManager:85)
org.apache.falcon.FalconWebException
	at org.apache.falcon.FalconWebException.newAPIException(FalconWebException.java:62)
	at org.apache.falcon.FalconWebException.newAPIException(FalconWebException.java:50)
	at org.apache.falcon.FalconWebException.newAPIException(FalconWebException.java:45)
	at org.apache.falcon.resource.AbstractSchedulableEntityManager.scheduleInternal(AbstractSchedulableEntityManager.java:106)
	at org.apache.falcon.resource.AbstractSchedulableEntityManager.schedule(AbstractSchedulableEntityManager.java:82)
	at org.apache.falcon.resource.SchedulableEntityManager.schedule(SchedulableEntityManager.java:262)
	at sun.reflect.GeneratedMethodAccessor101.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:497)
	at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)
	at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:185)
	at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75)
	at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:288)
	at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
	at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108)
	at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
	at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84)
	at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469)
	at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400)
	at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349)
	at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339)
	at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416)
	at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:537)
	at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:699)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
	at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
	at org.apache.falcon.security.FalconAuthorizationFilter.doFilter(FalconAuthorizationFilter.java:108)
	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
	at org.apache.falcon.security.FalconAuthenticationFilter$2.doFilter(FalconAuthenticationFilter.java:188)
	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:572)
	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:542)
	at org.apache.falcon.security.FalconAuthenticationFilter.doFilter(FalconAuthenticationFilter.java:197)
	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
	at org.apache.falcon.security.FalconAuditFilter.doFilter(FalconAuditFilter.java:64)
	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
	at org.apache.falcon.security.HostnameFilter.doFilter(HostnameFilter.java:82)
	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
	at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
	at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
	at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
	at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:767)
	at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
	at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
	at org.mortbay.jetty.Server.handle(Server.java:326)
	at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
	at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
	at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
	at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
	at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
	at org.mortbay.jetty.bio.SocketConnector$Connection.run(SocketConnector.java:228)
	at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
2016-04-05 14:03:57,660 DEBUG - [816508486@qtp-811301908-51 - 9f673069-d551-4e0c-8514-49bdf138cca3:]
~ Audit: dataqa/192.168.138.236 performed request http://192.168.138.236:15000/api/entities/schedule/process/OptionalInputTest-agregator-3?colo=ua1&&user.name=dataqa
(192.168.138.236) at time 2016-04-05T14:03Z (FalconAuditFilter:86)
2016-04-05 14:03:57,660 INFO  - [816508486@qtp-811301908-51 - 9f673069-d551-4e0c-8514-49bdf138cca3:]
~ Audit: dataqa@192.168.138.236 performed http://192.168.138.236:15000/api/entities/schedule/process/OptionalInputTest-agregator-3?colo=ua1&&user.name=dataqa
(192.168.138.236) at 2016-04-05T14:03Z (AUDIT:48)

{noformat}


Oozie logs :
{noformat}
2016-04-05 13:59:28,145 ERROR pool-6-thread-1 CoordActionInputCheckXCommand - SERVER[lda01]
USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0000618-160404070324284-oozie-oozi-C] ACTION[0000618-160404070324284-oozie-oozi-C@3]
XException, 
org.apache.oozie.command.CommandException: E1021: Coord Action Input Check Error: Unable to
evaluate :${dataIn('inputData0', 'null')}#${dataIn('inputData', 'null')}:

	at org.apache.oozie.command.coord.CoordActionInputCheckXCommand.execute(CoordActionInputCheckXCommand.java:279)
	at org.apache.oozie.command.coord.CoordActionInputCheckXCommand.execute(CoordActionInputCheckXCommand.java:71)
	at org.apache.oozie.command.XCommand.call(XCommand.java:286)
	at org.apache.oozie.service.CallableQueueService$CallableWrapper.run(CallableQueueService.java:175)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.Exception: Unable to evaluate :${dataIn('inputData0', 'null')}#${dataIn('inputData',
'null')}:

	at org.apache.oozie.coord.CoordELFunctions.evalAndWrap(CoordELFunctions.java:723)
	at org.apache.oozie.command.coord.CoordActionInputCheckXCommand.resolveTagContents(CoordActionInputCheckXCommand.java:398)
	at org.apache.oozie.command.coord.CoordActionInputCheckXCommand.materializeDataProperties(CoordActionInputCheckXCommand.java:379)
	at org.apache.oozie.command.coord.CoordActionInputCheckXCommand.resolveCoordConfiguration(CoordActionInputCheckXCommand.java:288)
	at org.apache.oozie.command.coord.CoordActionInputCheckXCommand.execute(CoordActionInputCheckXCommand.java:230)
	... 6 more
Caused by: java.lang.NullPointerException
	at org.apache.oozie.extensions.OozieELExtensions.getDSElement(OozieELExtensions.java:124)
	at org.apache.oozie.extensions.OozieELExtensions.ph3_dataIn(OozieELExtensions.java:74)
	at sun.reflect.GeneratedMethodAccessor106.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:497)
	at org.apache.commons.el.FunctionInvocation.evaluate(FunctionInvocation.java:172)
	at org.apache.commons.el.ExpressionString.evaluate(ExpressionString.java:114)
	at org.apache.commons.el.ExpressionEvaluatorImpl.evaluate(ExpressionEvaluatorImpl.java:274)
	at org.apache.commons.el.ExpressionEvaluatorImpl.evaluate(ExpressionEvaluatorImpl.java:190)
	at org.apache.oozie.util.ELEvaluator.evaluate(ELEvaluator.java:204)
	at org.apache.oozie.coord.CoordELFunctions.evalAndWrap(CoordELFunctions.java:714)
	... 10 more
2016-04-05 13:59:28,146  WARN pool-6-thread-1 CallableQueueService$CallableWrapper - SERVER[lda01]
USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0000618-160404070324284-oozie-oozi-C] ACTION[0000618-160404070324284-oozie-oozi-C@3]
exception callable [coord_action_input], E1021: Coord Action Input Check Error: Unable to
evaluate :${dataIn('inputData0', 'null')}#${dataIn('inputData', 'null')}:
{noformat}

workflow.xml :
{noformat}
<workflow-app xmlns="uri:oozie:workflow:0.2" name="aggregator-wf">
    <start to="aggregator"/>
    <action name="aggregator">
        <map-reduce>
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <prepare>
                <delete path="${outputData}"/>
            </prepare>
            <configuration>
                <property>
                    <name>mapred.job.queue.name</name>
                    <value>${queueName}</value>
                </property>
                <property>
                    <name>mapred.mapper.class</name>
                    <value>org.apache.hadoop.mapred.lib.IdentityMapper</value>
                </property>
                <property>
                    <name>mapred.reducer.class</name>
                    <value>org.apache.hadoop.mapred.lib.IdentityReducer</value>
                </property>
                <property>
                    <name>mapred.map.tasks</name>
                    <value>1</value>
                </property>
                <property>
                    <name>mapred.input.dir</name>
                    <value>${inputData}</value>
                </property>
                <!--<property>-->
                    <!--<name>mapred.input.dir</name>-->
                    <!--<value>${inputData0}</value>-->
                <!--</property>-->
                <property>
                    <name>mapred.output.dir</name>
                    <value>${outputData}</value>
                </property>
            </configuration>
        </map-reduce>
        <ok to="end"/>
        <error to="fail"/>
    </action>
    <kill name="fail">
        <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>
    <end name="end"/>
</workflow-app>

{noformat}

  was:
Process schedule fails in dry run when entity contains optional dataSet.

Process definition :
{noformat}
{noformat}


{noformat}
{noformat}



{noformat}
{noformat}


> Process schedule fails when entity contains optional dataSet
> ------------------------------------------------------------
>
>                 Key: FALCON-1887
>                 URL: https://issues.apache.org/jira/browse/FALCON-1887
>             Project: Falcon
>          Issue Type: Bug
>          Components: process
>    Affects Versions: trunk, 0.10
>            Reporter: Pragya Mittal
>
> Process schedule fails in dry run when entity contains optional dataSet.
> Process definition :
> {noformat}
> <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
> <process name="OptionalInputTest-agregator-3" xmlns="uri:falcon:process:0.1">
>     <clusters>
>         <cluster name="OptionalInputTest-corp-999c05510">
>             <validity start="2010-01-02T01:00Z" end="2010-01-02T01:12Z"/>
>         </cluster>
>     </clusters>
>     <parallel>2</parallel>
>     <order>FIFO</order>
>     <frequency>minutes(5)</frequency>
>     <timezone>UTC</timezone>
>     <inputs>
>         <input name="inputData0" feed="OptionalInputTest-raaw-logs16-cfe063e3-input0"
start="now(0,-10)" end="now(0,0)" optional="true"/>
>         <input name="inputData" feed="OptionalInputTest-raaw-logs16-cfe063e3-input1"
start="now(0,-10)" end="now(0,0)" optional="false"/>
>     </inputs>
>     <outputs>
>         <output name="outputData" feed="OptionalInputTest-raaw-logs16-cfe063e3-output0"
instance="now(0,0)"/>
>     </outputs>
>     <properties>
>         <property name="queueName" value="default"/>
>         <property name="fileTime" value="${formatTime(dateOffset(instanceTime(), 1,
'DAY'), 'yyyy-MMM-dd')}"/>
>         <property name="user" value="${user()}"/>
>     </properties>
>     <workflow path="/tmp/falcon-regression/OptionalInputTest/aggregator"/>
>     <retry policy="periodic" delay="minutes(3)" attempts="3"/>
>     <ACL owner="pragya" group="dataqa" permission="*"/>
> </process>
> {noformat}
> While scheduling with/without skipDryRun option
> {noformat}
> falcon entity -type process -schedule -name OptionalInputTest-agregator-3
> ERROR: Bad Request;ua1/E0803 : E0803: IO error, Unable to evaluate :${dataIn('inputData0',
'null')}#${dataIn('inputData', 'null')}:
> CausedBy: E0803: IO error, Unable to evaluate :${dataIn('inputData0', 'null')}#${dataIn('inputData',
'null')}:
> falcon entity -type process -schedule -name OptionalInputTest-agregator-3 -skipDryRun
> ua1/OptionalInputTest-agregator-bcjk-1(process) scheduled successfully
> {noformat}
> Server logs :
> {noformat}
> 2016-04-05 14:03:57,659 ERROR - [816508486@qtp-811301908-51 - 9f673069-d551-4e0c-8514-49bdf138cca3:dataqa:POST//entities/schedule/process/OptionalInputTest-agregator-3]
~ Entity schedule failed for process: OptionalInputTest-agregator-3 (AbstractSchedulableEntityManager:105)
> org.apache.falcon.FalconException: E0803 : E0803: IO error, Unable to evaluate :${dataIn('inputData0',
'null')}#${dataIn('inputData', 'null')}:
> 	at org.apache.falcon.workflow.engine.OozieWorkflowEngine.dryRunInternal(OozieWorkflowEngine.java:240)
> 	at org.apache.falcon.workflow.engine.OozieWorkflowEngine.schedule(OozieWorkflowEngine.java:178)
> 	at org.apache.falcon.resource.AbstractSchedulableEntityManager.scheduleInternal(AbstractSchedulableEntityManager.java:103)
> 	at org.apache.falcon.resource.AbstractSchedulableEntityManager.schedule(AbstractSchedulableEntityManager.java:82)
> 	at org.apache.falcon.resource.SchedulableEntityManager.schedule(SchedulableEntityManager.java:262)
> 	at sun.reflect.GeneratedMethodAccessor101.invoke(Unknown Source)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:497)
> 	at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)
> 	at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:185)
> 	at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75)
> 	at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:288)
> 	at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
> 	at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108)
> 	at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
> 	at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84)
> 	at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469)
> 	at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400)
> 	at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349)
> 	at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339)
> 	at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416)
> 	at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:537)
> 	at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:699)
> 	at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
> 	at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
> 	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
> 	at org.apache.falcon.security.FalconAuthorizationFilter.doFilter(FalconAuthorizationFilter.java:108)
> 	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
> 	at org.apache.falcon.security.FalconAuthenticationFilter$2.doFilter(FalconAuthenticationFilter.java:188)
> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:572)
> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:542)
> 	at org.apache.falcon.security.FalconAuthenticationFilter.doFilter(FalconAuthenticationFilter.java:197)
> 	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
> 	at org.apache.falcon.security.FalconAuditFilter.doFilter(FalconAuditFilter.java:64)
> 	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
> 	at org.apache.falcon.security.HostnameFilter.doFilter(HostnameFilter.java:82)
> 	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
> 	at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
> 	at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
> 	at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
> 	at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:767)
> 	at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
> 	at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> 	at org.mortbay.jetty.Server.handle(Server.java:326)
> 	at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
> 	at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
> 	at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
> 	at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
> 	at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
> 	at org.mortbay.jetty.bio.SocketConnector$Connection.run(SocketConnector.java:228)
> 	at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
> Caused by: E0803 : E0803: IO error, Unable to evaluate :${dataIn('inputData0', 'null')}#${dataIn('inputData',
'null')}:
> 	at org.apache.oozie.client.OozieClient.handleError(OozieClient.java:612)
> 	at org.apache.oozie.client.OozieClient$JobSubmit.call(OozieClient.java:695)
> 	at org.apache.oozie.client.OozieClient$JobSubmit.call(OozieClient.java:665)
> 	at org.apache.oozie.client.OozieClient$ClientCallable.call(OozieClient.java:560)
> 	at org.apache.oozie.client.OozieClient.dryrun(OozieClient.java:829)
> 	at org.apache.oozie.client.ProxyOozieClient.access$401(ProxyOozieClient.java:48)
> 	at org.apache.oozie.client.ProxyOozieClient$4.call(ProxyOozieClient.java:177)
> 	at org.apache.oozie.client.ProxyOozieClient$4.call(ProxyOozieClient.java:174)
> 	at org.apache.oozie.client.OozieClient.doAs(OozieClient.java:244)
> 	at org.apache.oozie.client.ProxyOozieClient.dryrun(ProxyOozieClient.java:174)
> 	at org.apache.falcon.workflow.engine.OozieWorkflowEngine.dryRunInternal(OozieWorkflowEngine.java:238)
> 	... 50 more
> 2016-04-05 14:03:57,659 INFO  - [816508486@qtp-811301908-51 - 9f673069-d551-4e0c-8514-49bdf138cca3:dataqa:POST//entities/schedule/process/OptionalInputTest-agregator-3]
~ Successfully released lock on (process) OptionalInputTest-agregator-3 by 816508486@qtp-811301908-51
- 9f673069-d551-4e0c-8514-49bdf138cca3 (MemoryLocks:70)
> 2016-04-05 14:03:57,659 INFO  - [816508486@qtp-811301908-51 - 9f673069-d551-4e0c-8514-49bdf138cca3:dataqa:POST//entities/schedule/process/OptionalInputTest-agregator-3]
~ Memory lock released for (process) OptionalInputTest-agregator-3 (AbstractSchedulableEntityManager:110)
> 2016-04-05 14:03:57,659 ERROR - [816508486@qtp-811301908-51 - 9f673069-d551-4e0c-8514-49bdf138cca3:dataqa:POST//entities/schedule/process/OptionalInputTest-agregator-3]
~ Unable to schedule workflow (AbstractSchedulableEntityManager:85)
> org.apache.falcon.FalconWebException
> 	at org.apache.falcon.FalconWebException.newAPIException(FalconWebException.java:62)
> 	at org.apache.falcon.FalconWebException.newAPIException(FalconWebException.java:50)
> 	at org.apache.falcon.FalconWebException.newAPIException(FalconWebException.java:45)
> 	at org.apache.falcon.resource.AbstractSchedulableEntityManager.scheduleInternal(AbstractSchedulableEntityManager.java:106)
> 	at org.apache.falcon.resource.AbstractSchedulableEntityManager.schedule(AbstractSchedulableEntityManager.java:82)
> 	at org.apache.falcon.resource.SchedulableEntityManager.schedule(SchedulableEntityManager.java:262)
> 	at sun.reflect.GeneratedMethodAccessor101.invoke(Unknown Source)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:497)
> 	at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)
> 	at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:185)
> 	at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75)
> 	at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:288)
> 	at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
> 	at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108)
> 	at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
> 	at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84)
> 	at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469)
> 	at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400)
> 	at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349)
> 	at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339)
> 	at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416)
> 	at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:537)
> 	at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:699)
> 	at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
> 	at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
> 	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
> 	at org.apache.falcon.security.FalconAuthorizationFilter.doFilter(FalconAuthorizationFilter.java:108)
> 	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
> 	at org.apache.falcon.security.FalconAuthenticationFilter$2.doFilter(FalconAuthenticationFilter.java:188)
> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:572)
> 	at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:542)
> 	at org.apache.falcon.security.FalconAuthenticationFilter.doFilter(FalconAuthenticationFilter.java:197)
> 	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
> 	at org.apache.falcon.security.FalconAuditFilter.doFilter(FalconAuditFilter.java:64)
> 	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
> 	at org.apache.falcon.security.HostnameFilter.doFilter(HostnameFilter.java:82)
> 	at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
> 	at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
> 	at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
> 	at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
> 	at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:767)
> 	at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
> 	at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> 	at org.mortbay.jetty.Server.handle(Server.java:326)
> 	at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
> 	at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
> 	at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
> 	at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
> 	at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
> 	at org.mortbay.jetty.bio.SocketConnector$Connection.run(SocketConnector.java:228)
> 	at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
> 2016-04-05 14:03:57,660 DEBUG - [816508486@qtp-811301908-51 - 9f673069-d551-4e0c-8514-49bdf138cca3:]
~ Audit: dataqa/192.168.138.236 performed request http://192.168.138.236:15000/api/entities/schedule/process/OptionalInputTest-agregator-3?colo=ua1&&user.name=dataqa
(192.168.138.236) at time 2016-04-05T14:03Z (FalconAuditFilter:86)
> 2016-04-05 14:03:57,660 INFO  - [816508486@qtp-811301908-51 - 9f673069-d551-4e0c-8514-49bdf138cca3:]
~ Audit: dataqa@192.168.138.236 performed http://192.168.138.236:15000/api/entities/schedule/process/OptionalInputTest-agregator-3?colo=ua1&&user.name=dataqa
(192.168.138.236) at 2016-04-05T14:03Z (AUDIT:48)
> {noformat}
> Oozie logs :
> {noformat}
> 2016-04-05 13:59:28,145 ERROR pool-6-thread-1 CoordActionInputCheckXCommand - SERVER[lda01]
USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0000618-160404070324284-oozie-oozi-C] ACTION[0000618-160404070324284-oozie-oozi-C@3]
XException, 
> org.apache.oozie.command.CommandException: E1021: Coord Action Input Check Error: Unable
to evaluate :${dataIn('inputData0', 'null')}#${dataIn('inputData', 'null')}:
> 	at org.apache.oozie.command.coord.CoordActionInputCheckXCommand.execute(CoordActionInputCheckXCommand.java:279)
> 	at org.apache.oozie.command.coord.CoordActionInputCheckXCommand.execute(CoordActionInputCheckXCommand.java:71)
> 	at org.apache.oozie.command.XCommand.call(XCommand.java:286)
> 	at org.apache.oozie.service.CallableQueueService$CallableWrapper.run(CallableQueueService.java:175)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> Caused by: java.lang.Exception: Unable to evaluate :${dataIn('inputData0', 'null')}#${dataIn('inputData',
'null')}:
> 	at org.apache.oozie.coord.CoordELFunctions.evalAndWrap(CoordELFunctions.java:723)
> 	at org.apache.oozie.command.coord.CoordActionInputCheckXCommand.resolveTagContents(CoordActionInputCheckXCommand.java:398)
> 	at org.apache.oozie.command.coord.CoordActionInputCheckXCommand.materializeDataProperties(CoordActionInputCheckXCommand.java:379)
> 	at org.apache.oozie.command.coord.CoordActionInputCheckXCommand.resolveCoordConfiguration(CoordActionInputCheckXCommand.java:288)
> 	at org.apache.oozie.command.coord.CoordActionInputCheckXCommand.execute(CoordActionInputCheckXCommand.java:230)
> 	... 6 more
> Caused by: java.lang.NullPointerException
> 	at org.apache.oozie.extensions.OozieELExtensions.getDSElement(OozieELExtensions.java:124)
> 	at org.apache.oozie.extensions.OozieELExtensions.ph3_dataIn(OozieELExtensions.java:74)
> 	at sun.reflect.GeneratedMethodAccessor106.invoke(Unknown Source)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:497)
> 	at org.apache.commons.el.FunctionInvocation.evaluate(FunctionInvocation.java:172)
> 	at org.apache.commons.el.ExpressionString.evaluate(ExpressionString.java:114)
> 	at org.apache.commons.el.ExpressionEvaluatorImpl.evaluate(ExpressionEvaluatorImpl.java:274)
> 	at org.apache.commons.el.ExpressionEvaluatorImpl.evaluate(ExpressionEvaluatorImpl.java:190)
> 	at org.apache.oozie.util.ELEvaluator.evaluate(ELEvaluator.java:204)
> 	at org.apache.oozie.coord.CoordELFunctions.evalAndWrap(CoordELFunctions.java:714)
> 	... 10 more
> 2016-04-05 13:59:28,146  WARN pool-6-thread-1 CallableQueueService$CallableWrapper -
SERVER[lda01] USER[-] GROUP[-] TOKEN[-] APP[-] JOB[0000618-160404070324284-oozie-oozi-C] ACTION[0000618-160404070324284-oozie-oozi-C@3]
exception callable [coord_action_input], E1021: Coord Action Input Check Error: Unable to
evaluate :${dataIn('inputData0', 'null')}#${dataIn('inputData', 'null')}:
> {noformat}
> workflow.xml :
> {noformat}
> <workflow-app xmlns="uri:oozie:workflow:0.2" name="aggregator-wf">
>     <start to="aggregator"/>
>     <action name="aggregator">
>         <map-reduce>
>             <job-tracker>${jobTracker}</job-tracker>
>             <name-node>${nameNode}</name-node>
>             <prepare>
>                 <delete path="${outputData}"/>
>             </prepare>
>             <configuration>
>                 <property>
>                     <name>mapred.job.queue.name</name>
>                     <value>${queueName}</value>
>                 </property>
>                 <property>
>                     <name>mapred.mapper.class</name>
>                     <value>org.apache.hadoop.mapred.lib.IdentityMapper</value>
>                 </property>
>                 <property>
>                     <name>mapred.reducer.class</name>
>                     <value>org.apache.hadoop.mapred.lib.IdentityReducer</value>
>                 </property>
>                 <property>
>                     <name>mapred.map.tasks</name>
>                     <value>1</value>
>                 </property>
>                 <property>
>                     <name>mapred.input.dir</name>
>                     <value>${inputData}</value>
>                 </property>
>                 <!--<property>-->
>                     <!--<name>mapred.input.dir</name>-->
>                     <!--<value>${inputData0}</value>-->
>                 <!--</property>-->
>                 <property>
>                     <name>mapred.output.dir</name>
>                     <value>${outputData}</value>
>                 </property>
>             </configuration>
>         </map-reduce>
>         <ok to="end"/>
>         <error to="fail"/>
>     </action>
>     <kill name="fail">
>         <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
>     </kill>
>     <end name="end"/>
> </workflow-app>
> {noformat}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)


Mime
View raw message