spark-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Reynold Xin (JIRA)" <j...@apache.org>
Subject [jira] [Assigned] (SPARK-9127) Rand/Randn codegen fails with long seed
Date Wed, 29 Jul 2015 07:48:05 GMT

     [ https://issues.apache.org/jira/browse/SPARK-9127?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Reynold Xin reassigned SPARK-9127:
----------------------------------

    Assignee: Reynold Xin  (was: Wenchen Fan)

> Rand/Randn codegen fails with long seed
> ---------------------------------------
>
>                 Key: SPARK-9127
>                 URL: https://issues.apache.org/jira/browse/SPARK-9127
>             Project: Spark
>          Issue Type: Bug
>          Components: SQL
>            Reporter: Davies Liu
>            Assignee: Reynold Xin
>            Priority: Blocker
>
> {code}
> ERROR GenerateMutableProjection: failed to compile:
>       public Object generate(org.apache.spark.sql.catalyst.expressions.Expression[] expr)
{
>         return new SpecificProjection(expr);
>       }
>       class SpecificProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection
{
>         private org.apache.spark.sql.catalyst.expressions.Expression[] expressions =
null;
>         private org.apache.spark.sql.catalyst.expressions.MutableRow mutableRow = null;
>         private org.apache.spark.util.random.XORShiftRandom rng4 = new org.apache.spark.util.random.XORShiftRandom(-5419823303878592871
+ org.apache.spark.TaskContext.getPartitionId());
>         public SpecificProjection(org.apache.spark.sql.catalyst.expressions.Expression[]
expr) {
>           expressions = expr;
>           mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericMutableRow(2);
>         }
>         public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection
target(org.apache.spark.sql.catalyst.expressions.MutableRow row) {
>           mutableRow = row;
>           return this;
>         }
>         /* Provide immutable access to the last projected row. */
>         public InternalRow currentValue() {
>           return (InternalRow) mutableRow;
>         }
>         public Object apply(Object _i) {
>           InternalRow i = (InternalRow) _i;
>         boolean isNull0 = i.isNullAt(0);
>         long primitive1 = isNull0 ?
>             -1L : (i.getLong(0));
>           if(isNull0)
>             mutableRow.setNullAt(0);
>           else
>             mutableRow.setLong(0, primitive1);
>       final double primitive3 = rng4.nextDouble();
>           if(false)
>             mutableRow.setNullAt(1);
>           else
>             mutableRow.setDouble(1, primitive3);
>           return mutableRow;
>         }
>       }
> org.codehaus.commons.compiler.CompileException: Line 10, Column 117: Value of decimal
integer literal '5419823303878592871' is out of range
> 	at org.codehaus.janino.UnitCompiler.compileException(UnitCompiler.java:10473)
> 	at org.codehaus.janino.UnitCompiler.getConstantValue2(UnitCompiler.java:4696)
> 	at org.codehaus.janino.UnitCompiler.access$9200(UnitCompiler.java:185)
> 	at org.codehaus.janino.UnitCompiler$11.visitIntegerLiteral(UnitCompiler.java:4402)
> 	at org.codehaus.janino.Java$IntegerLiteral.accept(Java.java:4321)
> 	at org.codehaus.janino.UnitCompiler.getConstantValue(UnitCompiler.java:4427)
> 	at org.codehaus.janino.UnitCompiler.getNegatedConstantValue2(UnitCompiler.java:4856)
> 	at org.codehaus.janino.UnitCompiler.getNegatedConstantValue2(UnitCompiler.java:4890)
> 	at org.codehaus.janino.UnitCompiler.access$10400(UnitCompiler.java:185)
> 	at org.codehaus.janino.UnitCompiler$12.visitIntegerLiteral(UnitCompiler.java:4823)
> 	at org.codehaus.janino.Java$IntegerLiteral.accept(Java.java:4321)
> 	at org.codehaus.janino.UnitCompiler.getNegatedConstantValue(UnitCompiler.java:4848)
> 	at org.codehaus.janino.UnitCompiler.getConstantValue2(UnitCompiler.java:4451)
> 	at org.codehaus.janino.UnitCompiler.access$8800(UnitCompiler.java:185)
> 	at org.codehaus.janino.UnitCompiler$11.visitUnaryOperation(UnitCompiler.java:4393)
> 	at org.codehaus.janino.Java$UnaryOperation.accept(Java.java:3647)
> 	at org.codehaus.janino.UnitCompiler.getConstantValue(UnitCompiler.java:4427)
> 	at org.codehaus.janino.UnitCompiler.getConstantValue2(UnitCompiler.java:4498)
> 	at org.codehaus.janino.UnitCompiler.access$8900(UnitCompiler.java:185)
> 	at org.codehaus.janino.UnitCompiler$11.visitBinaryOperation(UnitCompiler.java:4394)
> 	at org.codehaus.janino.Java$BinaryOperation.accept(Java.java:3768)
> 	at org.codehaus.janino.UnitCompiler.getConstantValue(UnitCompiler.java:4427)
> 	at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4360)
> 	at org.codehaus.janino.UnitCompiler.invokeConstructor(UnitCompiler.java:6681)
> 	at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:4126)
> 	at org.codehaus.janino.UnitCompiler.access$7600(UnitCompiler.java:185)
> 	at org.codehaus.janino.UnitCompiler$10.visitNewClassInstance(UnitCompiler.java:3275)
> 	at org.codehaus.janino.Java$NewClassInstance.accept(Java.java:4085)
> 	at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:3290)
> 	at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4368)
> 	at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1659)
> 	at org.codehaus.janino.UnitCompiler.access$800(UnitCompiler.java:185)
> 	at org.codehaus.janino.UnitCompiler$4.visitFieldDeclaration(UnitCompiler.java:933)
> 	at org.codehaus.janino.Java$FieldDeclaration.accept(Java.java:1818)
> 	at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958)
> 	at org.codehaus.janino.UnitCompiler.initializeInstanceVariablesAndInvokeInstanceInitializers(UnitCompiler.java:6101)
> 	at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2284)
> 	at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:518)
> 	at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:658)
> 	at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:662)
> 	at org.codehaus.janino.UnitCompiler.access$600(UnitCompiler.java:185)
> 	at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:350)
> 	at org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1035)
> 	at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:354)
> 	at org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:769)
> 	at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:532)
> 	at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:393)
> 	at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:185)
> 	at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:347)
> 	at org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1139)
> 	at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:354)
> 	at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:322)
> 	at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:383)
> 	at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:315)
> 	at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:233)
> 	at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:192)
> 	at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:84)
> 	at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:77)
> 	at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:268)
> 	at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:292)
> 	at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:289)
> 	at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
> 	at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
> 	at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
> 	at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2257)
> 	at com.google.common.cache.LocalCache.get(LocalCache.java:4000)
> 	at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:4004)
> 	at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
> 	at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.compile(CodeGenerator.scala:256)
> 	at org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection$.create(GenerateMutableProjection.scala:89)
> 	at org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection$.create(GenerateMutableProjection.scala:29)
> 	at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:305)
> 	at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:302)
> 	at org.apache.spark.sql.execution.SparkPlan.newMutableProjection(SparkPlan.scala:186)
> 	at org.apache.spark.sql.execution.Project.buildProjection$lzycompute(basicOperators.scala:42)
> 	at org.apache.spark.sql.execution.Project.buildProjection(basicOperators.scala:42)
> 	at org.apache.spark.sql.execution.Project$$anonfun$1.apply(basicOperators.scala:45)
> 	at org.apache.spark.sql.execution.Project$$anonfun$1.apply(basicOperators.scala:44)
> 	at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:686)
> 	at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:686)
> 	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)
> 	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)
> 	at org.apache.spark.rdd.RDD.iterator(RDD.scala:244)
> 	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)
> 	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)
> 	at org.apache.spark.rdd.RDD.iterator(RDD.scala:244)
> 	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:35)
> 	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)
> 	at org.apache.spark.rdd.RDD.iterator(RDD.scala:244)
> 	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:63)
> 	at org.apache.spark.scheduler.Task.run(Task.scala:70)
> 	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> 	at java.lang.Thread.run(Thread.java:745)
> 15/07/16 21:56:16 ERROR Project: Failed to generate mutable projection, fallback to interpreted
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org


Mime
View raw message