spark-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Sjoerd Mulder <sjoerdmul...@gmail.com>
Subject Re: If you use Spark 1.5 and disabled Tungsten mode ...
Date Tue, 27 Oct 2015 19:56:09 GMT
I have disabled it because of it started generating ERROR's when upgrading
from Spark 1.4 to 1.5.1

2015-10-27T20:50:11.574+0100 ERROR TungstenSort.newOrdering() - Failed to
generate ordering, fallback to interpreted
java.util.concurrent.ExecutionException: java.lang.Exception: failed to
compile: org.codehaus.commons.compiler.CompileException: Line 15, Column 9:
Invalid character input "@" (character code 64)

public SpecificOrdering
generate(org.apache.spark.sql.catalyst.expressions.Expression[] expr) {
  return new SpecificOrdering(expr);
}

class SpecificOrdering extends
org.apache.spark.sql.catalyst.expressions.codegen.BaseOrdering {

  private org.apache.spark.sql.catalyst.expressions.Expression[]
expressions;



  public
SpecificOrdering(org.apache.spark.sql.catalyst.expressions.Expression[]
expr) {
    expressions = expr;

  }

  @Override
  public int compare(InternalRow a, InternalRow b) {
    InternalRow i = null;  // Holds current row being evaluated.

    i = a;
    boolean isNullA2;
    long primitiveA3;
    {
      /* input[2, LongType] */

      boolean isNull0 = i.isNullAt(2);
      long primitive1 = isNull0 ? -1L : (i.getLong(2));

      isNullA2 = isNull0;
      primitiveA3 = primitive1;
    }
    i = b;
    boolean isNullB4;
    long primitiveB5;
    {
      /* input[2, LongType] */

      boolean isNull0 = i.isNullAt(2);
      long primitive1 = isNull0 ? -1L : (i.getLong(2));

      isNullB4 = isNull0;
      primitiveB5 = primitive1;
    }
    if (isNullA2 && isNullB4) {
      // Nothing
    } else if (isNullA2) {
      return 1;
    } else if (isNullB4) {
      return -1;
    } else {
      int comp = (primitiveA3 > primitiveB5 ? 1 : primitiveA3 < primitiveB5
? -1 : 0);
      if (comp != 0) {
        return -comp;
      }
    }

    return 0;
  }
}

at
org.spark-project.guava.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:306)
at
org.spark-project.guava.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:293)
at
org.spark-project.guava.util.concurrent.AbstractFuture.get(AbstractFuture.java:116)
at
org.spark-project.guava.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:135)
at
org.spark-project.guava.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2410)
at
org.spark-project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2380)
at
org.spark-project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
at
org.spark-project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
at org.spark-project.guava.cache.LocalCache.get(LocalCache.java:4000)
at org.spark-project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
at
org.spark-project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.compile(CodeGenerator.scala:362)
at
org.apache.spark.sql.catalyst.expressions.codegen.GenerateOrdering$.create(GenerateOrdering.scala:139)
at
org.apache.spark.sql.catalyst.expressions.codegen.GenerateOrdering$.create(GenerateOrdering.scala:37)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:425)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:422)
at org.apache.spark.sql.execution.SparkPlan.newOrdering(SparkPlan.scala:294)
at org.apache.spark.sql.execution.TungstenSort.org
$apache$spark$sql$execution$TungstenSort$$preparePartition$1(sort.scala:131)
at
org.apache.spark.sql.execution.TungstenSort$$anonfun$doExecute$3.apply(sort.scala:169)
at
org.apache.spark.sql.execution.TungstenSort$$anonfun$doExecute$3.apply(sort.scala:169)
at
org.apache.spark.rdd.MapPartitionsWithPreparationRDD.compute(MapPartitionsWithPreparationRDD.scala:59)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:264)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:264)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:264)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
at org.apache.spark.scheduler.Task.run(Task.scala:88)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)


2015-10-14 21:00 GMT+02:00 Reynold Xin <rxin@databricks.com>:

> Can you reply to this email and provide us with reasons why you disable it?
>
> Thanks.
>
>

Mime
View raw message