spark-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Ray Qiu (JIRA)" <j...@apache.org>
Subject [jira] [Issue Comment Deleted] (SPARK-18125) Spark generated code causes CompileException when groupByKey, reduceGroups and map(_._2) are used
Date Thu, 27 Oct 2016 23:55:00 GMT

     [ https://issues.apache.org/jira/browse/SPARK-18125?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Ray Qiu updated SPARK-18125:
----------------------------
    Comment: was deleted

(was: Same thing works fine in 2.0.0





-- 
Regards,
Ray
)

> Spark generated code causes CompileException when groupByKey, reduceGroups and map(_._2)
are used
> -------------------------------------------------------------------------------------------------
>
>                 Key: SPARK-18125
>                 URL: https://issues.apache.org/jira/browse/SPARK-18125
>             Project: Spark
>          Issue Type: Bug
>    Affects Versions: 2.0.1
>            Reporter: Ray Qiu
>            Priority: Critical
>
> Code logic looks like this:
> {noformat}
>             .groupByKey
>             .reduceGroups
>             .map(_._2)
> {noformat}
> Works fine with 2.0.0.
> 2.0.1 error Message: 
> {noformat}
> Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: failed to compile:
org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 206, Column 123:
Unknown variable or type "value4"
> /* 001 */ public java.lang.Object generate(Object[] references) {
> /* 002 */   return new SpecificMutableProjection(references);
> /* 003 */ }
> /* 004 */
> /* 005 */ class SpecificMutableProjection extends org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection
{
> /* 006 */
> /* 007 */   private Object[] references;
> /* 008 */   private MutableRow mutableRow;
> /* 009 */   private Object[] values;
> /* 010 */   private java.lang.String errMsg;
> /* 011 */   private java.lang.String errMsg1;
> /* 012 */   private boolean MapObjects_loopIsNull1;
> /* 013 */   private io.mistnet.analytics.lib.ConnLog MapObjects_loopValue0;
> /* 014 */   private java.lang.String errMsg2;
> /* 015 */   private Object[] values1;
> /* 016 */   private boolean MapObjects_loopIsNull3;
> /* 017 */   private java.lang.String MapObjects_loopValue2;
> /* 018 */   private boolean isNull_0;
> /* 019 */   private boolean value_0;
> /* 020 */   private boolean isNull_1;
> /* 021 */   private InternalRow value_1;
> /* 022 */
> /* 023 */   private void apply_4(InternalRow i) {
> /* 024 */
> /* 025 */     boolean isNull52 = MapObjects_loopIsNull1;
> /* 026 */     final double value52 = isNull52 ? -1.0 : MapObjects_loopValue0.ts();
> /* 027 */     if (isNull52) {
> /* 028 */       values1[8] = null;
> /* 029 */     } else {
> /* 030 */       values1[8] = value52;
> /* 031 */     }
> /* 032 */     boolean isNull54 = MapObjects_loopIsNull1;
> /* 033 */     final java.lang.String value54 = isNull54 ? null : (java.lang.String) MapObjects_loopValue0.uid();
> /* 034 */     isNull54 = value54 == null;
> /* 035 */     boolean isNull53 = isNull54;
> /* 036 */     final UTF8String value53 = isNull53 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value54);
> /* 037 */     isNull53 = value53 == null;
> /* 038 */     if (isNull53) {
> /* 039 */       values1[9] = null;
> /* 040 */     } else {
> /* 041 */       values1[9] = value53;
> /* 042 */     }
> /* 043 */     boolean isNull56 = MapObjects_loopIsNull1;
> /* 044 */     final java.lang.String value56 = isNull56 ? null : (java.lang.String) MapObjects_loopValue0.src();
> /* 045 */     isNull56 = value56 == null;
> /* 046 */     boolean isNull55 = isNull56;
> /* 047 */     final UTF8String value55 = isNull55 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value56);
> /* 048 */     isNull55 = value55 == null;
> /* 049 */     if (isNull55) {
> /* 050 */       values1[10] = null;
> /* 051 */     } else {
> /* 052 */       values1[10] = value55;
> /* 053 */     }
> /* 054 */   }
> /* 055 */
> /* 056 */
> /* 057 */   private void apply_7(InternalRow i) {
> /* 058 */
> /* 059 */     boolean isNull69 = MapObjects_loopIsNull1;
> /* 060 */     final scala.Option value69 = isNull69 ? null : (scala.Option) MapObjects_loopValue0.orig_bytes();
> /* 061 */     isNull69 = value69 == null;
> /* 062 */
> /* 063 */     final boolean isNull68 = isNull69 || value69.isEmpty();
> /* 064 */     long value68 = isNull68 ?
> /* 065 */     -1L : (Long) value69.get();
> /* 066 */     if (isNull68) {
> /* 067 */       values1[17] = null;
> /* 068 */     } else {
> /* 069 */       values1[17] = value68;
> /* 070 */     }
> /* 071 */     boolean isNull71 = MapObjects_loopIsNull1;
> /* 072 */     final scala.Option value71 = isNull71 ? null : (scala.Option) MapObjects_loopValue0.resp_bytes();
> /* 073 */     isNull71 = value71 == null;
> /* 074 */
> /* 075 */     final boolean isNull70 = isNull71 || value71.isEmpty();
> /* 076 */     long value70 = isNull70 ?
> /* 077 */     -1L : (Long) value71.get();
> /* 078 */     if (isNull70) {
> /* 079 */       values1[18] = null;
> /* 080 */     } else {
> /* 081 */       values1[18] = value70;
> /* 082 */     }
> /* 083 */     boolean isNull74 = MapObjects_loopIsNull1;
> /* 084 */     final scala.Option value74 = isNull74 ? null : (scala.Option) MapObjects_loopValue0.conn_state();
> /* 085 */     isNull74 = value74 == null;
> /* 086 */
> /* 087 */     final boolean isNull73 = isNull74 || value74.isEmpty();
> /* 088 */     java.lang.String value73 = isNull73 ?
> /* 089 */     null : (java.lang.String) value74.get();
> /* 090 */     boolean isNull72 = isNull73;
> /* 091 */     final UTF8String value72 = isNull72 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value73);
> /* 092 */     isNull72 = value72 == null;
> /* 093 */     if (isNull72) {
> /* 094 */       values1[19] = null;
> /* 095 */     } else {
> /* 096 */       values1[19] = value72;
> /* 097 */     }
> /* 098 */   }
> /* 099 */
> /* 100 */
> /* 101 */   private void apply_1(InternalRow i) {
> /* 102 */
> /* 103 */     boolean isNull37 = MapObjects_loopIsNull1;
> /* 104 */     final scala.Option value37 = isNull37 ? null : (scala.Option) MapObjects_loopValue0.sensor_name();
> /* 105 */     isNull37 = value37 == null;
> /* 106 */
> /* 107 */     final boolean isNull36 = isNull37 || value37.isEmpty();
> /* 108 */     java.lang.String value36 = isNull36 ?
> /* 109 */     null : (java.lang.String) value37.get();
> /* 110 */     boolean isNull35 = isNull36;
> /* 111 */     final UTF8String value35 = isNull35 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value36);
> /* 112 */     isNull35 = value35 == null;
> /* 113 */     if (isNull35) {
> /* 114 */       values1[2] = null;
> /* 115 */     } else {
> /* 116 */       values1[2] = value35;
> /* 117 */     }
> /* 118 */     boolean isNull40 = MapObjects_loopIsNull1;
> /* 119 */     final scala.Option value40 = isNull40 ? null : (scala.Option) MapObjects_loopValue0.ioa_uuid();
> /* 120 */     isNull40 = value40 == null;
> /* 121 */
> /* 122 */     final boolean isNull39 = isNull40 || value40.isEmpty();
> /* 123 */     java.lang.String value39 = isNull39 ?
> /* 124 */     null : (java.lang.String) value40.get();
> /* 125 */     boolean isNull38 = isNull39;
> /* 126 */     final UTF8String value38 = isNull38 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value39);
> /* 127 */     isNull38 = value38 == null;
> /* 128 */     if (isNull38) {
> /* 129 */       values1[3] = null;
> /* 130 */     } else {
> /* 131 */       values1[3] = value38;
> /* 132 */     }
> /* 133 */   }
> /* 134 */
> /* 135 */
> /* 136 */   private void apply_12(InternalRow i) {
> /* 137 */
> /* 138 */     boolean isNull98 = MapObjects_loopIsNull1;
> /* 139 */     final scala.Option value98 = isNull98 ? null : (scala.Option) MapObjects_loopValue0.cc();
> /* 140 */     isNull98 = value98 == null;
> /* 141 */
> /* 142 */     final boolean isNull97 = isNull98 || value98.isEmpty();
> /* 143 */     java.lang.String value97 = isNull97 ?
> /* 144 */     null : (java.lang.String) value98.get();
> /* 145 */     boolean isNull96 = isNull97;
> /* 146 */     final UTF8String value96 = isNull96 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value97);
> /* 147 */     isNull96 = value96 == null;
> /* 148 */     if (isNull96) {
> /* 149 */       values1[29] = null;
> /* 150 */     } else {
> /* 151 */       values1[29] = value96;
> /* 152 */     }
> /* 153 */     boolean isNull101 = MapObjects_loopIsNull1;
> /* 154 */     final scala.Option value101 = isNull101 ? null : (scala.Option) MapObjects_loopValue0.location();
> /* 155 */     isNull101 = value101 == null;
> /* 156 */
> /* 157 */     final boolean isNull100 = isNull101 || value101.isEmpty();
> /* 158 */     java.lang.String value100 = isNull100 ?
> /* 159 */     null : (java.lang.String) value101.get();
> /* 160 */     boolean isNull99 = isNull100;
> /* 161 */     final UTF8String value99 = isNull99 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value100);
> /* 162 */     isNull99 = value99 == null;
> /* 163 */     if (isNull99) {
> /* 164 */       values1[30] = null;
> /* 165 */     } else {
> /* 166 */       values1[30] = value99;
> /* 167 */     }
> /* 168 */   }
> /* 169 */
> /* 170 */
> /* 171 */   private void apply_9(InternalRow i) {
> /* 172 */
> /* 173 */     boolean isNull83 = MapObjects_loopIsNull1;
> /* 174 */     final scala.Option value83 = isNull83 ? null : (scala.Option) MapObjects_loopValue0.history();
> /* 175 */     isNull83 = value83 == null;
> /* 176 */
> /* 177 */     final boolean isNull82 = isNull83 || value83.isEmpty();
> /* 178 */     java.lang.String value82 = isNull82 ?
> /* 179 */     null : (java.lang.String) value83.get();
> /* 180 */     boolean isNull81 = isNull82;
> /* 181 */     final UTF8String value81 = isNull81 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value82);
> /* 182 */     isNull81 = value81 == null;
> /* 183 */     if (isNull81) {
> /* 184 */       values1[23] = null;
> /* 185 */     } else {
> /* 186 */       values1[23] = value81;
> /* 187 */     }
> /* 188 */     boolean isNull85 = MapObjects_loopIsNull1;
> /* 189 */     final scala.Option value85 = isNull85 ? null : (scala.Option) MapObjects_loopValue0.orig_pkts();
> /* 190 */     isNull85 = value85 == null;
> /* 191 */
> /* 192 */     final boolean isNull84 = isNull85 || value85.isEmpty();
> /* 193 */     long value84 = isNull84 ?
> /* 194 */     -1L : (Long) value85.get();
> /* 195 */     if (isNull84) {
> /* 196 */       values1[24] = null;
> /* 197 */     } else {
> /* 198 */       values1[24] = value84;
> /* 199 */     }
> /* 200 */   }
> /* 201 */
> /* 202 */
> /* 203 */   private void apply1_1(InternalRow i) {
> /* 204 */
> /* 205 */     boolean isNull25 = false;
> /* 206 */     final io.mistnet.analytics.scan.SrcDstGrouped value25 = isNull25 ? null
: (io.mistnet.analytics.scan.SrcDstGrouped) value4._2();
> /* 207 */     isNull25 = value25 == null;
> /* 208 */
> /* 209 */     if (isNull25) {
> /* 210 */       throw new RuntimeException(errMsg2);
> /* 211 */     }
> /* 212 */
> /* 213 */     boolean isNull23 = false;
> /* 214 */     final scala.collection.Seq value23 = isNull23 ? null : (scala.collection.Seq)
value25.cs();
> /* 215 */     isNull23 = value23 == null;
> /* 216 */     ArrayData value22 = null;
> /* 217 */
> /* 218 */     if (!isNull23) {
> /* 219 */
> /* 220 */       InternalRow[] convertedArray1 = null;
> /* 221 */       int dataLength1 = value23.size();
> /* 222 */       convertedArray1 = new InternalRow[dataLength1];
> /* 223 */
> /* 224 */       int loopIndex1 = 0;
> /* 225 */       while (loopIndex1 < dataLength1) {
> /* 226 */         MapObjects_loopValue0 = (io.mistnet.analytics.lib.ConnLog) (value23.apply(loopIndex1));
> /* 227 */         MapObjects_loopIsNull1 = MapObjects_loopValue0 == null;
> /* 228 */
> /* 229 */
> /* 230 */         boolean isNull26 = false;
> /* 231 */         InternalRow value26 = null;
> /* 232 */         if (!false && MapObjects_loopIsNull1) {
> /* 233 */
> /* 234 */           final InternalRow value28 = null;
> /* 235 */           isNull26 = true;
> /* 236 */           value26 = value28;
> /* 237 */         } else {
> /* 238 */
> /* 239 */           boolean isNull29 = false;
> /* 240 */           values1 = new Object[31];apply_0(i);
> /* 241 */           apply_1(i);
> /* 242 */           apply_2(i);
> /* 243 */           apply_3(i);
> /* 244 */           apply_4(i);
> /* 245 */           apply_5(i);
> /* 246 */           apply_6(i);
> /* 247 */           apply_7(i);
> /* 248 */           apply_8(i);
> /* 249 */           apply_9(i);
> /* 250 */           apply_10(i);
> /* 251 */           apply_11(i);
> /* 252 */           apply_12(i);
> /* 253 */           final InternalRow value29 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values1);
> /* 254 */           this.values1 = null;
> /* 255 */           isNull26 = isNull29;
> /* 256 */           value26 = value29;
> /* 257 */         }
> /* 258 */         if (isNull26) {
> /* 259 */           convertedArray1[loopIndex1] = null;
> /* 260 */         } else {
> /* 261 */           convertedArray1[loopIndex1] = value26 instanceof UnsafeRow? value26.copy()
: value26;
> /* 262 */         }
> /* 263 */
> /* 264 */         loopIndex1 += 1;
> /* 265 */       }
> /* 266 */
> /* 267 */       value22 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray1);
> /* 268 */     }
> /* 269 */     if (isNull23) {
> /* 270 */       values[2] = null;
> /* 271 */     } else {
> /* 272 */       values[2] = value22;
> /* 273 */     }
> /* 274 */   }
> /* 275 */
> /* 276 */
> /* 277 */   private void apply_3(InternalRow i) {
> /* 278 */
> /* 279 */     boolean isNull49 = MapObjects_loopIsNull1;
> /* 280 */     final scala.Option value49 = isNull49 ? null : (scala.Option) MapObjects_loopValue0.date();
> /* 281 */     isNull49 = value49 == null;
> /* 282 */
> /* 283 */     final boolean isNull48 = isNull49 || value49.isEmpty();
> /* 284 */     java.lang.String value48 = isNull48 ?
> /* 285 */     null : (java.lang.String) value49.get();
> /* 286 */     boolean isNull47 = isNull48;
> /* 287 */     final UTF8String value47 = isNull47 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value48);
> /* 288 */     isNull47 = value47 == null;
> /* 289 */     if (isNull47) {
> /* 290 */       values1[6] = null;
> /* 291 */     } else {
> /* 292 */       values1[6] = value47;
> /* 293 */     }
> /* 294 */     boolean isNull51 = MapObjects_loopIsNull1;
> /* 295 */     final scala.Option value51 = isNull51 ? null : (scala.Option) MapObjects_loopValue0.hour();
> /* 296 */     isNull51 = value51 == null;
> /* 297 */
> /* 298 */     final boolean isNull50 = isNull51 || value51.isEmpty();
> /* 299 */     int value50 = isNull50 ?
> /* 300 */     -1 : (Integer) value51.get();
> /* 301 */     if (isNull50) {
> /* 302 */       values1[7] = null;
> /* 303 */     } else {
> /* 304 */       values1[7] = value50;
> /* 305 */     }
> /* 306 */   }
> /* 307 */
> /* 308 */
> /* 309 */   private void apply_6(InternalRow i) {
> /* 310 */
> /* 311 */     boolean isNull65 = MapObjects_loopIsNull1;
> /* 312 */     final scala.Option value65 = isNull65 ? null : (scala.Option) MapObjects_loopValue0.service();
> /* 313 */     isNull65 = value65 == null;
> /* 314 */
> /* 315 */     final boolean isNull64 = isNull65 || value65.isEmpty();
> /* 316 */     java.lang.String value64 = isNull64 ?
> /* 317 */     null : (java.lang.String) value65.get();
> /* 318 */     boolean isNull63 = isNull64;
> /* 319 */     final UTF8String value63 = isNull63 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value64);
> /* 320 */     isNull63 = value63 == null;
> /* 321 */     if (isNull63) {
> /* 322 */       values1[15] = null;
> /* 323 */     } else {
> /* 324 */       values1[15] = value63;
> /* 325 */     }
> /* 326 */     boolean isNull67 = MapObjects_loopIsNull1;
> /* 327 */     final scala.Option value67 = isNull67 ? null : (scala.Option) MapObjects_loopValue0.duration();
> /* 328 */     isNull67 = value67 == null;
> /* 329 */
> /* 330 */     final boolean isNull66 = isNull67 || value67.isEmpty();
> /* 331 */     double value66 = isNull66 ?
> /* 332 */     -1.0 : (Double) value67.get();
> /* 333 */     if (isNull66) {
> /* 334 */       values1[16] = null;
> /* 335 */     } else {
> /* 336 */       values1[16] = value66;
> /* 337 */     }
> /* 338 */   }
> /* 339 */
> /* 340 */
> /* 341 */   private void apply_0(InternalRow i) {
> /* 342 */
> /* 343 */     boolean isNull32 = MapObjects_loopIsNull1;
> /* 344 */     final scala.Option value32 = isNull32 ? null : (scala.Option) MapObjects_loopValue0.log_type();
> /* 345 */     isNull32 = value32 == null;
> /* 346 */
> /* 347 */     final boolean isNull31 = isNull32 || value32.isEmpty();
> /* 348 */     java.lang.String value31 = isNull31 ?
> /* 349 */     null : (java.lang.String) value32.get();
> /* 350 */     boolean isNull30 = isNull31;
> /* 351 */     final UTF8String value30 = isNull30 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value31);
> /* 352 */     isNull30 = value30 == null;
> /* 353 */     if (isNull30) {
> /* 354 */       values1[0] = null;
> /* 355 */     } else {
> /* 356 */       values1[0] = value30;
> /* 357 */     }
> /* 358 */     boolean isNull34 = MapObjects_loopIsNull1;
> /* 359 */     final scala.Option value34 = isNull34 ? null : (scala.Option) MapObjects_loopValue0.timestamp();
> /* 360 */     isNull34 = value34 == null;
> /* 361 */
> /* 362 */     final boolean isNull33 = isNull34 || value34.isEmpty();
> /* 363 */     long value33 = isNull33 ?
> /* 364 */     -1L : (Long) value34.get();
> /* 365 */     if (isNull33) {
> /* 366 */       values1[1] = null;
> /* 367 */     } else {
> /* 368 */       values1[1] = value33;
> /* 369 */     }
> /* 370 */   }
> /* 371 */
> /* 372 */
> /* 373 */   private void apply_11(InternalRow i) {
> /* 374 */
> /* 375 */     boolean isNull94 = MapObjects_loopIsNull1;
> /* 376 */     final scala.Option value94 = isNull94 ? null : (scala.Option) MapObjects_loopValue0.tunnel_parents();
> /* 377 */     isNull94 = value94 == null;
> /* 378 */
> /* 379 */     final boolean isNull93 = isNull94 || value94.isEmpty();
> /* 380 */     scala.collection.Seq value93 = isNull93 ?
> /* 381 */     null : (scala.collection.Seq) value94.get();
> /* 382 */     ArrayData value92 = null;
> /* 383 */
> /* 384 */     if (!isNull93) {
> /* 385 */
> /* 386 */       UTF8String[] convertedArray = null;
> /* 387 */       int dataLength = value93.size();
> /* 388 */       convertedArray = new UTF8String[dataLength];
> /* 389 */
> /* 390 */       int loopIndex = 0;
> /* 391 */       while (loopIndex < dataLength) {
> /* 392 */         MapObjects_loopValue2 = (java.lang.String) (value93.apply(loopIndex));
> /* 393 */         MapObjects_loopIsNull3 = MapObjects_loopValue2 == null;
> /* 394 */
> /* 395 */
> /* 396 */         boolean isNull95 = MapObjects_loopIsNull3;
> /* 397 */         final UTF8String value95 = isNull95 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(MapObjects_loopValue2);
> /* 398 */         isNull95 = value95 == null;
> /* 399 */         if (isNull95) {
> /* 400 */           convertedArray[loopIndex] = null;
> /* 401 */         } else {
> /* 402 */           convertedArray[loopIndex] = value95;
> /* 403 */         }
> /* 404 */
> /* 405 */         loopIndex += 1;
> /* 406 */       }
> /* 407 */
> /* 408 */       value92 = new org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray);
> /* 409 */     }
> /* 410 */     if (isNull93) {
> /* 411 */       values1[28] = null;
> /* 412 */     } else {
> /* 413 */       values1[28] = value92;
> /* 414 */     }
> /* 415 */   }
> /* 416 */
> /* 417 */
> /* 418 */   private void apply_8(InternalRow i) {
> /* 419 */
> /* 420 */     boolean isNull76 = MapObjects_loopIsNull1;
> /* 421 */     final scala.Option value76 = isNull76 ? null : (scala.Option) MapObjects_loopValue0.local_orig();
> /* 422 */     isNull76 = value76 == null;
> /* 423 */
> /* 424 */     final boolean isNull75 = isNull76 || value76.isEmpty();
> /* 425 */     boolean value75 = isNull75 ?
> /* 426 */     false : (Boolean) value76.get();
> /* 427 */     if (isNull75) {
> /* 428 */       values1[20] = null;
> /* 429 */     } else {
> /* 430 */       values1[20] = value75;
> /* 431 */     }
> /* 432 */     boolean isNull78 = MapObjects_loopIsNull1;
> /* 433 */     final scala.Option value78 = isNull78 ? null : (scala.Option) MapObjects_loopValue0.local_resp();
> /* 434 */     isNull78 = value78 == null;
> /* 435 */
> /* 436 */     final boolean isNull77 = isNull78 || value78.isEmpty();
> /* 437 */     boolean value77 = isNull77 ?
> /* 438 */     false : (Boolean) value78.get();
> /* 439 */     if (isNull77) {
> /* 440 */       values1[21] = null;
> /* 441 */     } else {
> /* 442 */       values1[21] = value77;
> /* 443 */     }
> /* 444 */     boolean isNull80 = MapObjects_loopIsNull1;
> /* 445 */     final scala.Option value80 = isNull80 ? null : (scala.Option) MapObjects_loopValue0.missed_bytes();
> /* 446 */     isNull80 = value80 == null;
> /* 447 */
> /* 448 */     final boolean isNull79 = isNull80 || value80.isEmpty();
> /* 449 */     long value79 = isNull79 ?
> /* 450 */     -1L : (Long) value80.get();
> /* 451 */     if (isNull79) {
> /* 452 */       values1[22] = null;
> /* 453 */     } else {
> /* 454 */       values1[22] = value79;
> /* 455 */     }
> /* 456 */   }
> /* 457 */
> /* 458 */
> /* 459 */   private void apply1_0(InternalRow i) {
> /* 460 */
> /* 461 */     boolean isNull17 = false;
> /* 462 */     final io.mistnet.analytics.scan.SrcDstGrouped value17 = isNull17 ? null
: (io.mistnet.analytics.scan.SrcDstGrouped) value4._2();
> /* 463 */     isNull17 = value17 == null;
> /* 464 */
> /* 465 */     if (isNull17) {
> /* 466 */       throw new RuntimeException(errMsg);
> /* 467 */     }
> /* 468 */
> /* 469 */     boolean isNull15 = false;
> /* 470 */     final java.lang.String value15 = isNull15 ? null : (java.lang.String) value17.src();
> /* 471 */     isNull15 = value15 == null;
> /* 472 */     boolean isNull14 = isNull15;
> /* 473 */     final UTF8String value14 = isNull14 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value15);
> /* 474 */     isNull14 = value14 == null;
> /* 475 */     if (isNull14) {
> /* 476 */       values[0] = null;
> /* 477 */     } else {
> /* 478 */       values[0] = value14;
> /* 479 */     }
> /* 480 */     boolean isNull21 = false;
> /* 481 */     final io.mistnet.analytics.scan.SrcDstGrouped value21 = isNull21 ? null
: (io.mistnet.analytics.scan.SrcDstGrouped) value4._2();
> /* 482 */     isNull21 = value21 == null;
> /* 483 */
> /* 484 */     if (isNull21) {
> /* 485 */       throw new RuntimeException(errMsg1);
> /* 486 */     }
> /* 487 */
> /* 488 */     boolean isNull19 = false;
> /* 489 */     final java.lang.String value19 = isNull19 ? null : (java.lang.String) value21.dest();
> /* 490 */     isNull19 = value19 == null;
> /* 491 */     boolean isNull18 = isNull19;
> /* 492 */     final UTF8String value18 = isNull18 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value19);
> /* 493 */     isNull18 = value18 == null;
> /* 494 */     if (isNull18) {
> /* 495 */       values[1] = null;
> /* 496 */     } else {
> /* 497 */       values[1] = value18;
> /* 498 */     }
> /* 499 */   }
> /* 500 */
> /* 501 */
> /* 502 */   private void apply_2(InternalRow i) {
> /* 503 */
> /* 504 */     boolean isNull43 = MapObjects_loopIsNull1;
> /* 505 */     final scala.Option value43 = isNull43 ? null : (scala.Option) MapObjects_loopValue0.user_uuid();
> /* 506 */     isNull43 = value43 == null;
> /* 507 */
> /* 508 */     final boolean isNull42 = isNull43 || value43.isEmpty();
> /* 509 */     java.lang.String value42 = isNull42 ?
> /* 510 */     null : (java.lang.String) value43.get();
> /* 511 */     boolean isNull41 = isNull42;
> /* 512 */     final UTF8String value41 = isNull41 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value42);
> /* 513 */     isNull41 = value41 == null;
> /* 514 */     if (isNull41) {
> /* 515 */       values1[4] = null;
> /* 516 */     } else {
> /* 517 */       values1[4] = value41;
> /* 518 */     }
> /* 519 */     boolean isNull46 = MapObjects_loopIsNull1;
> /* 520 */     final scala.Option value46 = isNull46 ? null : (scala.Option) MapObjects_loopValue0.host_uuid();
> /* 521 */     isNull46 = value46 == null;
> /* 522 */
> /* 523 */     final boolean isNull45 = isNull46 || value46.isEmpty();
> /* 524 */     java.lang.String value45 = isNull45 ?
> /* 525 */     null : (java.lang.String) value46.get();
> /* 526 */     boolean isNull44 = isNull45;
> /* 527 */     final UTF8String value44 = isNull44 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value45);
> /* 528 */     isNull44 = value44 == null;
> /* 529 */     if (isNull44) {
> /* 530 */       values1[5] = null;
> /* 531 */     } else {
> /* 532 */       values1[5] = value44;
> /* 533 */     }
> /* 534 */   }
> /* 535 */
> /* 536 */
> /* 537 */   private void apply_5(InternalRow i) {
> /* 538 */
> /* 539 */     boolean isNull57 = MapObjects_loopIsNull1;
> /* 540 */     final int value57 = isNull57 ? -1 : MapObjects_loopValue0.src_port();
> /* 541 */     if (isNull57) {
> /* 542 */       values1[11] = null;
> /* 543 */     } else {
> /* 544 */       values1[11] = value57;
> /* 545 */     }
> /* 546 */     boolean isNull59 = MapObjects_loopIsNull1;
> /* 547 */     final java.lang.String value59 = isNull59 ? null : (java.lang.String) MapObjects_loopValue0.dest();
> /* 548 */     isNull59 = value59 == null;
> /* 549 */     boolean isNull58 = isNull59;
> /* 550 */     final UTF8String value58 = isNull58 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value59);
> /* 551 */     isNull58 = value58 == null;
> /* 552 */     if (isNull58) {
> /* 553 */       values1[12] = null;
> /* 554 */     } else {
> /* 555 */       values1[12] = value58;
> /* 556 */     }
> /* 557 */     boolean isNull60 = MapObjects_loopIsNull1;
> /* 558 */     final int value60 = isNull60 ? -1 : MapObjects_loopValue0.dest_port();
> /* 559 */     if (isNull60) {
> /* 560 */       values1[13] = null;
> /* 561 */     } else {
> /* 562 */       values1[13] = value60;
> /* 563 */     }
> /* 564 */     boolean isNull62 = MapObjects_loopIsNull1;
> /* 565 */     final java.lang.String value62 = isNull62 ? null : (java.lang.String) MapObjects_loopValue0.proto();
> /* 566 */     isNull62 = value62 == null;
> /* 567 */     boolean isNull61 = isNull62;
> /* 568 */     final UTF8String value61 = isNull61 ? null : org.apache.spark.unsafe.types.UTF8String.fromString(value62);
> /* 569 */     isNull61 = value61 == null;
> /* 570 */     if (isNull61) {
> /* 571 */       values1[14] = null;
> /* 572 */     } else {
> /* 573 */       values1[14] = value61;
> /* 574 */     }
> /* 575 */   }
> /* 576 */
> /* 577 */
> /* 578 */   private void apply_10(InternalRow i) {
> /* 579 */
> /* 580 */     boolean isNull87 = MapObjects_loopIsNull1;
> /* 581 */     final scala.Option value87 = isNull87 ? null : (scala.Option) MapObjects_loopValue0.orig_ip_bytes();
> /* 582 */     isNull87 = value87 == null;
> /* 583 */
> /* 584 */     final boolean isNull86 = isNull87 || value87.isEmpty();
> /* 585 */     long value86 = isNull86 ?
> /* 586 */     -1L : (Long) value87.get();
> /* 587 */     if (isNull86) {
> /* 588 */       values1[25] = null;
> /* 589 */     } else {
> /* 590 */       values1[25] = value86;
> /* 591 */     }
> /* 592 */     boolean isNull89 = MapObjects_loopIsNull1;
> /* 593 */     final scala.Option value89 = isNull89 ? null : (scala.Option) MapObjects_loopValue0.resp_pkts();
> /* 594 */     isNull89 = value89 == null;
> /* 595 */
> /* 596 */     final boolean isNull88 = isNull89 || value89.isEmpty();
> /* 597 */     long value88 = isNull88 ?
> /* 598 */     -1L : (Long) value89.get();
> /* 599 */     if (isNull88) {
> /* 600 */       values1[26] = null;
> /* 601 */     } else {
> /* 602 */       values1[26] = value88;
> /* 603 */     }
> /* 604 */     boolean isNull91 = MapObjects_loopIsNull1;
> /* 605 */     final scala.Option value91 = isNull91 ? null : (scala.Option) MapObjects_loopValue0.resp_ip_bytes();
> /* 606 */     isNull91 = value91 == null;
> /* 607 */
> /* 608 */     final boolean isNull90 = isNull91 || value91.isEmpty();
> /* 609 */     long value90 = isNull90 ?
> /* 610 */     -1L : (Long) value91.get();
> /* 611 */     if (isNull90) {
> /* 612 */       values1[27] = null;
> /* 613 */     } else {
> /* 614 */       values1[27] = value90;
> /* 615 */     }
> /* 616 */   }
> /* 617 */
> /* 618 */
> /* 619 */   public SpecificMutableProjection(Object[] references) {
> /* 620 */     this.references = references;
> /* 621 */     mutableRow = new org.apache.spark.sql.catalyst.expressions.GenericMutableRow(2);
> /* 622 */     this.values = null;
> /* 623 */     this.errMsg = (java.lang.String) references[3];
> /* 624 */     this.errMsg1 = (java.lang.String) references[4];
> /* 625 */
> /* 626 */
> /* 627 */     this.errMsg2 = (java.lang.String) references[5];
> /* 628 */     this.values1 = null;
> /* 629 */
> /* 630 */
> /* 631 */     this.isNull_0 = true;
> /* 632 */     this.value_0 = false;
> /* 633 */     this.isNull_1 = true;
> /* 634 */     this.value_1 = null;
> /* 635 */   }
> /* 636 */
> /* 637 */   public org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection
target(MutableRow row) {
> /* 638 */     mutableRow = row;
> /* 639 */     return this;
> /* 640 */   }
> /* 641 */
> /* 642 */   /* Provide immutable access to the last projected row. */
> /* 643 */   public InternalRow currentValue() {
> /* 644 */     return (InternalRow) mutableRow;
> /* 645 */   }
> /* 646 */
> /* 647 */   public java.lang.Object apply(java.lang.Object _i) {
> /* 648 */     InternalRow i = (InternalRow) _i;
> /* 649 */
> /* 650 */
> /* 651 */
> /* 652 */     Object obj = ((Expression) references[0]).eval(null);
> /* 653 */     scala.Tuple2 value1 = (scala.Tuple2) obj;
> /* 654 */
> /* 655 */     boolean isNull2 = false;
> /* 656 */     final boolean value2 = isNull2 ? false : (Boolean) value1._1();
> /* 657 */     this.isNull_0 = isNull2;
> /* 658 */     this.value_0 = value2;
> /* 659 */
> /* 660 */
> /* 661 */     Object obj1 = ((Expression) references[1]).eval(null);
> /* 662 */     scala.Tuple2 value4 = (scala.Tuple2) obj1;
> /* 663 */
> /* 664 */     boolean isNull8 = false;
> /* 665 */     final io.mistnet.analytics.scan.SrcDstGrouped value8 = isNull8 ? null :
(io.mistnet.analytics.scan.SrcDstGrouped) value4._2();
> /* 666 */     isNull8 = value8 == null;
> /* 667 */     boolean isNull6 = false;
> /* 668 */     boolean value6 = true;
> /* 669 */
> /* 670 */     if (!false && isNull8) {
> /* 671 */     } else {
> /* 672 */
> /* 673 */       Object obj2 = ((Expression) references[2]).eval(null);
> /* 674 */       scala.None$ value10 = (scala.None$) obj2;
> /* 675 */
> /* 676 */       boolean isNull11 = false;
> /* 677 */       final io.mistnet.analytics.scan.SrcDstGrouped value11 = isNull11 ? null
: (io.mistnet.analytics.scan.SrcDstGrouped) value4._2();
> /* 678 */       isNull11 = value11 == null;
> /* 679 */       boolean isNull9 = false || isNull11;
> /* 680 */       final boolean value9 = isNull9 ? false : value10.equals(value11);
> /* 681 */       if (!isNull9 && value9) {
> /* 682 */       } else if (!false && !isNull9) {
> /* 683 */         value6 = false;
> /* 684 */       } else {
> /* 685 */         isNull6 = true;
> /* 686 */       }
> /* 687 */     }
> /* 688 */     boolean isNull5 = false;
> /* 689 */     InternalRow value5 = null;
> /* 690 */     if (!isNull6 && value6) {
> /* 691 */
> /* 692 */       final InternalRow value12 = null;
> /* 693 */       isNull5 = true;
> /* 694 */       value5 = value12;
> /* 695 */     } else {
> /* 696 */
> /* 697 */       boolean isNull13 = false;
> /* 698 */       this.values = new Object[3];apply1_0(i);
> /* 699 */       apply1_1(i);
> /* 700 */       final InternalRow value13 = new org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
> /* 701 */       this.values = null;
> /* 702 */       isNull5 = isNull13;
> /* 703 */       value5 = value13;
> /* 704 */     }
> /* 705 */     this.isNull_1 = isNull5;
> /* 706 */     this.value_1 = value5;
> /* 707 */
> /* 708 */     // copy all the results into MutableRow
> /* 709 */
> /* 710 */     if (!this.isNull_0) {
> /* 711 */       mutableRow.setBoolean(0, this.value_0);
> /* 712 */     } else {
> /* 713 */       mutableRow.setNullAt(0);
> /* 714 */     }
> /* 715 */
> /* 716 */     if (!this.isNull_1) {
> /* 717 */       mutableRow.update(1, this.value_1);
> /* 718 */     } else {
> /* 719 */       mutableRow.setNullAt(1);
> /* 720 */     }
> /* 721 */
> /* 722 */     return mutableRow;
> /* 723 */   }
> /* 724 */ }
> 	at org.spark_project.guava.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:306)
> 	at org.spark_project.guava.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:293)
> 	at org.spark_project.guava.util.concurrent.AbstractFuture.get(AbstractFuture.java:116)
> 	at org.spark_project.guava.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:135)
> 	at org.spark_project.guava.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2410)
> 	at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2380)
> 	at org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
> 	at org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
> 	at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000)
> 	at org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
> 	at org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
> 	at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:841)
> 	at org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection$.create(GenerateMutableProjection.scala:140)
> 	at org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection$.generate(GenerateMutableProjection.scala:44)
> 	at org.apache.spark.sql.execution.SparkPlan.newMutableProjection(SparkPlan.scala:369)
> 	at org.apache.spark.sql.execution.aggregate.SortAggregateExec$$anonfun$doExecute$1$$anonfun$3$$anonfun$4.apply(SortAggregateExec.scala:93)
> 	at org.apache.spark.sql.execution.aggregate.SortAggregateExec$$anonfun$doExecute$1$$anonfun$3$$anonfun$4.apply(SortAggregateExec.scala:92)
> 	at org.apache.spark.sql.execution.aggregate.AggregationIterator.<init>(AggregationIterator.scala:143)
> 	at org.apache.spark.sql.execution.aggregate.SortBasedAggregationIterator.<init>(SortBasedAggregationIterator.scala:39)
> 	at org.apache.spark.sql.execution.aggregate.SortAggregateExec$$anonfun$doExecute$1$$anonfun$3.apply(SortAggregateExec.scala:84)
> 	at org.apache.spark.sql.execution.aggregate.SortAggregateExec$$anonfun$doExecute$1$$anonfun$3.apply(SortAggregateExec.scala:75)
> 	at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$24.apply(RDD.scala:803)
> 	at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$24.apply(RDD.scala:803)
> 	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
> 	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:319)
> 	at org.apache.spark.rdd.RDD.iterator(RDD.scala:283)
> 	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
> 	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:319)
> 	at org.apache.spark.rdd.RDD.iterator(RDD.scala:283)
> 	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:79)
> 	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:47)
> 	at org.apache.spark.scheduler.Task.run(Task.scala:86)
> 	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> {noformat}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org


Mime
View raw message