spark-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Kazuaki Ishizaki (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (SPARK-15467) Getting stack overflow when attempting to query a wide Dataset (>200 fields)
Date Sun, 22 May 2016 10:21:12 GMT

    [ https://issues.apache.org/jira/browse/SPARK-15467?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15295499#comment-15295499
] 

Kazuaki Ishizaki commented on SPARK-15467:
------------------------------------------

IMHO, this issue is due to Janino's implementation.

First of all, [the specification of JVM|https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-4.html#jvms-4.11]
defines that the max size of an operand stack is 65535. On the other hand, in Janino's {{flowAnalysis}},
the max size of an operand stack seems to be 254 based on [this declaration|https://github.com/aunkrig/janino/blob/40d1a42562745dbb445588ff407bf1f63c26d37e/janino/src/org/codehaus/janino/CodeContext.java#L58].
A stack height is store into a {{short}} variable declared at [here|https://github.com/aunkrig/janino/blob/40d1a42562745dbb445588ff407bf1f63c26d37e/janino/src/org/codehaus/janino/CodeContext.java#L359].
It should be declared as {{int}}.

> Getting stack overflow when attempting to query a wide Dataset (>200 fields)
> ----------------------------------------------------------------------------
>
>                 Key: SPARK-15467
>                 URL: https://issues.apache.org/jira/browse/SPARK-15467
>             Project: Spark
>          Issue Type: Bug
>          Components: SQL
>    Affects Versions: 2.0.0
>            Reporter: Don Drake
>
> This can be duplicated in a spark-shell, I am running Spark 2.0.0-preview.
> {code}
> import spark.implicits._
> case class Wide(
> val f0:String = "",
> val f1:String = "",
> val f2:String = "",
> val f3:String = "",
> val f4:String = "",
> val f5:String = "",
> val f6:String = "",
> val f7:String = "",
> val f8:String = "",
> val f9:String = "",
> val f10:String = "",
> val f11:String = "",
> val f12:String = "",
> val f13:String = "",
> val f14:String = "",
> val f15:String = "",
> val f16:String = "",
> val f17:String = "",
> val f18:String = "",
> val f19:String = "",
> val f20:String = "",
> val f21:String = "",
> val f22:String = "",
> val f23:String = "",
> val f24:String = "",
> val f25:String = "",
> val f26:String = "",
> val f27:String = "",
> val f28:String = "",
> val f29:String = "",
> val f30:String = "",
> val f31:String = "",
> val f32:String = "",
> val f33:String = "",
> val f34:String = "",
> val f35:String = "",
> val f36:String = "",
> val f37:String = "",
> val f38:String = "",
> val f39:String = "",
> val f40:String = "",
> val f41:String = "",
> val f42:String = "",
> val f43:String = "",
> val f44:String = "",
> val f45:String = "",
> val f46:String = "",
> val f47:String = "",
> val f48:String = "",
> val f49:String = "",
> val f50:String = "",
> val f51:String = "",
> val f52:String = "",
> val f53:String = "",
> val f54:String = "",
> val f55:String = "",
> val f56:String = "",
> val f57:String = "",
> val f58:String = "",
> val f59:String = "",
> val f60:String = "",
> val f61:String = "",
> val f62:String = "",
> val f63:String = "",
> val f64:String = "",
> val f65:String = "",
> val f66:String = "",
> val f67:String = "",
> val f68:String = "",
> val f69:String = "",
> val f70:String = "",
> val f71:String = "",
> val f72:String = "",
> val f73:String = "",
> val f74:String = "",
> val f75:String = "",
> val f76:String = "",
> val f77:String = "",
> val f78:String = "",
> val f79:String = "",
> val f80:String = "",
> val f81:String = "",
> val f82:String = "",
> val f83:String = "",
> val f84:String = "",
> val f85:String = "",
> val f86:String = "",
> val f87:String = "",
> val f88:String = "",
> val f89:String = "",
> val f90:String = "",
> val f91:String = "",
> val f92:String = "",
> val f93:String = "",
> val f94:String = "",
> val f95:String = "",
> val f96:String = "",
> val f97:String = "",
> val f98:String = "",
> val f99:String = "",
> val f100:String = "",
> val f101:String = "",
> val f102:String = "",
> val f103:String = "",
> val f104:String = "",
> val f105:String = "",
> val f106:String = "",
> val f107:String = "",
> val f108:String = "",
> val f109:String = "",
> val f110:String = "",
> val f111:String = "",
> val f112:String = "",
> val f113:String = "",
> val f114:String = "",
> val f115:String = "",
> val f116:String = "",
> val f117:String = "",
> val f118:String = "",
> val f119:String = "",
> val f120:String = "",
> val f121:String = "",
> val f122:String = "",
> val f123:String = "",
> val f124:String = "",
> val f125:String = "",
> val f126:String = "",
> val f127:String = "",
> val f128:String = "",
> val f129:String = "",
> val f130:String = "",
> val f131:String = "",
> val f132:String = "",
> val f133:String = "",
> val f134:String = "",
> val f135:String = "",
> val f136:String = "",
> val f137:String = "",
> val f138:String = "",
> val f139:String = "",
> val f140:String = "",
> val f141:String = "",
> val f142:String = "",
> val f143:String = "",
> val f144:String = "",
> val f145:String = "",
> val f146:String = "",
> val f147:String = "",
> val f148:String = "",
> val f149:String = "",
> val f150:String = "",
> val f151:String = "",
> val f152:String = "",
> val f153:String = "",
> val f154:String = "",
> val f155:String = "",
> val f156:String = "",
> val f157:String = "",
> val f158:String = "",
> val f159:String = "",
> val f160:String = "",
> val f161:String = "",
> val f162:String = "",
> val f163:String = "",
> val f164:String = "",
> val f165:String = "",
> val f166:String = "",
> val f167:String = "",
> val f168:String = "",
> val f169:String = "",
> val f170:String = "",
> val f171:String = "",
> val f172:String = "",
> val f173:String = "",
> val f174:String = "",
> val f175:String = "",
> val f176:String = "",
> val f177:String = "",
> val f178:String = "",
> val f179:String = "",
> val f180:String = "",
> val f181:String = "",
> val f182:String = "",
> val f183:String = "",
> val f184:String = "",
> val f185:String = "",
> val f186:String = "",
> val f187:String = "",
> val f188:String = "",
> val f189:String = "",
> val f190:String = "",
> val f191:String = "",
> val f192:String = "",
> val f193:String = "",
> val f194:String = "",
> val f195:String = "",
> val f196:String = "",
> val f197:String = "",
> val f198:String = "",
> val f199:String = "",
> val f200:String = "",
> val f201:String = "",
> val f202:String = "",
> val f203:String = "",
> val f204:String = "",
> val f205:String = "",
> val f206:String = "",
> val f207:String = "",
> val f208:String = "",
> val f209:String = "",
> val f210:String = "",
> val f211:String = "",
> val f212:String = "",
> val f213:String = "",
> val f214:String = "",
> val f215:String = "",
> val f216:String = "",
> val f217:String = "",
> val f218:String = "",
> val f219:String = "",
> val f220:String = "",
> val f221:String = "",
> val f222:String = "",
> val f223:String = "",
> val f224:String = "",
> val f225:String = "",
> val f226:String = "",
> val f227:String = "",
> val f228:String = "",
> val f229:String = "",
> val f230:String = "",
> val f231:String = "",
> val f232:String = "",
> val f233:String = "",
> val f234:String = "",
> val f235:String = "",
> val f236:String = "",
> val f237:String = "",
> val f238:String = "",
> val f239:String = "",
> val f240:String = "",
> val f241:String = "",
> val f242:String = "",
> val f243:String = "",
> val f244:String = "",
> val f245:String = "",
> val f246:String = "",
> val f247:String = "",
> val f248:String = "",
> val f249:String = "",
> val f250:String = "",
> val f251:String = "",
> val f252:String = "")
> val ds = Seq(Wide(), Wide()).toDS
> ds.groupByKey(x => x.f0).mapGroups( (k, vals) => { k } ).show
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org


Mime
View raw message