spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vofque <...@git.apache.org>
Subject [GitHub] spark pull request #22708: [SPARK-21402][SQL] Fix java array of structs dese...
Date Wed, 17 Oct 2018 07:39:34 GMT
Github user vofque commented on a diff in the pull request:

    https://github.com/apache/spark/pull/22708#discussion_r225810177
  
    --- Diff: sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanWithArraySuite.java
---
    @@ -0,0 +1,222 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *    http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +package test.org.apache.spark.sql;
    +
    +import java.util.ArrayList;
    +import java.util.Arrays;
    +import java.util.Collection;
    +import java.util.Iterator;
    +import java.util.List;
    +
    +import org.junit.After;
    +import org.junit.Assert;
    +import org.junit.Before;
    +import org.junit.Test;
    +
    +import org.apache.spark.sql.Dataset;
    +import org.apache.spark.sql.Encoder;
    +import org.apache.spark.sql.Encoders;
    +import org.apache.spark.sql.test.TestSparkSession;
    +import org.apache.spark.sql.types.ArrayType;
    +import org.apache.spark.sql.types.DataType;
    +import org.apache.spark.sql.types.DataTypes;
    +import org.apache.spark.sql.types.Metadata;
    +import org.apache.spark.sql.types.StructField;
    +import org.apache.spark.sql.types.StructType;
    +
    +public class JavaBeanWithArraySuite {
    +
    +    private static final List<Record> RECORDS = new ArrayList<>();
    +
    +    static {
    +        RECORDS.add(new Record(1,
    +                Arrays.asList(new Interval(111, 211), new Interval(121, 221)),
    +                Arrays.asList(11, 21, 31, 41)
    +        ));
    +        RECORDS.add(new Record(2,
    +                Arrays.asList(new Interval(112, 212), new Interval(122, 222)),
    +                Arrays.asList(12, 22, 32, 42)
    +        ));
    +        RECORDS.add(new Record(3,
    +                Arrays.asList(new Interval(113, 213), new Interval(123, 223)),
    +                Arrays.asList(13, 23, 33, 43)
    +        ));
    +    }
    +
    +    private TestSparkSession spark;
    +
    +    @Before
    +    public void setUp() {
    +        spark = new TestSparkSession();
    +    }
    +
    +    @After
    +    public void tearDown() {
    +        spark.stop();
    +        spark = null;
    +    }
    +
    +    @Test
    +    public void testBeanWithArrayFieldsDeserialization() {
    +
    +        StructType schema = createSchema();
    +        Encoder<Record> encoder = Encoders.bean(Record.class);
    +
    +        Dataset<Record> dataset = spark
    +                .read()
    +                .format("json")
    +                .schema(schema)
    +                .load("src/test/resources/test-data/with-array-fields")
    +                .as(encoder);
    +
    +        List<Record> records = dataset.collectAsList();
    +
    +        Assert.assertTrue(Util.equals(records, RECORDS));
    +    }
    +
    +    private static StructType createSchema() {
    +        StructField[] intervalFields = {
    +                new StructField("startTime", DataTypes.LongType, true, Metadata.empty()),
    +                new StructField("endTime", DataTypes.LongType, true, Metadata.empty())
    +        };
    +        DataType intervalType = new StructType(intervalFields);
    +
    +        DataType intervalsType = new ArrayType(intervalType, true);
    +
    +        DataType valuesType = new ArrayType(DataTypes.IntegerType, true);
    +
    +        StructField[] fields = {
    +                new StructField("id", DataTypes.IntegerType, true, Metadata.empty()),
    +                new StructField("intervals", intervalsType, true, Metadata.empty()),
    +                new StructField("values", valuesType, true, Metadata.empty())
    +        };
    +        return new StructType(fields);
    +    }
    +
    +    public static class Record {
    +
    +        private int id;
    +        private List<Interval> intervals;
    +        private List<Integer> values;
    --- End diff --
    
    The intention was to test a non struct case too. But I think, it's really not critical
and we can get rid of it.


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org


Mime
View raw message