parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ziva...@apache.org
Subject [parquet-mr] branch master updated: PARQUET-1556 Use try-with-resource in Apache Avro tests (#639)
Date Thu, 30 May 2019 12:12:04 GMT
This is an automated email from the ASF dual-hosted git repository.

zivanfi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git


The following commit(s) were added to refs/heads/master by this push:
     new 54777b4  PARQUET-1556 Use try-with-resource in Apache Avro tests (#639)
54777b4 is described below

commit 54777b46da768df3ff88548c6b6b70bd52fba0dd
Author: Fokko Driesprong <fokko@apache.org>
AuthorDate: Thu May 30 14:11:58 2019 +0200

    PARQUET-1556 Use try-with-resource in Apache Avro tests (#639)
---
 .../java/org/apache/parquet/avro/AvroTestUtil.java |  26 ++---
 .../parquet/avro/TestArrayCompatibility.java       |   9 +-
 .../org/apache/parquet/avro/TestReadWrite.java     |  99 ++++++++--------
 .../parquet/avro/TestReadWriteOldListBehavior.java | 128 ++++++++++-----------
 4 files changed, 131 insertions(+), 131 deletions(-)

diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java b/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java
index 39c6d2a..801f2bd 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/AvroTestUtil.java
@@ -85,19 +85,16 @@ public class AvroTestUtil {
     Configuration conf = new Configuration(false);
     AvroReadSupport.setRequestedProjection(conf, schema);
     AvroReadSupport.setAvroReadSchema(conf, schema);
-    ParquetReader<D> fileReader = AvroParquetReader
-        .<D>builder(new Path(file.toString()))
-        .withDataModel(model) // reflect disables compatibility
-        .withConf(conf)
-        .build();
 
-    try {
+    try (ParquetReader<D> fileReader = AvroParquetReader
+      .<D>builder(new Path(file.toString()))
+      .withDataModel(model) // reflect disables compatibility
+      .withConf(conf)
+      .build()) {
       D datum;
       while ((datum = fileReader.read()) != null) {
         data.add(datum);
       }
-    } finally {
-      fileReader.close();
     }
 
     return data;
@@ -107,18 +104,15 @@ public class AvroTestUtil {
   public static <D> File write(TemporaryFolder temp, GenericData model, Schema schema,
D... data) throws IOException {
     File file = temp.newFile();
     Assert.assertTrue(file.delete());
-    ParquetWriter<D> writer = AvroParquetWriter
-        .<D>builder(new Path(file.toString()))
-        .withDataModel(model)
-        .withSchema(schema)
-        .build();
 
-    try {
+    try (ParquetWriter<D> writer = AvroParquetWriter
+      .<D>builder(new Path(file.toString()))
+      .withDataModel(model)
+      .withSchema(schema)
+      .build()) {
       for (D datum : data) {
         writer.write(datum);
       }
-    } finally {
-      writer.close();
     }
 
     return file;
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java
b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java
index aa577ab..1f47ba7 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java
@@ -1114,12 +1114,13 @@ public class TestArrayCompatibility extends DirectWriterTest {
                 instance(structWithElementField, "element", 34.0F))));
 
     // check the schema
-    ParquetFileReader reader = ParquetFileReader
-        .open(new Configuration(), test);
-    MessageType fileSchema = reader.getFileMetaData().getSchema();
-    Assert.assertEquals("Converted schema should assume 2-layer structure",
+    final MessageType fileSchema;
+    try (ParquetFileReader reader = ParquetFileReader.open(new Configuration(), test)) {
+      fileSchema = reader.getFileMetaData().getSchema();
+      Assert.assertEquals("Converted schema should assume 2-layer structure",
         oldSchema,
         new AvroSchemaConverter(OLD_BEHAVIOR_CONF).convert(fileSchema));
+    }
 
     // both should default to the 2-layer structure
     assertReaderContains(oldBehaviorReader(test), oldSchema, oldRecord);
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWrite.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWrite.java
index 60ff269..1e726c3 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWrite.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWrite.java
@@ -89,26 +89,27 @@ public class TestReadWrite {
     Schema schema = new Schema.Parser().parse(
         Resources.getResource("array.avsc").openStream());
 
+    // Write a record with an empty array.
+    List<Integer> emptyArray = new ArrayList<>();
+
     Path file = new Path(createTempFile().getPath());
 
-    ParquetWriter<GenericRecord> writer = AvroParquetWriter
+    try(ParquetWriter<GenericRecord> writer = AvroParquetWriter
         .<GenericRecord>builder(file)
         .withSchema(schema)
         .withConf(testConf)
-        .build();
-
-    // Write a record with an empty array.
-    List<Integer> emptyArray = new ArrayList<Integer>();
-    GenericData.Record record = new GenericRecordBuilder(schema)
+        .build()) {
+      GenericData.Record record = new GenericRecordBuilder(schema)
         .set("myarray", emptyArray).build();
-    writer.write(record);
-    writer.close();
+      writer.write(record);
+    }
 
-    AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf,
file);
-    GenericRecord nextRecord = reader.read();
+    try (AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf,
file)) {
+      GenericRecord nextRecord = reader.read();
 
-    assertNotNull(nextRecord);
-    assertEquals(emptyArray, nextRecord.get("myarray"));
+      assertNotNull(nextRecord);
+      assertEquals(emptyArray, nextRecord.get("myarray"));
+    }
   }
 
   @Test
@@ -131,11 +132,12 @@ public class TestReadWrite {
       writer.write(record);
     }
 
-    AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf,
file);
-    GenericRecord nextRecord = reader.read();
+    try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf,
file)) {
+      GenericRecord nextRecord = reader.read();
 
-    assertNotNull(nextRecord);
-    assertEquals(emptyMap, nextRecord.get("mymap"));
+      assertNotNull(nextRecord);
+      assertEquals(emptyMap, nextRecord.get("mymap"));
+    }
   }
 
   @Test
@@ -162,11 +164,12 @@ public class TestReadWrite {
       writer.write(record);
     }
 
-    AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf,
file);
-    GenericRecord nextRecord = reader.read();
+    try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf,
file)) {
+      GenericRecord nextRecord = reader.read();
 
-    assertNotNull(nextRecord);
-    assertEquals(map, nextRecord.get("mymap"));
+      assertNotNull(nextRecord);
+      assertEquals(map, nextRecord.get("mymap"));
+    }
   }
 
   @Test(expected=RuntimeException.class)
@@ -215,11 +218,12 @@ public class TestReadWrite {
       writer.write(record);
     }
 
-    AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf,
file);
-    GenericRecord nextRecord = reader.read();
+    try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf,
file)) {
+      GenericRecord nextRecord = reader.read();
 
-    assertNotNull(nextRecord);
-    assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
+      assertNotNull(nextRecord);
+      assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
+    }
   }
 
   @Rule
@@ -606,25 +610,25 @@ public class TestReadWrite {
     GenericFixed genericFixed = new GenericData.Fixed(
         Schema.createFixed("fixed", null, null, 1), new byte[] { (byte) 65 });
 
-    AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf,
file);
-    GenericRecord nextRecord = reader.read();
-    assertNotNull(nextRecord);
-    assertEquals(true, nextRecord.get("myboolean"));
-    assertEquals(1, nextRecord.get("myint"));
-    assertEquals(2L, nextRecord.get("mylong"));
-    assertEquals(3.1f, nextRecord.get("myfloat"));
-    assertEquals(4.1, nextRecord.get("mydouble"));
-    assertEquals(ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8)), nextRecord.get("mybytes"));
-    assertEquals(str("hello"), nextRecord.get("mystring"));
-    assertEquals(str("a"), nextRecord.get("myenum")); // enum symbols are unknown
-    assertEquals(nestedRecord, nextRecord.get("mynestedrecord"));
-    assertEquals(integerArray, nextRecord.get("myarray"));
-    assertEquals(integerArray, nextRecord.get("myoptionalarray"));
-    assertEquals(ingeterArrayWithNulls, nextRecord.get("myarrayofoptional"));
-    assertEquals(genericRecordArray, nextRecord.get("myrecordarray"));
-    assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
-    assertEquals(genericFixed, nextRecord.get("myfixed"));
-
+    try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf,
file)) {
+      GenericRecord nextRecord = reader.read();
+      assertNotNull(nextRecord);
+      assertEquals(true, nextRecord.get("myboolean"));
+      assertEquals(1, nextRecord.get("myint"));
+      assertEquals(2L, nextRecord.get("mylong"));
+      assertEquals(3.1f, nextRecord.get("myfloat"));
+      assertEquals(4.1, nextRecord.get("mydouble"));
+      assertEquals(ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8)), nextRecord.get("mybytes"));
+      assertEquals(str("hello"), nextRecord.get("mystring"));
+      assertEquals(str("a"), nextRecord.get("myenum")); // enum symbols are unknown
+      assertEquals(nestedRecord, nextRecord.get("mynestedrecord"));
+      assertEquals(integerArray, nextRecord.get("myarray"));
+      assertEquals(integerArray, nextRecord.get("myoptionalarray"));
+      assertEquals(ingeterArrayWithNulls, nextRecord.get("myarrayofoptional"));
+      assertEquals(genericRecordArray, nextRecord.get("myrecordarray"));
+      assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
+      assertEquals(genericFixed, nextRecord.get("myfixed"));
+    }
   }
 
   @Test
@@ -648,11 +652,12 @@ public class TestReadWrite {
       parquetWriter.write(record);
     }
 
-    AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf,
file);
-    GenericRecord nextRecord = reader.read();
+    try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf,
file)) {
+      GenericRecord nextRecord = reader.read();
 
-    assertNotNull(nextRecord);
-    assertEquals(str("theValue"), nextRecord.get("value"));
+      assertNotNull(nextRecord);
+      assertEquals(str("theValue"), nextRecord.get("value"));
+    }
   }
 
   @Test
diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWriteOldListBehavior.java
b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWriteOldListBehavior.java
index 4542ef2..f12417c 100644
--- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWriteOldListBehavior.java
+++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestReadWriteOldListBehavior.java
@@ -94,11 +94,12 @@ public class TestReadWriteOldListBehavior {
       writer.write(record);
     }
 
-    AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf,
file);
-    GenericRecord nextRecord = reader.read();
+    try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf,
file)) {
+      GenericRecord nextRecord = reader.read();
 
-    assertNotNull(nextRecord);
-    assertEquals(emptyArray, nextRecord.get("myarray"));
+      assertNotNull(nextRecord);
+      assertEquals(emptyArray, nextRecord.get("myarray"));
+    }
   }
 
   @Test
@@ -119,11 +120,12 @@ public class TestReadWriteOldListBehavior {
       writer.write(record);
     }
 
-    AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf,
file);
-    GenericRecord nextRecord = reader.read();
+    try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf,
file)) {
+      GenericRecord nextRecord = reader.read();
 
-    assertNotNull(nextRecord);
-    assertEquals(emptyMap, nextRecord.get("mymap"));
+      assertNotNull(nextRecord);
+      assertEquals(emptyMap, nextRecord.get("mymap"));
+    }
   }
 
   @Test
@@ -148,11 +150,12 @@ public class TestReadWriteOldListBehavior {
       writer.write(record);
     }
 
-    AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf,
file);
-    GenericRecord nextRecord = reader.read();
+    try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf,
file)) {
+      GenericRecord nextRecord = reader.read();
 
-    assertNotNull(nextRecord);
-    assertEquals(map, nextRecord.get("mymap"));
+      assertNotNull(nextRecord);
+      assertEquals(map, nextRecord.get("mymap"));
+    }
   }
 
   @Test(expected=RuntimeException.class)
@@ -197,11 +200,12 @@ public class TestReadWriteOldListBehavior {
       writer.write(record);
     }
 
-    AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf,
file);
-    GenericRecord nextRecord = reader.read();
+    try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf,
file)) {
+      GenericRecord nextRecord = reader.read();
 
-    assertNotNull(nextRecord);
-    assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
+      assertNotNull(nextRecord);
+      assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
+    }
   }
 
   @Test
@@ -252,30 +256,31 @@ public class TestReadWriteOldListBehavior {
       writer.write(record);
     }
 
-    AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf,
file);
-    GenericRecord nextRecord = reader.read();
+    try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf,
file)) {
+      GenericRecord nextRecord = reader.read();
 
-    Object expectedEnumSymbol = compat ? "a" :
+      Object expectedEnumSymbol = compat ? "a" :
         new GenericData.EnumSymbol(schema.getField("myenum").schema(), "a");
 
-    assertNotNull(nextRecord);
-    assertEquals(null, nextRecord.get("mynull"));
-    assertEquals(true, nextRecord.get("myboolean"));
-    assertEquals(1, nextRecord.get("myint"));
-    assertEquals(2L, nextRecord.get("mylong"));
-    assertEquals(3.1f, nextRecord.get("myfloat"));
-    assertEquals(4.1, nextRecord.get("mydouble"));
-    assertEquals(ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8)), nextRecord.get("mybytes"));
-    assertEquals(str("hello"), nextRecord.get("mystring"));
-    assertEquals(expectedEnumSymbol, nextRecord.get("myenum"));
-    assertEquals(nestedRecord, nextRecord.get("mynestedrecord"));
-    assertEquals(integerArray, nextRecord.get("myarray"));
-    assertEquals(emptyArray, nextRecord.get("myemptyarray"));
-    assertEquals(integerArray, nextRecord.get("myoptionalarray"));
-    assertEquals(integerArray, nextRecord.get("myarrayofoptional"));
-    assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
-    assertEquals(emptyMap, nextRecord.get("myemptymap"));
-    assertEquals(genericFixed, nextRecord.get("myfixed"));
+      assertNotNull(nextRecord);
+      assertEquals(null, nextRecord.get("mynull"));
+      assertEquals(true, nextRecord.get("myboolean"));
+      assertEquals(1, nextRecord.get("myint"));
+      assertEquals(2L, nextRecord.get("mylong"));
+      assertEquals(3.1f, nextRecord.get("myfloat"));
+      assertEquals(4.1, nextRecord.get("mydouble"));
+      assertEquals(ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8)), nextRecord.get("mybytes"));
+      assertEquals(str("hello"), nextRecord.get("mystring"));
+      assertEquals(expectedEnumSymbol, nextRecord.get("myenum"));
+      assertEquals(nestedRecord, nextRecord.get("mynestedrecord"));
+      assertEquals(integerArray, nextRecord.get("myarray"));
+      assertEquals(emptyArray, nextRecord.get("myemptyarray"));
+      assertEquals(integerArray, nextRecord.get("myoptionalarray"));
+      assertEquals(integerArray, nextRecord.get("myarrayofoptional"));
+      assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
+      assertEquals(emptyMap, nextRecord.get("myemptymap"));
+      assertEquals(genericFixed, nextRecord.get("myfixed"));
+    }
   }
 
   @Test
@@ -305,7 +310,7 @@ public class TestReadWriteOldListBehavior {
     Schema arrayOfOptionalIntegers = Schema.createArray(
         optional(Schema.create(Schema.Type.INT)));
     GenericData.Array<Integer> genericIntegerArrayWithNulls =
-        new GenericData.Array<Integer>(
+        new GenericData.Array<>(
             arrayOfOptionalIntegers,
             Arrays.asList(1, null, 2, null, 3));
 
@@ -329,17 +334,12 @@ public class TestReadWriteOldListBehavior {
         .set("myfixed", genericFixed)
         .build();
 
-    final AvroParquetWriter<GenericRecord> writer =
-        new AvroParquetWriter<GenericRecord>(file, schema);
-
-    try {
+    try (AvroParquetWriter<GenericRecord> writer = new AvroParquetWriter<>(file,
schema)) {
       writer.write(record);
       fail("Should not succeed writing an array with null values");
     } catch (Exception e) {
       Assert.assertTrue("Error message should provide context and help",
-          e.getMessage().contains("parquet.avro.write-old-list-structure"));
-    } finally {
-      writer.close();
+        e.getMessage().contains("parquet.avro.write-old-list-structure"));
     }
   }
 
@@ -554,25 +554,25 @@ public class TestReadWriteOldListBehavior {
                 elementBuilder.set("element", 3).build()
             ));
 
-    AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf,
file);
-    GenericRecord nextRecord = reader.read();
-    assertNotNull(nextRecord);
-    assertEquals(true, nextRecord.get("myboolean"));
-    assertEquals(1, nextRecord.get("myint"));
-    assertEquals(2L, nextRecord.get("mylong"));
-    assertEquals(3.1f, nextRecord.get("myfloat"));
-    assertEquals(4.1, nextRecord.get("mydouble"));
-    assertEquals(ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8)), nextRecord.get("mybytes"));
-    assertEquals(str("hello"), nextRecord.get("mystring"));
-    assertEquals(str("a"), nextRecord.get("myenum"));
-    assertEquals(nestedRecord, nextRecord.get("mynestedrecord"));
-    assertEquals(integerArray, nextRecord.get("myarray"));
-    assertEquals(integerArray, nextRecord.get("myoptionalarray"));
-    assertEquals(genericRecordArrayWithNullIntegers, nextRecord.get("myarrayofoptional"));
-    assertEquals(genericRecordArray, nextRecord.get("myrecordarray"));
-    assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
-    assertEquals(genericFixed, nextRecord.get("myfixed"));
-
+    try(AvroParquetReader<GenericRecord> reader = new AvroParquetReader<>(testConf,
file)) {
+      GenericRecord nextRecord = reader.read();
+      assertNotNull(nextRecord);
+      assertEquals(true, nextRecord.get("myboolean"));
+      assertEquals(1, nextRecord.get("myint"));
+      assertEquals(2L, nextRecord.get("mylong"));
+      assertEquals(3.1f, nextRecord.get("myfloat"));
+      assertEquals(4.1, nextRecord.get("mydouble"));
+      assertEquals(ByteBuffer.wrap("hello".getBytes(StandardCharsets.UTF_8)), nextRecord.get("mybytes"));
+      assertEquals(str("hello"), nextRecord.get("mystring"));
+      assertEquals(str("a"), nextRecord.get("myenum"));
+      assertEquals(nestedRecord, nextRecord.get("mynestedrecord"));
+      assertEquals(integerArray, nextRecord.get("myarray"));
+      assertEquals(integerArray, nextRecord.get("myoptionalarray"));
+      assertEquals(genericRecordArrayWithNullIntegers, nextRecord.get("myarrayofoptional"));
+      assertEquals(genericRecordArray, nextRecord.get("myrecordarray"));
+      assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
+      assertEquals(genericFixed, nextRecord.get("myfixed"));
+    }
   }
 
   /**


Mime
View raw message