avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marti...@apache.org
Subject svn commit: r1722355 - in /avro/branches/branch-1.8: CHANGES.txt lang/c/src/datafile.c lang/c/tests/CMakeLists.txt lang/c/tests/test_avro_1572.c
Date Wed, 30 Dec 2015 15:32:02 GMT
Author: martinkl
Date: Wed Dec 30 15:32:01 2015
New Revision: 1722355

URL: http://svn.apache.org/viewvc?rev=1722355&view=rev
Log:
AVRO-1572: C: Fix EOF handling on data files that are multiples of 4096 bytes. (Ben Walsh
via martinkl)

Added:
    avro/branches/branch-1.8/lang/c/tests/test_avro_1572.c
Modified:
    avro/branches/branch-1.8/CHANGES.txt
    avro/branches/branch-1.8/lang/c/src/datafile.c
    avro/branches/branch-1.8/lang/c/tests/CMakeLists.txt

Modified: avro/branches/branch-1.8/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.8/CHANGES.txt?rev=1722355&r1=1722354&r2=1722355&view=diff
==============================================================================
--- avro/branches/branch-1.8/CHANGES.txt (original)
+++ avro/branches/branch-1.8/CHANGES.txt Wed Dec 30 15:32:01 2015
@@ -227,6 +227,9 @@ Avro 1.8.0 (15 December 2015)
 
     AVRO-1728: Java: Add LICENSE and NOTICE files to jars. (blue)
 
+    AVRO-1572: C: Fix EOF handling on data files that are multiples of
+    4096 bytes. (Ben Walsh via martinkl)
+
 Avro 1.7.7 (23 July 2014)
 
   NEW FEATURES

Modified: avro/branches/branch-1.8/lang/c/src/datafile.c
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.8/lang/c/src/datafile.c?rev=1722355&r1=1722354&r2=1722355&view=diff
==============================================================================
--- avro/branches/branch-1.8/lang/c/src/datafile.c (original)
+++ avro/branches/branch-1.8/lang/c/src/datafile.c Wed Dec 30 15:32:01 2015
@@ -439,7 +439,15 @@ static int file_read_block_count(avro_fi
 	int rval;
 	int64_t len;
 	const avro_encoding_t *enc = &avro_binary_encoding;
-	check_prefix(rval, enc->read_long(r->reader, &r->blocks_total),
+
+	/* For a correctly formatted file, EOF will occur here */
+	rval = enc->read_long(r->reader, &r->blocks_total);
+
+	if (rval == EILSEQ && avro_reader_is_eof(r->reader)) {
+		return EOF;
+	}
+
+	check_prefix(rval, rval,
 		     "Cannot read file block count: ");
 	check_prefix(rval, enc->read_long(r->reader, &len),
 		     "Cannot read file block size: ");
@@ -710,6 +718,7 @@ avro_file_reader_read_value(avro_file_re
 	check_param(EINVAL, value, "value");
 
 	if (r->blocks_read == r->blocks_total) {
+		/* reads sync bytes and buffers further bytes */
 		check(rval, avro_read(r->reader, sync, sizeof(sync)));
 		if (memcmp(r->sync, sync, sizeof(r->sync)) != 0) {
 			/* wrong sync bytes */
@@ -717,10 +726,6 @@ avro_file_reader_read_value(avro_file_re
 			return EILSEQ;
 		}
 
-		/* Did we just hit the end of the file? */
-		if (avro_reader_is_eof(r->reader))
-			return EOF;
-
 		check(rval, file_read_block_count(r));
 	}
 

Modified: avro/branches/branch-1.8/lang/c/tests/CMakeLists.txt
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.8/lang/c/tests/CMakeLists.txt?rev=1722355&r1=1722354&r2=1722355&view=diff
==============================================================================
--- avro/branches/branch-1.8/lang/c/tests/CMakeLists.txt (original)
+++ avro/branches/branch-1.8/lang/c/tests/CMakeLists.txt Wed Dec 30 15:32:01 2015
@@ -58,6 +58,7 @@ add_avro_test(test_avro_1237)
 add_avro_test(test_avro_1238)
 add_avro_test(test_avro_1279)
 add_avro_test(test_avro_1405)
+add_avro_test(test_avro_1572)
 add_avro_test(test_avro_data)
 add_avro_test(test_refcount)
 add_avro_test(test_cpp test_cpp.cpp)

Added: avro/branches/branch-1.8/lang/c/tests/test_avro_1572.c
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.8/lang/c/tests/test_avro_1572.c?rev=1722355&view=auto
==============================================================================
--- avro/branches/branch-1.8/lang/c/tests/test_avro_1572.c (added)
+++ avro/branches/branch-1.8/lang/c/tests/test_avro_1572.c Wed Dec 30 15:32:01 2015
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied.  See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+#include <stdio.h>
+#include <sys/stat.h>
+#include "avro.h"
+
+#define BUFFER_LEN 4096
+
+#define HEADER_LEN 116
+#define SYNC_LEN 16
+#define BLOCKINFO_LEN 4
+#define RECORD_LEN 1
+
+static const int NUM_RECORDS1 = (BUFFER_LEN - HEADER_LEN - 2 * SYNC_LEN - BLOCKINFO_LEN)
/ RECORD_LEN;
+static const int NUM_RECORDS2 = (BUFFER_LEN - SYNC_LEN - BLOCKINFO_LEN) / RECORD_LEN;
+
+static const char  PERSON_SCHEMA[] =
+	"{"
+	"    \"type\":\"record\","
+	"    \"name\":\"Person\","
+	"    \"fields\": ["
+	"        {\"name\": \"ab\", \"type\": \"int\"}"
+	"    ]"
+	"}";
+
+static const char *filename = "avro_file.dat";
+
+static int read_data() {
+	int rval;
+	int records_read = 0;
+
+	avro_file_reader_t reader;
+	avro_value_iface_t *iface;
+	avro_value_t value;
+
+	fprintf(stderr, "\nReading...\n");
+
+	avro_file_reader(filename, &reader);
+	avro_schema_t schema = avro_file_reader_get_writer_schema(reader);
+
+	iface = avro_generic_class_from_schema(schema);
+	avro_generic_value_new(iface, &value);
+
+	while ((rval = avro_file_reader_read_value(reader, &value)) == 0) {
+		records_read++;
+		avro_value_reset(&value);
+	}
+
+	avro_value_decref(&value);
+	avro_value_iface_decref(iface);
+	avro_schema_decref(schema);
+	avro_file_reader_close(reader);
+
+	fprintf(stderr, "wanted %d records, read %d records.\n", NUM_RECORDS1 + NUM_RECORDS2, records_read);
+
+	if (rval != EOF || records_read != (NUM_RECORDS1 + NUM_RECORDS2)) {
+		fprintf(stderr, "Error: %s\n", avro_strerror());
+		return EXIT_FAILURE;
+	}
+
+	return EXIT_SUCCESS;
+}
+
+static off_t fsize(const char *filename) {
+	struct stat st;
+
+	if (stat(filename, &st) == 0) {
+		return st.st_size;
+	}
+
+	return -1;
+}
+
+static int write_data() {
+	int  i;
+	avro_schema_t schema;
+	avro_schema_error_t error;
+	avro_file_writer_t writer;
+	avro_value_iface_t *iface;
+	avro_value_t value;
+	avro_value_t field;
+
+	fprintf(stderr, "\nWriting...\n");
+
+	if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) {
+		fprintf(stderr, "Unable to parse schema\n");
+		return EXIT_FAILURE;
+	}
+
+	iface = avro_generic_class_from_schema(schema);
+	avro_generic_value_new(iface, &value);
+
+	if (avro_file_writer_create(filename, schema, &writer)) {
+		fprintf(stderr, "There was an error creating file: %s\n", avro_strerror());
+		return EXIT_FAILURE;
+	}
+
+	avro_value_get_by_name(&value, "ab", &field, NULL);
+	avro_value_set_int(&field, 1);
+
+	fprintf(stderr, "NUM_RECORDS1 = %d NUM_RECORDS2 = %d\n", NUM_RECORDS1, NUM_RECORDS2);
+
+	for (i = 0; i < NUM_RECORDS1; i++) {
+		avro_file_writer_append_value(writer, &value);
+	}
+
+	avro_file_writer_close(writer);
+
+	/* Make sure the sync ends at a BUFFER_LEN boundary */
+	if (fsize(filename) != BUFFER_LEN) {
+		fprintf(stderr, "internal error\n");
+		return EXIT_FAILURE;
+	}
+
+	avro_file_writer_open(filename, &writer);
+
+	for (i = 0; i < NUM_RECORDS2; i++) {
+		avro_file_writer_append_value(writer, &value);
+	}
+
+	avro_file_writer_close(writer);
+
+	/* Make sure the whole file ends at a BUFFER_LEN boundary. */
+	if (fsize(filename) != 2 * BUFFER_LEN) {
+	      fprintf(stderr, "internal error\n");
+	      return EXIT_FAILURE;
+	}
+
+	avro_value_iface_decref(iface);
+	avro_value_decref(&value);
+	avro_schema_decref(schema);
+
+	return EXIT_SUCCESS;
+}
+
+
+int main()
+{
+	int read_data_result;
+
+	if (write_data()) {
+		return EXIT_FAILURE;
+	}
+
+	read_data_result = read_data();
+	remove(filename);
+
+	return read_data_result;
+}



Mime
View raw message