avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mas...@apache.org
Subject svn commit: r834276 - in /hadoop/avro/trunk: ./ src/c/ src/c/config/ src/test/schemata/
Date Mon, 09 Nov 2009 23:08:33 GMT
Author: massie
Date: Mon Nov  9 23:08:32 2009
New Revision: 834276

URL: http://svn.apache.org/viewvc?rev=834276&view=rev
Log:
AVRO-186. Full read-path interoperability test

Added:
    hadoop/avro/trunk/src/c/config/
    hadoop/avro/trunk/src/c/config/.gitignore
    hadoop/avro/trunk/src/c/test_avro_data.c
      - copied, changed from r833621, hadoop/avro/trunk/src/c/test_avro_primitives.c
Removed:
    hadoop/avro/trunk/src/c/test_avro_primitives.c
Modified:
    hadoop/avro/trunk/CHANGES.txt
    hadoop/avro/trunk/src/c/Makefile.am
    hadoop/avro/trunk/src/c/README
    hadoop/avro/trunk/src/c/avro_array.c
    hadoop/avro/trunk/src/c/avro_boolean.c
    hadoop/avro/trunk/src/c/avro_bytes.c
    hadoop/avro/trunk/src/c/avro_double.c
    hadoop/avro/trunk/src/c/avro_enum.c
    hadoop/avro/trunk/src/c/avro_fixed.c
    hadoop/avro/trunk/src/c/avro_float.c
    hadoop/avro/trunk/src/c/avro_int.c
    hadoop/avro/trunk/src/c/avro_io_memory.c
    hadoop/avro/trunk/src/c/avro_long.c
    hadoop/avro/trunk/src/c/avro_map.c
    hadoop/avro/trunk/src/c/avro_null.c
    hadoop/avro/trunk/src/c/avro_record.c
    hadoop/avro/trunk/src/c/avro_string.c
    hadoop/avro/trunk/src/c/avro_union.c
    hadoop/avro/trunk/src/c/avro_value.c
    hadoop/avro/trunk/src/c/configure.in
    hadoop/avro/trunk/src/c/json.c
    hadoop/avro/trunk/src/c/json.h
    hadoop/avro/trunk/src/c/test_avro_interop.c
    hadoop/avro/trunk/src/test/schemata/interop.avsc

Modified: hadoop/avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/CHANGES.txt?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/CHANGES.txt (original)
+++ hadoop/avro/trunk/CHANGES.txt Mon Nov  9 23:08:32 2009
@@ -47,6 +47,8 @@
 
     AVRO-177. Upgrade Java dependencies to recent versions. (cutting)
 
+    AVRO-186. Full read-path interoperability test (massie)
+
   OPTIMIZATIONS
 
     AVRO-172. More efficient schema processing (massie)

Modified: hadoop/avro/trunk/src/c/Makefile.am
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/Makefile.am?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/Makefile.am (original)
+++ hadoop/avro/trunk/src/c/Makefile.am Mon Nov  9 23:08:32 2009
@@ -23,7 +23,7 @@
         -export-dynamic
 
 
-check_PROGRAMS=test_json_parser test_avro_schema test_avro_primitives
+check_PROGRAMS=test_json_parser test_avro_schema test_avro_data test_avro_interop
 
 dist-hook: docs
 
@@ -35,13 +35,13 @@
 test_avro_schema=test_avro_schema.c
 test_avro_schema_LDADD=$(test_LDADD)
 
-test_avro_primitives=test_avro_primitives.c
-test_avro_primitives_LDADD=$(test_LDADD)
-# use -std=c99 to avoid warings about universal character names
-test_avro_primitives_CFLAGS=$(AM_CFLAGS) -std=c99
+test_avro_data=test_avro_data.c
+test_avro_data_LDADD=$(test_LDADD)
+# use -std=c99 to avoid warnings about universal character names
+test_avro_data_CFLAGS=$(AM_CFLAGS) -std=c99
 
-#test_avro_interop=test_avro_interop.c
-#test_avro_interop_LDADD=$(test_LDADD)
+test_avro_interop=test_avro_interop.c
+test_avro_interop_LDADD=$(test_LDADD)
 
 TESTS=$(check_PROGRAMS)
 

Modified: hadoop/avro/trunk/src/c/README
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/README?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/README (original)
+++ hadoop/avro/trunk/src/c/README Mon Nov  9 23:08:32 2009
@@ -1,20 +1,14 @@
-Avro C Bindings (libavro)
+    ___                      ______
+   /   |_   ___________     / ____/
+  / /| | | / / ___/ __ \   / /     
+ / ___ | |/ / /  / /_/ /  / /___   
+/_/  |_|___/_/   \____/   \____/   
 
-The code in this directory is the C bindings for Avro.
+======================================================
+Please see the documentation in the "docs" directory.
 
-Currently, there are only the Avro primitives with their corresponding
-unit tests.  The unit tests currently only check for proper
-encoding/decoding within the C library.  Interoperability tests are
-comming soon.
+Documentation can also be (re)generated by running...
 
-Only the memory-backed Avro handle is working right now although
-adding file and network io support is easy thank to the Apache
-Portable Runtime which the Avro C bindings depend on.
+$ make docs
 
-For API details, view the file avro.h.  The documentation is a work in
-progress.  The test cases (test_avro_*.c) also serve as great examples
-of how to use the API.
 
-libavro compile and passes all tests on Linux.  On MacOS X, 4 out of 5
-unit tests pass.  Portability is a definite design goal but currently
-the development emphasis is on Linux.

Modified: hadoop/avro/trunk/src/c/avro_array.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_array.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_array.c (original)
+++ hadoop/avro/trunk/src/c/avro_array.c Mon Nov  9 23:08:32 2009
@@ -21,13 +21,10 @@
 struct avro_array_value
 {
   apr_array_header_t *values;
-
-  struct avro_value *items;
+  const JSON_value *item_schema;
 
   apr_pool_t *pool;
 
-  int value_set;
-
   /* Need the ctx to dupe value */
   struct avro_value_ctx *ctx;
 
@@ -37,12 +34,21 @@
 static void
 avro_array_print (struct avro_value *value, FILE * fp)
 {
+  int i;
   struct avro_array_value *self =
     container_of (value, struct avro_array_value, base_value);
 
   avro_value_indent (value, fp);
-  fprintf (fp, "array(%p) value items\n", self);
-  avro_value_print_info (self->items, fp);
+  fprintf (fp, "array(%p) with %d items\n", self,
+	   self->values ? self->values->nelts : 0);
+  if (self->values)
+    {
+      for (i = 0; i < self->values->nelts; i++)
+	{
+	  avro_value *item = ((avro_value **) self->values->elts)[i];
+	  avro_value_print_info (item, fp);
+	}
+    }
 }
 
 static avro_status_t
@@ -55,8 +61,6 @@
     container_of (value, struct avro_array_value, base_value);
   struct avro_io_reader *io;
 
-  apr_pool_clear (self->pool);
-
   if (!reader)
     {
       return AVRO_FAILURE;
@@ -76,9 +80,59 @@
   if (count < 0)
     {
       /* TODO */
+      return AVRO_FAILURE;
     }
 
-  /* TODO */
+  /* Clear the pool to of any previous state */
+  apr_pool_clear (self->pool);
+
+  self->values =
+    skip ? NULL : apr_array_make (self->pool, count, sizeof (avro_value *));
+
+  while (count > 0)
+    {
+      /* Read in the count number of items */
+      if (skip)
+	{
+	  avro_value *item =
+	    avro_value_from_json (self->ctx, self->base_value.parent,
+				  self->item_schema);
+	  for (i = 0; i < count; i++)
+	    {
+	      status = avro_value_skip_data (item, reader);
+	      if (status != AVRO_OK)
+		{
+		  return status;
+		}
+	    }
+	}
+      else
+	{
+	  for (i = 0; i < count; i++)
+	    {
+	      avro_value *item =
+		avro_value_from_json (self->ctx, self->base_value.parent,
+				      self->item_schema);
+	      if (!item)
+		{
+		  return AVRO_FAILURE;
+		}
+	      status = avro_value_read_data (item, reader);
+	      if (status != AVRO_OK)
+		{
+		  return status;
+		}
+	      /* Save the item */
+	      *(avro_value **) apr_array_push (self->values) = item;
+	    }
+	}
+
+      status = avro_read_long (io, &count);
+      if (status != AVRO_OK)
+	{
+	  return status;
+	}
+    }
   return AVRO_OK;
 }
 
@@ -108,13 +162,15 @@
 avro_array_create (struct avro_value_ctx *ctx, struct avro_value *parent,
 		   apr_pool_t * pool, const JSON_value * json)
 {
+  avro_value *value;
   struct avro_array_value *self;
-  const JSON_value *items;
+  apr_pool_t *tmp_pool;
 
   self = apr_palloc (pool, sizeof (struct avro_array_value));
   if (!self)
     {
       return NULL;
+      self->values = NULL;
     }
   self->base_value.type = AVRO_ARRAY;
   self->base_value.pool = pool;
@@ -122,22 +178,31 @@
   self->base_value.schema = json;
 
   /* collect and save required items */
-  items = json_attr_get (json, L"items");
-  if (!items)
+  self->item_schema = json_attr_get (json, L"items");
+  if (!self->item_schema)
+    {
+      return NULL;
+    }
+
+  /* Validate the item schema */
+  if (apr_pool_create (&tmp_pool, pool) != APR_SUCCESS)
     {
       return NULL;
     }
-  self->items = avro_value_from_json (ctx, parent, items);
-  if (!self->items)
+  value = avro_value_from_json (ctx, parent, self->item_schema);
+  apr_pool_clear (tmp_pool);
+  if (!value)
     {
+      /* Invalid item schema */
       return NULL;
     }
+
   /* Create a pool for the value processing */
   if (apr_pool_create (&self->pool, pool) != APR_SUCCESS)
     {
       return NULL;
     }
-  self->value_set = 0;
+  self->values = NULL;
   self->ctx = ctx;
   return &self->base_value;
 }

Modified: hadoop/avro/trunk/src/c/avro_boolean.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_boolean.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_boolean.c (original)
+++ hadoop/avro/trunk/src/c/avro_boolean.c Mon Nov  9 23:08:32 2009
@@ -31,7 +31,7 @@
   struct avro_boolean_value *self =
     container_of (value, struct avro_boolean_value, base_value);
   avro_value_indent (value, fp);
-  fprintf (fp, "boolean");
+  fprintf (fp, "boolean(%p)", self);
   if (self->value_set)
     {
       fprintf (fp, " value=%d", self->value);

Modified: hadoop/avro/trunk/src/c/avro_bytes.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_bytes.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_bytes.c (original)
+++ hadoop/avro/trunk/src/c/avro_bytes.c Mon Nov  9 23:08:32 2009
@@ -35,7 +35,7 @@
     container_of (value, struct avro_bytes_value, base_value);
 
   avro_value_indent (value, fp);
-  fprintf (fp, "bytes");
+  fprintf (fp, "bytes(%p)", self);
   if (self->value_set)
     {
       fprintf (fp, " size=%ld ", self->size);
@@ -95,9 +95,8 @@
 static avro_status_t
 avro_bytes_write (struct avro_value *value, struct avro_writer *writer)
 {
-  struct avro_bytes_value *self =
-    container_of (value, struct avro_bytes_value, base_value);
-  return AVRO_OK;
+  /* TODO */
+  return AVRO_FAILURE;
 }
 
 static struct avro_value *

Modified: hadoop/avro/trunk/src/c/avro_double.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_double.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_double.c (original)
+++ hadoop/avro/trunk/src/c/avro_double.c Mon Nov  9 23:08:32 2009
@@ -31,7 +31,7 @@
   struct avro_double_value *self =
     container_of (value, struct avro_double_value, base_value);
   avro_value_indent (value, fp);
-  fprintf (fp, "double");
+  fprintf (fp, "double(%p)", self);
   if (self->value_set)
     {
       fprintf (fp, " value=%f", self->value);

Modified: hadoop/avro/trunk/src/c/avro_enum.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_enum.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_enum.c (original)
+++ hadoop/avro/trunk/src/c/avro_enum.c Mon Nov  9 23:08:32 2009
@@ -33,7 +33,7 @@
   struct avro_avro_enum *self =
     container_of (value, struct avro_avro_enum, base_value);
   avro_value_indent (value, fp);
-  fprintf (fp, "enum name=%ls", self->name);
+  fprintf (fp, "enum(%p) name=%ls", self, self->name);
   if (self->value_set)
     {
       fprintf (fp, " value=%ls",
@@ -127,7 +127,7 @@
 
   /* register self with named objects */
   apr_hash_set (ctx->named_objects, self->name,
-		wcslen (self->name) * sizeof (wchar_t), &self->base_value);
+		wcslen (self->name) * sizeof (wchar_t), json);
 
   /* collect and save required symbols */
   symbols = json_attr_get_check_type (json, L"symbols", JSON_ARRAY);

Modified: hadoop/avro/trunk/src/c/avro_fixed.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_fixed.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_fixed.c (original)
+++ hadoop/avro/trunk/src/c/avro_fixed.c Mon Nov  9 23:08:32 2009
@@ -17,6 +17,7 @@
 under the License.
 */
 #include "avro_private.h"
+#include "dump.h"
 
 struct avro_fixed_value
 {
@@ -24,6 +25,8 @@
   avro_long_t size;
   void *value;
 
+  int value_set;
+
   avro_value base_value;
 };
 
@@ -33,31 +36,51 @@
   struct avro_fixed_value *self =
     container_of (value, struct avro_fixed_value, base_value);
   avro_value_indent (value, fp);
-  fprintf (fp, "fixed name=%ls size=%ld\n", self->name, self->size);
+  fprintf (fp, "fixed(%p) name=%ls size=%lld ", self, self->name, self->size);
+  if (self->value_set)
+    {
+      dump (fp, self->value, self->size);
+    }
 }
 
 static avro_status_t
-avro_fixed_read (struct avro_value *value, struct avro_reader *reader)
+avro_fixed_read_skip (struct avro_value *value, struct avro_reader *reader,
+		      int skip)
 {
   struct avro_fixed_value *self =
     container_of (value, struct avro_fixed_value, base_value);
-  return AVRO_OK;
+  avro_io_reader *io;
+
+  if (!reader)
+    {
+      return AVRO_FAILURE;
+    }
+  io = reader->io;
+  if (!io)
+    {
+      return AVRO_FAILURE;
+    }
+  self->value_set = !skip;
+  return io->read (io, self->value, self->size);
+}
+
+static avro_status_t
+avro_fixed_read (struct avro_value *value, struct avro_reader *reader)
+{
+  return avro_fixed_read_skip (value, reader, 0);
 }
 
 static avro_status_t
 avro_fixed_skip (struct avro_value *value, struct avro_reader *reader)
 {
-  struct avro_fixed_value *self =
-    container_of (value, struct avro_fixed_value, base_value);
-  return AVRO_OK;
+  return avro_fixed_read_skip (value, reader, 1);
 }
 
 static avro_status_t
 avro_fixed_write (struct avro_value *value, struct avro_writer *writer)
 {
-  struct avro_fixed_value *self =
-    container_of (value, struct avro_fixed_value, base_value);
-  return AVRO_OK;
+  /* TODO */
+  return AVRO_FAILURE;
 }
 
 static struct avro_value *
@@ -86,7 +109,10 @@
       return NULL;
     }
   self->size = size->json_number;
-
+  if (self->size < 0)
+    {
+      return NULL;
+    }
 
   /* collect and save the require name */
   name = json_attr_get_check_type (json, L"name", JSON_STRING);
@@ -97,7 +123,7 @@
   self->name = name->json_string;
   /* register self with named objects */
   apr_hash_set (ctx->named_objects, self->name,
-		wcslen (self->name) * sizeof (wchar_t), &self->base_value);
+		wcslen (self->name) * sizeof (wchar_t), json);
 
   self->value = apr_palloc (pool, self->size);
   if (!self->value)
@@ -105,6 +131,7 @@
       return NULL;
     }
 
+  self->value_set = 0;
   return &self->base_value;
 }
 

Modified: hadoop/avro/trunk/src/c/avro_float.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_float.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_float.c (original)
+++ hadoop/avro/trunk/src/c/avro_float.c Mon Nov  9 23:08:32 2009
@@ -31,7 +31,7 @@
   struct avro_float_value *self =
     container_of (value, struct avro_float_value, base_value);
   avro_value_indent (value, fp);
-  fprintf (fp, "float");
+  fprintf (fp, "float(%p)", self);
   if (self->value_set)
     {
       fprintf (fp, " value=%f", self->value);

Modified: hadoop/avro/trunk/src/c/avro_int.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_int.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_int.c (original)
+++ hadoop/avro/trunk/src/c/avro_int.c Mon Nov  9 23:08:32 2009
@@ -31,7 +31,7 @@
   struct avro_int_value *self =
     container_of (value, struct avro_int_value, base_value);
   avro_value_indent (value, fp);
-  fprintf (fp, "int");
+  fprintf (fp, "int(%p)", self);
   if (self->value_set)
     {
       fprintf (fp, " value=%d", self->value);

Modified: hadoop/avro/trunk/src/c/avro_io_memory.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_io_memory.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_io_memory.c (original)
+++ hadoop/avro/trunk/src/c/avro_io_memory.c Mon Nov  9 23:08:32 2009
@@ -80,7 +80,8 @@
   self->used += len;
   DEBUG (fprintf
 	 (stderr, "avro_io_memory_write %d bytes, %d/%d used\n", len,
-	  self->used, self->len); dump (stderr, addr, len));
+	  self->used, self->len);
+	 dump (stderr, addr, len));
   return AVRO_OK;
 }
 

Modified: hadoop/avro/trunk/src/c/avro_long.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_long.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_long.c (original)
+++ hadoop/avro/trunk/src/c/avro_long.c Mon Nov  9 23:08:32 2009
@@ -31,7 +31,7 @@
   struct avro_long_value *self =
     container_of (value, struct avro_long_value, base_value);
   avro_value_indent (value, fp);
-  fprintf (fp, "long");
+  fprintf (fp, "long(%p)", self);
   if (self->value_set)
     {
       fprintf (fp, " value=%ld", self->value);

Modified: hadoop/avro/trunk/src/c/avro_map.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_map.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_map.c (original)
+++ hadoop/avro/trunk/src/c/avro_map.c Mon Nov  9 23:08:32 2009
@@ -21,8 +21,13 @@
 struct avro_map_value
 {
   apr_hash_t *map;
-  struct avro_value *keys;
-  struct avro_value *values;
+  apr_pool_t *pool;
+  const JSON_value *key_schema;
+  const JSON_value *value_schema;
+
+  /* Need the ctx to dupe value */
+  struct avro_value_ctx *ctx;
+
   struct avro_value base_value;
 };
 
@@ -31,34 +36,115 @@
 {
   struct avro_map_value *self =
     container_of (value, struct avro_map_value, base_value);
+
   avro_value_indent (value, fp);
-  fprintf (fp, "map(%p) key/value\n", self);
-  avro_value_print_info (self->keys, fp);
-  avro_value_print_info (self->values, fp);
+  fprintf (fp, "map(%p)", self);
+  if (self->map)
+    {
+      int i;
+      apr_pool_t *subpool;
+      apr_hash_index_t *hi;
+      apr_ssize_t len;
+
+      fprintf (fp, " start\n");
+      apr_pool_create (&subpool, self->pool);
+      for (i = 0, hi = apr_hash_first (subpool, self->map); hi;
+	   hi = apr_hash_next (hi), i++)
+	{
+	  avro_value *key;
+	  avro_value *value;
+	  apr_hash_this (hi, (void *) &key, &len, (void *) &value);
+	  avro_value_print_info (key, fp);
+	  avro_value_print_info (value, fp);
+	}
+      apr_pool_clear (subpool);
+      avro_value_indent (value, fp);
+      fprintf (fp, "map(%p) end", self);
+    }
+  fprintf (fp, "\n");
 }
 
 static avro_status_t
-avro_map_read (struct avro_value *value, struct avro_reader *reader)
+avro_map_read_skip (struct avro_value *value, struct avro_reader *reader,
+		    int skip)
 {
   struct avro_map_value *self =
     container_of (value, struct avro_map_value, base_value);
+  avro_status_t status;
+  avro_long_t i, count;
+  struct avro_io_reader *io;
+
+  if (!reader)
+    {
+      return AVRO_FAILURE;
+    }
+  io = reader->io;
+  if (!io)
+    {
+      return AVRO_FAILURE;
+    }
+  status = avro_read_long (io, &count);
+  if (status != AVRO_OK)
+    {
+      return status;
+    }
+  apr_pool_clear (self->pool);
+  self->map = skip ? NULL : apr_hash_make (self->pool);
+
+  while (count > 0)
+    {
+      for (i = 0; i < count; i++)
+	{
+	  avro_value *key =
+	    avro_value_from_json (self->ctx, self->base_value.parent,
+				  self->key_schema);
+	  avro_value *value =
+	    avro_value_from_json (self->ctx, self->base_value.parent,
+				  self->value_schema);
+	  if (!key || !value)
+	    {
+	      return AVRO_FAILURE;
+	    }
+	  status = avro_value_read_data (key, reader);
+	  if (status != AVRO_OK)
+	    {
+	      return status;
+	    }
+	  status = avro_value_read_data (value, reader);
+	  if (status != AVRO_OK)
+	    {
+	      return status;
+	    }
+	  apr_hash_set (self->map, key, sizeof (struct avro_value), value);
+	}
+
+      status = avro_read_long (io, &count);
+      if (status != AVRO_OK)
+	{
+	  return status;
+	}
+    }
+
   return AVRO_OK;
 }
 
 static avro_status_t
+avro_map_read (struct avro_value *value, struct avro_reader *reader)
+{
+  return avro_map_read_skip (value, reader, 0);
+}
+
+static avro_status_t
 avro_map_skip (struct avro_value *value, struct avro_reader *reader)
 {
-  struct avro_map_value *self =
-    container_of (value, struct avro_map_value, base_value);
-  return AVRO_OK;
+  return avro_map_read_skip (value, reader, 1);
 }
 
 static avro_status_t
 avro_map_write (struct avro_value *value, struct avro_writer *writer)
 {
-  struct avro_map_value *self =
-    container_of (value, struct avro_map_value, base_value);
-  return AVRO_OK;
+  /* TODO */
+  return AVRO_FAILURE;
 }
 
 static struct avro_value *
@@ -66,8 +152,8 @@
 		 apr_pool_t * pool, const JSON_value * json)
 {
   struct avro_map_value *self;
-  const JSON_value *keys;
-  const JSON_value *values;
+  avro_value *value;
+  apr_pool_t *tmp_pool;
 
   DEBUG (fprintf (stderr, "Creating map\n"));
   self = apr_palloc (pool, sizeof (struct avro_map_value));
@@ -80,35 +166,36 @@
   self->base_value.parent = parent;
   self->base_value.schema = json;
 
-  /* collect and save required keys */
-  keys = json_attr_get (json, L"keys");
-  if (keys)
-    {
-      self->keys = avro_value_from_json (ctx, &self->base_value, keys);
-      if (!self->keys)
-	{
-	  return NULL;
-	}
-    }
-  else
+  /* map keys are always assumed to be strings */
+  char *default_key = "{\"type\":\"string\"}";
+  self->key_schema = JSON_parse (pool, default_key, strlen (default_key));
+  if (!self->key_schema)
     {
-      /* TODO: should keys default to string? */
-      self->keys =
-	avro_value_registry[AVRO_STRING]->create (ctx, &self->base_value,
-						  pool, NULL);
+      return NULL;
     }
 
   /* collect and save required values */
-  values = json_attr_get (json, L"values");
-  if (!values)
+  self->value_schema = json_attr_get (json, L"values");
+  if (!self->value_schema)
     {
       return NULL;
     }
-  self->values = avro_value_from_json (ctx, &self->base_value, values);
-  if (!self->values)
+  /* verify that the value schema is valid */
+  apr_pool_create (&tmp_pool, pool);
+  value = avro_value_from_json (ctx, parent, self->value_schema);
+  apr_pool_clear (tmp_pool);
+  if (!value)
+    {
+      return NULL;
+    }
+
+  /* Create a value pool */
+  if (apr_pool_create (&self->pool, pool) != APR_SUCCESS)
     {
       return NULL;
     }
+  self->map = NULL;
+  self->ctx = ctx;
   return &self->base_value;
 }
 

Modified: hadoop/avro/trunk/src/c/avro_null.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_null.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_null.c (original)
+++ hadoop/avro/trunk/src/c/avro_null.c Mon Nov  9 23:08:32 2009
@@ -34,7 +34,7 @@
 avro_null_print (struct avro_value *value, FILE * fp)
 {
   avro_value_indent (value, fp);
-  fprintf (fp, "null\n");
+  fprintf (fp, "null(%p)\n", value);
 }
 
 static struct avro_value *

Modified: hadoop/avro/trunk/src/c/avro_record.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_record.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_record.c (original)
+++ hadoop/avro/trunk/src/c/avro_record.c Mon Nov  9 23:08:32 2009
@@ -53,7 +53,7 @@
   struct avro_record_value *self =
     container_of (value, struct avro_record_value, base_value);
   avro_value_indent (value, fp);
-  fprintf (fp, "record name=%ls\n", self->name);
+  fprintf (fp, "record(%p) name=%ls\n", self, self->name);
   for (i = 0; i < self->fields->nelts; i++)
     {
       struct avro_value *field =
@@ -182,10 +182,16 @@
 static avro_status_t
 avro_record_write (struct avro_value *value, struct avro_writer *writer)
 {
-/* TODO:
-  struct avro_record_value *record =
+  int i;
+  struct avro_record_value *self =
     container_of (value, struct avro_record_value, base_value);
-*/
+
+  for (i = 0; i < self->fields->nelts; i++)
+    {
+      struct avro_value *field =
+	((struct avro_value **) self->fields->elts)[i];
+      avro_value_write_data (field, writer);
+    }
   return AVRO_OK;
 }
 
@@ -223,7 +229,7 @@
 
   /* register self with named objects */
   apr_hash_set (ctx->named_objects, self->name,
-		wcslen (self->name) * sizeof (wchar_t), &self->base_value);
+		wcslen (self->name) * sizeof (wchar_t), json);
 
   /* collect and save optional namespace */
   self->space = NULL;

Modified: hadoop/avro/trunk/src/c/avro_string.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_string.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_string.c (original)
+++ hadoop/avro/trunk/src/c/avro_string.c Mon Nov  9 23:08:32 2009
@@ -32,7 +32,7 @@
   struct avro_string_value *self =
     container_of (value, struct avro_string_value, base_value);
   avro_value_indent (value, fp);
-  fprintf (fp, "string");
+  fprintf (fp, "string(%p)", self);
   if (self->value_set)
     {
       fprintf (fp, " value=%ls", self->value);

Modified: hadoop/avro/trunk/src/c/avro_union.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_union.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_union.c (original)
+++ hadoop/avro/trunk/src/c/avro_union.c Mon Nov  9 23:08:32 2009
@@ -21,6 +21,8 @@
 struct avro_union_value
 {
   apr_array_header_t *schemas;
+  int value_set;
+  avro_long_t value_index;
   struct avro_value base_value;
 };
 
@@ -34,34 +36,78 @@
 }
 
 static avro_status_t
-avro_union_read (struct avro_value *value, struct avro_reader *reader)
+avro_union_read_skip (struct avro_value *value, struct avro_reader *reader,
+		      int skip)
 {
+  avro_status_t status;
   struct avro_union_value *self =
     container_of (value, struct avro_union_value, base_value);
-  return AVRO_OK;
+  struct avro_io_reader *io;
+  avro_value *requested_value;
+
+  io = reader->io;
+  if (!io)
+    {
+      return AVRO_FAILURE;
+    }
+
+  status = avro_read_long (io, &self->value_index);
+  if (status != AVRO_OK)
+    {
+      return status;
+    }
+
+  if (self->value_index < 0 || self->value_index > self->schemas->nelts)
+    {
+      return AVRO_FAILURE;
+    }
+
+  requested_value = ((avro_value **) self->schemas->elts)[self->value_index];
+  if (!requested_value)
+    {
+      return AVRO_FAILURE;
+    }
+
+  self->value_set = !skip;
+
+  if (skip)
+    {
+      status = avro_value_skip_data (requested_value, reader);
+    }
+  else
+    {
+      status = avro_value_read_data (requested_value, reader);
+    }
+  return status;
+}
+
+static avro_status_t
+avro_union_read (struct avro_value *value, struct avro_reader *reader)
+{
+  return avro_union_read_skip (value, reader, 0);
 }
 
 static avro_status_t
 avro_union_skip (struct avro_value *value, struct avro_reader *reader)
 {
-  struct avro_union_value *self =
-    container_of (value, struct avro_union_value, base_value);
-  return AVRO_OK;
+  return avro_union_read_skip (value, reader, 1);
 }
 
 static avro_status_t
 avro_union_write (struct avro_value *value, struct avro_writer *writer)
 {
-  struct avro_union_value *self =
-    container_of (value, struct avro_union_value, base_value);
-  return AVRO_OK;
+  /* TODO */
+  return AVRO_FAILURE;
 }
 
 static struct avro_value *
 avro_union_create (struct avro_value_ctx *ctx, struct avro_value *parent,
 		   apr_pool_t * pool, const JSON_value * json)
 {
+  avro_value *value;
   struct avro_union_value *self;
+  int i;
+
   DEBUG (fprintf (stderr, "Creating a union\n"));
 
   if (json->type != JSON_ARRAY)
@@ -77,7 +123,26 @@
   self->base_value.pool = pool;
   self->base_value.parent = parent;
   self->base_value.schema = json;
-  /* TODO: check the schemas ... and save them */
+
+  self->schemas =
+    apr_array_make (pool, json->json_array->nelts, sizeof (avro_value *));
+  if (!self->schemas)
+    {
+      return NULL;
+    }
+  for (i = 0; i < json->json_array->nelts; i++)
+    {
+      value =
+	avro_value_from_json (ctx, self->base_value.parent,
+			      ((JSON_value **) json->json_array->elts)[i]);
+      if (!value)
+	{
+	  /* Invalid union schema */
+	  return NULL;
+	}
+      *(avro_value **) apr_array_push (self->schemas) = value;
+    }
+  self->value_set = 0;
   return &self->base_value;
 }
 

Modified: hadoop/avro/trunk/src/c/avro_value.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_value.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_value.c (original)
+++ hadoop/avro/trunk/src/c/avro_value.c Mon Nov  9 23:08:32 2009
@@ -106,6 +106,7 @@
 
 struct avro_decorator_value
 {
+  struct avro_value_ctx *ctx;
   struct avro_value *decoratee;
   struct avro_value base_value;
 };
@@ -116,37 +117,51 @@
   struct avro_decorator_value *self =
     container_of (value, struct avro_decorator_value, base_value);
   avro_value_indent (value, fp);
-  fprintf (fp, "avro decorator\n");
+  fprintf (fp, "decorator(%p)\n", self);
+  avro_value_print_info (self->decoratee, fp);
 }
 
 static avro_status_t
-avro_decorator_read (struct avro_value *value, struct avro_reader *reader)
+avro_decorator_read_skip (struct avro_value *value,
+			  struct avro_reader *reader, int skip)
 {
   struct avro_decorator_value *self =
     container_of (value, struct avro_decorator_value, base_value);
+  /* Create a new decoratee */
+  self->decoratee = avro_value_from_json (self->ctx, value, value->schema);
+  if (!self->decoratee)
+    {
+      return AVRO_FAILURE;
+    }
+  if (skip)
+    {
+      return avro_value_skip_data (self->decoratee, reader);
+    }
   return avro_value_read_data (self->decoratee, reader);
 }
 
 static avro_status_t
+avro_decorator_read (struct avro_value *value, struct avro_reader *reader)
+{
+  return avro_decorator_read_skip (value, reader, 0);
+}
+
+static avro_status_t
 avro_decorator_skip (struct avro_value *value, struct avro_reader *reader)
 {
-  struct avro_decorator_value *self =
-    container_of (value, struct avro_decorator_value, base_value);
-  return avro_value_skip_data (self->decoratee, reader);
+  return avro_decorator_read_skip (value, reader, 1);
 }
 
 static avro_status_t
 avro_decorator_write (struct avro_value *value, struct avro_writer *writer)
 {
-  struct avro_decorator_value *self =
-    container_of (value, struct avro_decorator_value, base_value);
-  return avro_value_write_data (self->decoratee, writer);
+  return AVRO_FAILURE;
 }
 
 /* Used for recursive schemas */
 struct avro_value *
 avro_decorator_create (struct avro_value_ctx *ctx, struct avro_value *parent,
-		       apr_pool_t * pool, struct avro_value *decoratee)
+		       apr_pool_t * pool, const JSON_value * json)
 {
   struct avro_decorator_value *self =
     apr_palloc (pool, sizeof (struct avro_decorator_value));
@@ -155,28 +170,20 @@
     {
       return NULL;
     }
+  /* Save away the context */
+  self->ctx = ctx;
   self->base_value.type = AVRO_DECORATOR;
   self->base_value.pool = pool;
   self->base_value.parent = parent;
-  self->base_value.schema = decoratee->schema;
-  /* object we're decorating */
-  self->decoratee = decoratee;
+  self->base_value.schema = json;
   return &self->base_value;
 }
 
-static struct avro_value *
-avro_decorator_create_noop (struct avro_value_ctx *ctx,
-			    struct avro_value *parent, apr_pool_t * pool,
-			    const JSON_value * json)
-{
-  return NULL;
-}
-
 const struct avro_value_info avro_decorator_info = {
   .name = L"decorator",
   .type = AVRO_DECORATOR,
   .private = 1,
-  .create = avro_decorator_create_noop,
+  .create = avro_decorator_create,
   .formats = {{
 	       .read_data = avro_decorator_read,
 	       .skip_data = avro_decorator_skip,
@@ -198,6 +205,7 @@
   apr_status_t status;
   avro_type_t avro_type;
   apr_pool_t *subpool;
+  const JSON_value *schema = json;
 
   status = apr_pool_create (&subpool, parent ? parent->pool : NULL);
   if (status != APR_SUCCESS)
@@ -205,25 +213,25 @@
       return NULL;
     }
 
-  avro_status = avro_type_from_json (json, &avro_type);
+  avro_status = avro_type_from_json (schema, &avro_type);
   if (avro_status != AVRO_OK)
     {
-      if (json->type == JSON_STRING)
+      if (!ctx || json->type != JSON_STRING)
 	{
-	  avro_value *named_object =
-	    apr_hash_get (ctx->named_objects, json->json_string,
-			  wcslen (json->json_string) * sizeof (wchar_t));
-	  if (named_object)
-	    {
-	      return avro_decorator_create (ctx, parent, subpool,
-					    named_object);
-	    }
+	  return NULL;
 	}
-      return NULL;
+      schema =
+	apr_hash_get (ctx->named_objects, json->json_string,
+		      wcslen (json->json_string) * sizeof (wchar_t));
+      if (!schema)
+	{
+	  return NULL;
+	}
+      avro_type = AVRO_DECORATOR;
     }
 
-  return avro_value_registry[avro_type]->private ? NULL :
-    avro_value_registry[avro_type]->create (ctx, parent, subpool, json);
+  return avro_value_registry[avro_type]->create (ctx, parent, subpool,
+						 schema);
 }
 
 struct avro_value *

Added: hadoop/avro/trunk/src/c/config/.gitignore
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/config/.gitignore?rev=834276&view=auto
==============================================================================
--- hadoop/avro/trunk/src/c/config/.gitignore (added)
+++ hadoop/avro/trunk/src/c/config/.gitignore Mon Nov  9 23:08:32 2009
@@ -0,0 +1 @@
+*

Modified: hadoop/avro/trunk/src/c/configure.in
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/configure.in?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/configure.in (original)
+++ hadoop/avro/trunk/src/c/configure.in Mon Nov  9 23:08:32 2009
@@ -78,12 +78,13 @@
 AC_OUTPUT
 
 echo
-echo "C bindings for"
-echo "    _                  "
-echo "   / \__   ___ __ ___  "
-echo "  / _ \ \ / / '__/ _ \ "
-echo " / ___ \ V /| | | (_) |"
-echo "/_/   \_\_/ |_|  \___/ "
+echo
+# figlet -f slant "Avro C"
+echo "    ___                      ______"
+echo "   /   |_   ___________     / ____/"
+echo "  / /| | | / / ___/ __ \   / /     "
+echo " / ___ | |/ / /  / /_/ /  / /___   "
+echo "/_/  |_|___/_/   \____/   \____/   "
 echo
 echo "Version: $VERSION"
 echo "Library: $LIBAVRO_VERSION" 	       

Modified: hadoop/avro/trunk/src/c/json.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/json.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/json.c (original)
+++ hadoop/avro/trunk/src/c/json.c Mon Nov  9 23:08:32 2009
@@ -34,7 +34,7 @@
 }
 
 static void
-JSON_print_private (FILE * file, JSON_value * value, int *depth)
+JSON_print_private (FILE * file, const JSON_value * value, int *depth)
 {
   int i;
   switch (value->type)
@@ -105,7 +105,7 @@
 }
 
 void
-JSON_print (FILE * file, JSON_value * value)
+JSON_print (FILE * file, const JSON_value * value)
 {
   int depth = 0;
   if (!file || !value)

Modified: hadoop/avro/trunk/src/c/json.h
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/json.h?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/json.h (original)
+++ hadoop/avro/trunk/src/c/json.h Mon Nov  9 23:08:32 2009
@@ -61,7 +61,7 @@
 
 JSON_value *JSON_value_new (apr_pool_t * pool, int type);
 
-void JSON_print (FILE * file, JSON_value * value);
+void JSON_print (FILE * file, const JSON_value * value);
 
 struct JSON_ctx
 {

Copied: hadoop/avro/trunk/src/c/test_avro_data.c (from r833621, hadoop/avro/trunk/src/c/test_avro_primitives.c)
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/test_avro_data.c?p2=hadoop/avro/trunk/src/c/test_avro_data.c&p1=hadoop/avro/trunk/src/c/test_avro_primitives.c&r1=833621&r2=834276&rev=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/test_avro_primitives.c (original)
+++ hadoop/avro/trunk/src/c/test_avro_data.c Mon Nov  9 23:08:32 2009
@@ -174,13 +174,14 @@
   for (i = 0; i < num_rand_tests; i++)
     {
       avro_long_t long_in, long_out;
-      avro_int_t int_in, int_out, *intp;
+      avro_int_t int_in, int_out;
       rand_val = random ();
       if (is_long)
 	{
-	  intp = (avro_int_t *) & long_in;
-	  intp[0] = random ();
-	  intp[1] = random ();
+	  avro_long_t a, b;
+	  a = random ();
+	  b = random ();
+	  long_in = (a << 32) | b;
 	}
       else
 	{
@@ -349,6 +350,48 @@
   return AVRO_OK;
 }
 
+avro_status_t
+test_record (apr_pool_t * pool, avro_io_reader * reader,
+	     avro_io_writer * writer)
+{
+  return AVRO_OK;
+}
+
+avro_status_t
+test_enum (apr_pool_t * pool, avro_io_reader * reader,
+	   avro_io_writer * writer)
+{
+  return AVRO_OK;
+}
+
+avro_status_t
+test_array (apr_pool_t * pool, avro_io_reader * reader,
+	    avro_io_writer * writer)
+{
+  return AVRO_OK;
+}
+
+avro_status_t
+test_map (apr_pool_t * pool, avro_io_reader * reader, avro_io_writer * writer)
+{
+  return AVRO_OK;
+}
+
+avro_status_t
+test_union (apr_pool_t * pool, avro_io_reader * reader,
+	    avro_io_writer * writer)
+{
+  return AVRO_OK;
+}
+
+avro_status_t
+test_fixed (apr_pool_t * pool, avro_io_reader * reader,
+	    avro_io_writer * writer)
+{
+  return AVRO_OK;
+}
+
+
 int
 main (void)
 {
@@ -378,7 +421,17 @@
     {
     "boolean", test_boolean},
     {
-    "null", test_null}
+    "null", test_null},
+    {
+    "record", test_record},
+    {
+    "enum", test_enum},
+    {
+    "array", test_array},
+    {
+    "map", test_map},
+    {
+    "fixed", test_fixed}
   };
 
   srandom (time (NULL));

Modified: hadoop/avro/trunk/src/c/test_avro_interop.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/test_avro_interop.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/test_avro_interop.c (original)
+++ hadoop/avro/trunk/src/c/test_avro_interop.c Mon Nov  9 23:08:32 2009
@@ -5,11 +5,6 @@
 apr_pool_t *pool;
 
 /*
-TODO:
-
-We still need to create a File Object container to finish this test.
-For now, we're only validating that we can parse the schema.
-
 $ ant generate-test-data
 needs to be called to make sure that test data is created.
 
@@ -25,6 +20,9 @@
   apr_size_t jsonlen;
   struct avro_value *value;
   struct avro_reader *reader;
+  char *suffixes[] = { "py", "java" };
+  char *suffix;
+  int i;
 
   if (!srcdir)
     {
@@ -43,8 +41,10 @@
   jsontext = avro_util_file_read_full (pool, path, &jsonlen);
   if (!jsontext)
     {
-      fprintf (stderr, "Failed to parse the JSON in file %s\n", path);
-      return EXIT_FAILURE;
+      fprintf (stderr,
+	       "Couldn't find file: %s. Can't run interop test out of tree\n",
+	       path);
+      return EXIT_SUCCESS;
     }
   value = avro_value_create (pool, jsontext, jsonlen);
   if (!value)
@@ -53,23 +53,29 @@
       return EXIT_FAILURE;
     }
 
-  path =
-    apr_pstrcat (pool, srcdir, "/../../build/test/data-files/test.py.avro",
-		 NULL);
-  if (!path)
-    {
-      return EXIT_FAILURE;
-    }
-  reader =
-    avro_reader_file_container_create (pool, path, APR_READ, APR_OS_DEFAULT);
-  if (!reader)
+  for (i = 0; i < sizeof (suffixes) / sizeof (suffixes[0]); i++)
     {
-      fprintf (stderr, "Failed to open data file %s\n", path);
-      return EXIT_FAILURE;
-    }
+      suffix = suffixes[i];
 
-  avro_value_read_data (value, reader);
-  avro_value_print_info (value, stderr);
+      path =
+	apr_pstrcat (pool, srcdir, "/../../build/test/data-files/test.",
+		     suffix, ".avro", NULL);
+      if (!path)
+	{
+	  return EXIT_FAILURE;
+	}
+      fprintf (stderr, "Running test on %s...\n", path);
+      reader =
+	avro_reader_file_container_create (pool, path, APR_READ,
+					   APR_OS_DEFAULT);
+      if (!reader)
+	{
+	  fprintf (stderr, "Failed to open data file %s\n", path);
+	  return EXIT_FAILURE;
+	}
+      avro_value_read_data (value, reader);
+      avro_value_print_info (value, stderr);
+    }
   apr_pool_destroy (pool);
   return EXIT_SUCCESS;
 }

Modified: hadoop/avro/trunk/src/test/schemata/interop.avsc
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/schemata/interop.avsc?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/test/schemata/interop.avsc (original)
+++ hadoop/avro/trunk/src/test/schemata/interop.avsc Mon Nov  9 23:08:32 2009
@@ -10,7 +10,7 @@
       {"name": "nullField", "type": "null"},
       {"name": "arrayField", "type": {"type": "array", "items": "double"}},
       {"name": "mapField", "type":
-       {"type": "map", "keys": "long", "values":
+       {"type": "map", "values":
         {"type": "record", "name": "Foo",
          "fields": [{"name": "label", "type": "string"}]}}},
       {"name": "unionField", "type":



Mime
View raw message