aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wfar...@apache.org
Subject git commit: Generate wrapper classes for all thrift structs in api.thrift.
Date Thu, 26 Jun 2014 19:09:47 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master 6df1d020d -> ea2e8c4ec


Generate wrapper classes for all thrift structs in api.thrift.

Reviewed at https://reviews.apache.org/r/22998/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/ea2e8c4e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/ea2e8c4e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/ea2e8c4e

Branch: refs/heads/master
Commit: ea2e8c4ec29618d743bfcc3dd5e25fd4c7d4bd75
Parents: 6df1d02
Author: Bill Farner <wfarner@apache.org>
Authored: Thu Jun 26 11:58:15 2014 -0700
Committer: Bill Farner <wfarner@apache.org>
Committed: Thu Jun 26 11:58:15 2014 -0700

----------------------------------------------------------------------
 build.gradle                                    |  21 +--
 .../aurora/tools/java/thrift_wrapper_codegen.py | 144 ++++++++-----------
 2 files changed, 63 insertions(+), 102 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/ea2e8c4e/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index be2ff05..751bde2 100644
--- a/build.gradle
+++ b/build.gradle
@@ -401,22 +401,11 @@ task generateSources(dependsOn: ['bootstrapThrift', 'checkPython'])
{
         args = ['--gen', 'java:hashcode', '--gen', 'js:jquery', '-o', outputDir, file]
       }
     }
-    // TODO(wfarner): Change codegen tool to generate for all structs in a thrift file.
-    def iStructs = ['JobConfiguration',
-                    'JobStats',
-                    'Lock',
-                    'ResourceAggregate',
-                    'ScheduledTask',
-                    'ServerInfo']
-    iStructs.each {
-      def structName = it
-      exec {
-        executable = project.py
-        args = ['src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py',
-                'src/main/thrift/org/apache/aurora/gen/api.thrift',
-                structName,
-                generatedJavaDir]
-      }
+    exec {
+      executable = project.py
+      args = ['src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py',
+              'src/main/thrift/org/apache/aurora/gen/api.thrift',
+              generatedJavaDir]
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/ea2e8c4e/src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py b/src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py
index 9dc92fd..d16cfdd 100644
--- a/src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py
+++ b/src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py
@@ -117,7 +117,7 @@ IMMUTABLE_COLLECTION_ASSIGNMENT = '''this.%(field)s = !wrapped.%(isset)s()
         : Immutable%(collection)s.copyOf(wrapped.%(fn_name)s());'''
 
 
-# Tempalte string for assignment for a collection field containing a struct.
+# Template string for assignment for a collection field containing a struct.
 STRUCT_COLLECTION_FIELD_ASSIGNMENT = '''this.%(field)s = !wrapped.%(isset)s()
         ? Immutable%(collection)s.<%(params)s>of()
         : FluentIterable.from(wrapped.%(fn_name)s())
@@ -300,6 +300,17 @@ STRUCT_RE = '(?P<kind>enum|struct|union)\s+(?P<name>\w+)\s+{(?P<body>[^}]+)}'
 #     15: Map<String, TaskConfig> configs  # Configs mapped by name.
 FIELD_RE = '\s*\d+:\s+(?:(?:required|optional)\s+)?(%s)\s+(?P<name>\w+).*' % TYPE_PATTERN
 
+THRIFT_TYPES = {
+  'bool': PrimitiveType('boolean', 'Boolean'),
+  'i32': PrimitiveType('int', 'Integer'),
+  'i64': PrimitiveType('long', 'Long'),
+  'double': PrimitiveType('double', 'Double'),
+  'string': PrimitiveType('String', 'String'),
+  'list': Type('List'),
+  'set': Type('Set'),
+  'map': Type('Map'),
+  'binary': PrimitiveType('byte[]', 'byte[]'),
+}
 
 def parse_structs(thrift_defs):
   '''Read all thrift structures found in a file.
@@ -310,24 +321,36 @@ def parse_structs(thrift_defs):
   # Capture all namespace definitions.
   namespaces = dict(re.findall(NAMESPACE_RE, thrift_defs))
 
+  # Keep track of structs already seen, to identify referenced types.
+  structs = []
+
   def parse_field(field):
+    def make_type(name):
+      if name in ['list', 'map', 'set']:
+        return Type(name.title())
+      elif name in THRIFT_TYPES:
+        return THRIFT_TYPES[name]
+      else:
+        return [s for s in structs if s.name == name][0]
+
     type_name = field.group('type')
     type_params = field.group('params')
     if type_params:
-      params = [Type(p) for p in type_params.replace(' ', '').split(',')]
-      ttype = ParameterizedType(type_name, params)
+      params = [make_type(p) for p in type_params.replace(' ', '').split(',')]
+      ttype = ParameterizedType(type_name.title(), params)
     else:
-      ttype = Type(type_name)
+      ttype = make_type(type_name)
     return Field(ttype, field.group('name'))
 
   def parse_fields(field_str):
     return map(parse_field, re.finditer(FIELD_RE, field_str))
 
-  return [StructType(s.group('name'),
-                     namespaces['java'],
-                     s.group('kind'),
-                     parse_fields(s.group('body')))
-          for s in re.finditer(STRUCT_RE, thrift_defs, flags=re.MULTILINE)]
+  for s in re.finditer(STRUCT_RE, thrift_defs, flags=re.MULTILINE):
+    structs.append(StructType(s.group('name'),
+                              namespaces['java'],
+                              s.group('kind'),
+                              parse_fields(s.group('body'))))
+  return structs
 
 
 def generate_java(struct):
@@ -345,11 +368,9 @@ def generate_java(struct):
 
   # Accessor for each field.
   for field in struct.fields:
-    if not (isinstance(field.ttype, StructType) and (
-        field.ttype.kind == 'enum' or struct.kind == 'union')):
-      code.add_accessor(IMMUTABLE_FIELD_TEMPLATE
-                        % {'type': 'boolean',
-                           'fn_name': field.isset_method()})
+    code.add_accessor(IMMUTABLE_FIELD_TEMPLATE
+                      % {'type': 'boolean',
+                         'fn_name': field.isset_method()})
     if field.ttype.immutable:
       code.add_accessor(IMMUTABLE_FIELD_TEMPLATE % {'type': field.ttype.name,
                                                     'fn_name': field.accessor_method()})
@@ -357,7 +378,14 @@ def generate_java(struct):
       if isinstance(field.ttype, StructType):
         return_type = field.ttype.codegen_name
       elif isinstance(field.ttype, ParameterizedType):
-        return_type = '%s<%s>' % (field.ttype.name, field.ttype.param_names())
+        # Add imports for any referenced enum types. This is not necessary for other
+        # types since they are either primitives or struct types, which will be in
+        # the same package.
+        for param_type in field.ttype.params:
+          if isinstance(param_type, StructType) and param_type.kind == 'enum':
+            code.add_import(param_type.absolute_name())
+
+        return_type = 'Immutable%s<%s>' % (field.ttype.name, field.ttype.param_names())
       else:
         return_type = field.ttype.name
       code.add_accessor(FIELD_TEMPLATE % {'type': return_type,
@@ -389,7 +417,6 @@ def generate_java(struct):
       # Add necessary imports, supporting only List, Map, Set.
       assert field.ttype.name in ['List', 'Map', 'Set'], 'Unrecognized type %s' % field.ttype.name
       code.add_import('com.google.common.collect.Immutable%s' % field.ttype.name)
-      code.add_import('java.util.%s' % field.ttype.name)
 
       params = field.ttype.params
       if all([p.immutable for p in params]):
@@ -408,24 +435,8 @@ def generate_java(struct):
               'isset': field.isset_method(),
               'params': field.ttype.param_names()}
       code.add_assignment(IMMUTABLE_COLLECTION_DECLARATION % args, assignment % args)
-    elif not field.ttype.immutable:
-      assert False, 'Making type %s immutable is not supported.' % field.ttype.name
   return code
 
-
-THRIFT_ALIASES = {
-  'bool': 'boolean',
-  'i32': 'int',
-  'i64': 'long',
-  'double': 'double',
-  'string': 'String',
-  'list': 'List',
-  'set': 'Set',
-  'map': 'Map',
-  'binary': 'byte[]',
-}
-
-
 if __name__ == '__main__':
   parser = OptionParser()
   parser.add_option('-v', '--verbose',
@@ -438,63 +449,24 @@ if __name__ == '__main__':
     if options.verbose:
       print(value)
 
-  if len(args) != 3:
-    print('usage: %s thrift_file struct_name output_directory' % sys.argv[0])
+  if len(args) != 2:
+    print('usage: %s thrift_file output_directory' % sys.argv[0])
     sys.exit(1)
 
-  thrift_file, struct_name, output_directory = args
-  log('Searching for %s in %s' % (struct_name, thrift_file))
+  thrift_file, output_directory = args
   with open(thrift_file) as f:
     # Load all structs found in the thrift file.
     structs = parse_structs(f.read())
 
-    # The symbol table stores information about types we recognize.
-    # As new symbols are parsed, they are accumulated here.
-    # This is also seeded with JDK types.
-    # Note: byte[] is not immutable, but we'd rather accept it than copy.
-    primitives = dict((t, PrimitiveType(t, b)) for (t, b) in [('boolean', 'Boolean'),
-                                                             ('int', 'Integer'),
-                                                             ('long', 'Long'),
-                                                             ('double', 'Double')])
-    lang_symbols = dict((t, Type(t, 'java.lang', immutable=True)) for t in ['String', 'byte[]'])
-    util_symbols = dict((t, Type(t, 'java.util')) for t in ['List', 'Map', 'Set'])
-    symbol_table = dict(primitives.items() + lang_symbols.items() + util_symbols.items())
-
-    def load_dependencies(struct):
-      # Fill in type information for fields by searching for dependencies.
-      for field in struct.fields:
-        if isinstance(field.ttype, ParameterizedType):
-          field.ttype.name = find_symbol(field.ttype.name).name
-          field.ttype.params = [find_symbol(p.name) for p in field.ttype.params]
-        else:
-          field.ttype = find_symbol(field.ttype.name)
-
-    def find_symbol(name):
-      name = THRIFT_ALIASES.get(name, name)
-      if name in symbol_table:
-        return symbol_table[name]
-
-      symbol = next((s for s in structs if s.name == name), None)
-      assert symbol, 'Failed to find required struct %s' % name
-      load_dependencies(symbol)
-      symbol_table[name] = symbol
-      return symbol
-
-    find_symbol(struct_name)
-    log('Symbol table:')
-    for _, symbol in symbol_table.items():
-      log('    %s' % symbol)
-
-    for _, symbol in symbol_table.items():
-      if isinstance(symbol, StructType):
-        if symbol.kind == 'enum':
-          log('Skipping code generation for %s, since it is immutable' % symbol.name)
-        else:
-          package_dir = os.path.join(output_directory, PACKAGE_NAME.replace('.', os.path.sep))
-          if not os.path.isdir(package_dir):
-            os.makedirs(package_dir)
-          gen_file = os.path.join(package_dir, '%s.java' % symbol.codegen_name)
-          log('Generating %s' % gen_file)
-          with open(gen_file, 'w') as f:
-            code = generate_java(symbol)
-            code.dump(f)
+    package_dir = os.path.join(output_directory, PACKAGE_NAME.replace('.', os.path.sep))
+    if not os.path.isdir(package_dir):
+      os.makedirs(package_dir)
+    for struct in structs:
+      # Skip generation for enums, since they are immutable.
+      if struct.kind == 'enum':
+        continue
+      gen_file = os.path.join(package_dir, '%s.java' % struct.codegen_name)
+      log('Generating %s' % gen_file)
+      with open(gen_file, 'w') as f:
+        code = generate_java(struct)
+        code.dump(f)


Mime
View raw message