Return-Path: X-Original-To: apmail-kylin-commits-archive@minotaur.apache.org Delivered-To: apmail-kylin-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1B48C105FF for ; Wed, 14 Jan 2015 14:16:24 +0000 (UTC) Received: (qmail 4790 invoked by uid 500); 14 Jan 2015 14:16:25 -0000 Delivered-To: apmail-kylin-commits-archive@kylin.apache.org Received: (qmail 4758 invoked by uid 500); 14 Jan 2015 14:16:25 -0000 Mailing-List: contact commits-help@kylin.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kylin.incubator.apache.org Delivered-To: mailing list commits@kylin.incubator.apache.org Received: (qmail 4749 invoked by uid 99); 14 Jan 2015 14:16:25 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 14 Jan 2015 14:16:25 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED,T_RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Wed, 14 Jan 2015 14:16:01 +0000 Received: (qmail 3220 invoked by uid 99); 14 Jan 2015 14:15:38 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 14 Jan 2015 14:15:38 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id E9D0FA4192C; Wed, 14 Jan 2015 14:15:37 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: lukehan@apache.org To: commits@kylin.incubator.apache.org Date: Wed, 14 Jan 2015 14:16:10 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [35/51] [partial] incubator-kylin git commit: cleanup for migration from github.com X-Virus-Checked: Checked by ClamAV on apache.org http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/dictionary/.settings/org.eclipse.core.resources.prefs ---------------------------------------------------------------------- diff --git a/dictionary/.settings/org.eclipse.core.resources.prefs b/dictionary/.settings/org.eclipse.core.resources.prefs deleted file mode 100644 index 04cfa2c..0000000 --- a/dictionary/.settings/org.eclipse.core.resources.prefs +++ /dev/null @@ -1,6 +0,0 @@ -eclipse.preferences.version=1 -encoding//src/main/java=UTF-8 -encoding//src/main/resources=UTF-8 -encoding//src/test/java=UTF-8 -encoding//src/test/resources=UTF-8 -encoding/=UTF-8 http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/dictionary/.settings/org.eclipse.jdt.core.prefs ---------------------------------------------------------------------- diff --git a/dictionary/.settings/org.eclipse.jdt.core.prefs b/dictionary/.settings/org.eclipse.jdt.core.prefs deleted file mode 100644 index a903301..0000000 --- a/dictionary/.settings/org.eclipse.jdt.core.prefs +++ /dev/null @@ -1,379 +0,0 @@ -eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled -org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore -org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull -org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault -org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable -org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled -org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 -org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.7 -org.eclipse.jdt.core.compiler.debug.lineNumber=generate -org.eclipse.jdt.core.compiler.debug.localVariable=generate -org.eclipse.jdt.core.compiler.debug.sourceFile=generate -org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning -org.eclipse.jdt.core.compiler.problem.assertIdentifier=error -org.eclipse.jdt.core.compiler.problem.autoboxing=ignore -org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning -org.eclipse.jdt.core.compiler.problem.deadCode=warning -org.eclipse.jdt.core.compiler.problem.deprecation=warning -org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled -org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled -org.eclipse.jdt.core.compiler.problem.discouragedReference=warning -org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore -org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore -org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore -org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled -org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore -org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning -org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning -org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning -org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning -org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled -org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning -org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning -org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore -org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore -org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning -org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore -org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore -org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled -org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore -org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore -org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled -org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning -org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore -org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning -org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning -org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore -org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning -org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error -org.eclipse.jdt.core.compiler.problem.nullReference=warning -org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error -org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning -org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning -org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore -org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore -org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore -org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore -org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning -org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning -org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore -org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore -org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore -org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore -org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore -org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled -org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning -org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled -org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled -org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled -org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore -org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning -org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled -org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning -org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning -org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore -org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning -org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore -org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore -org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled -org.eclipse.jdt.core.compiler.problem.unusedImport=warning -org.eclipse.jdt.core.compiler.problem.unusedLabel=warning -org.eclipse.jdt.core.compiler.problem.unusedLocal=warning -org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore -org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore -org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled -org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled -org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled -org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning -org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore -org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning -org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning -org.eclipse.jdt.core.compiler.source=1.7 -org.eclipse.jdt.core.formatter.align_type_members_on_columns=false -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16 -org.eclipse.jdt.core.formatter.alignment_for_assignment=0 -org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16 -org.eclipse.jdt.core.formatter.alignment_for_compact_if=16 -org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80 -org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0 -org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16 -org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0 -org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16 -org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80 -org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16 -org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16 -org.eclipse.jdt.core.formatter.blank_lines_after_imports=1 -org.eclipse.jdt.core.formatter.blank_lines_after_package=1 -org.eclipse.jdt.core.formatter.blank_lines_before_field=0 -org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0 -org.eclipse.jdt.core.formatter.blank_lines_before_imports=1 -org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1 -org.eclipse.jdt.core.formatter.blank_lines_before_method=1 -org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1 -org.eclipse.jdt.core.formatter.blank_lines_before_package=0 -org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1 -org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1 -org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line -org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false -org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false -org.eclipse.jdt.core.formatter.comment.format_block_comments=false -org.eclipse.jdt.core.formatter.comment.format_header=false -org.eclipse.jdt.core.formatter.comment.format_html=true -org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=false -org.eclipse.jdt.core.formatter.comment.format_line_comments=false -org.eclipse.jdt.core.formatter.comment.format_source_code=true -org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true -org.eclipse.jdt.core.formatter.comment.indent_root_tags=true -org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert -org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert -org.eclipse.jdt.core.formatter.comment.line_length=80 -org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true -org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true -org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false -org.eclipse.jdt.core.formatter.compact_else_if=true -org.eclipse.jdt.core.formatter.continuation_indentation=2 -org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2 -org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off -org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on -org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false -org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=true -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true -org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true -org.eclipse.jdt.core.formatter.indent_empty_lines=false -org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true -org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true -org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true -org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false -org.eclipse.jdt.core.formatter.indentation.size=4 -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert -org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert -org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert -org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert -org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert -org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert -org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert -org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert -org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert -org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert -org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert -org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert -org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert -org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.join_lines_in_comments=true -org.eclipse.jdt.core.formatter.join_wrapped_lines=true -org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false -org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false -org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false -org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false -org.eclipse.jdt.core.formatter.lineSplit=999 -org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false -org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false -org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0 -org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1 -org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true -org.eclipse.jdt.core.formatter.tabulation.char=space -org.eclipse.jdt.core.formatter.tabulation.size=4 -org.eclipse.jdt.core.formatter.use_on_off_tags=false -org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false -org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true -org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true -org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/dictionary/.settings/org.eclipse.jdt.ui.prefs ---------------------------------------------------------------------- diff --git a/dictionary/.settings/org.eclipse.jdt.ui.prefs b/dictionary/.settings/org.eclipse.jdt.ui.prefs deleted file mode 100644 index dece0e6..0000000 --- a/dictionary/.settings/org.eclipse.jdt.ui.prefs +++ /dev/null @@ -1,7 +0,0 @@ -eclipse.preferences.version=1 -formatter_profile=_Space Indent & Long Lines -formatter_settings_version=12 -org.eclipse.jdt.ui.ignorelowercasenames=true -org.eclipse.jdt.ui.importorder=java;javax;org;com; -org.eclipse.jdt.ui.ondemandthreshold=99 -org.eclipse.jdt.ui.staticondemandthreshold=1 http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/dictionary/pom.xml ---------------------------------------------------------------------- diff --git a/dictionary/pom.xml b/dictionary/pom.xml deleted file mode 100644 index 1d09903..0000000 --- a/dictionary/pom.xml +++ /dev/null @@ -1,78 +0,0 @@ - - 4.0.0 - - kylin-dictionary - jar - Kylin:Dictionary - - - com.kylinolap - kylin - 0.6.3-SNAPSHOT - - - - - - - - - com.kylinolap - kylin-metadata - ${project.parent.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - commons-io - commons-io - - - commons-configuration - commons-configuration - - - com.google.guava - guava - - - - - - org.apache.hadoop - hadoop-common - provided - - - org.apache.hadoop - hadoop-hdfs - provided - - - - com.google.protobuf - protobuf-java - - - - - org.apache.hbase - hbase-common - provided - - - org.apache.hbase - hbase-client - provided - - - junit - junit - test - - - - http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/dictionary/src/main/java/com/kylinolap/dict/BytesConverter.java ---------------------------------------------------------------------- diff --git a/dictionary/src/main/java/com/kylinolap/dict/BytesConverter.java b/dictionary/src/main/java/com/kylinolap/dict/BytesConverter.java deleted file mode 100644 index cff65c5..0000000 --- a/dictionary/src/main/java/com/kylinolap/dict/BytesConverter.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.kylinolap.dict; - -public interface BytesConverter { - - public byte[] convertToBytes(T v); - - public T convertFromBytes(byte[] b, int offset, int length); -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java ---------------------------------------------------------------------- diff --git a/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java b/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java deleted file mode 100644 index ed26aae..0000000 --- a/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.kylinolap.dict; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.io.PrintStream; -import java.io.UnsupportedEncodingException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Map; -import java.util.TimeZone; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.commons.lang.StringUtils; - -/** - * A dictionary for date string (date only, no time). - * - * Dates are numbered from 1970-1-1 -- 0 for 1970-1-1, 1 for 1-2, 2 for 1-3 and - * so on. With 2 bytes, 65536 states, can express dates up to the year of 2149. - * - * Note the implementation is not thread-safe. - * - * @author yangli9 - */ -public class DateStrDictionary extends Dictionary { - - static final String DEFAULT_DATE_PATTERN = "yyyy-MM-dd"; - static final String DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS = "yyyy-MM-dd HH:mm:ss"; - static final String DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS = "yyyy-MM-dd HH:mm:ss.SSS"; - - static final private Map> threadLocalMap = new ConcurrentHashMap>(); - - static SimpleDateFormat getDateFormat(String datePattern) { - ThreadLocal formatThreadLocal = threadLocalMap.get(datePattern); - if (formatThreadLocal == null) { - threadLocalMap.put(datePattern, formatThreadLocal = new ThreadLocal()); - } - SimpleDateFormat format = formatThreadLocal.get(); - if (format == null) { - format = new SimpleDateFormat(datePattern); - format.setTimeZone(TimeZone.getTimeZone("GMT")); // NOTE: this must - // be GMT to - // calculate - // epoch date - // correctly - formatThreadLocal.set(format); - } - return format; - } - - public static String dateToString(Date date) { - return dateToString(date, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS); - } - - public static String dateToString(Date date, String pattern) { - return getDateFormat(pattern).format(date); - } - - public static Date stringToDate(String str) { - return stringToDate(str, DEFAULT_DATE_PATTERN); - } - - public static Date stringToDate(String str, String pattern) { - Date date = null; - try { - date = getDateFormat(pattern).parse(str); - } catch (ParseException e) { - throw new IllegalArgumentException("'" + str + "' is not a valid date of pattern '" + pattern + "'", e); - } - return date; - } - - public static long stringToMillis(String str) { - if (str.length() == 10) { - return stringToDate(str, DEFAULT_DATE_PATTERN).getTime(); - } else if (str.length() == 19) { - return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS).getTime(); - } else if (str.length() == 23) { - return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).getTime(); - } else { - throw new IllegalArgumentException("there is no valid date pattern for:" + str); - } - } - - // ============================================================================ - - private String pattern; - private int baseId; - - public DateStrDictionary() { - init(DEFAULT_DATE_PATTERN, 0); - } - - public DateStrDictionary(String datePattern, int baseId) { - init(datePattern, baseId); - } - - private void init(String datePattern, int baseId) { - this.pattern = datePattern; - this.baseId = baseId; - } - - @Override - public int getMinId() { - return baseId; - } - - @Override - public int getMaxId() { - return Integer.MAX_VALUE; - } - - @Override - public int getSizeOfId() { - return 3; - } - - @Override - public int getSizeOfValue() { - return pattern.length(); - } - - @Override - protected boolean isNullByteForm(byte[] value, int offset, int len) { - return value == null || len == 0; - } - - @Override - final protected int getIdFromValueImpl(String value, int roundFlag) { - Date date = stringToDate(value, pattern); - int id = calcIdFromSeqNo(getNumOfDaysSince0000(date)); - if (id < 0 || id >= 16777216) - throw new IllegalArgumentException("'" + value + "' encodes to '" + id + "' which is out of range of 3 bytes"); - - return id; - } - - @Override - final protected String getValueFromIdImpl(int id) { - if (id < baseId) - throw new IllegalArgumentException("ID '" + id + "' must not be less than base ID " + baseId); - Date d = getDateFromNumOfDaysSince0000(calcSeqNoFromId(id)); - return dateToString(d, pattern); - } - - private int getNumOfDaysSince0000(Date d) { - // 86400000 = 1000 * 60 * 60 * 24 - // -719530 is offset of 0000-01-01 - return (int) (d.getTime() / 86400000 + 719530); - } - - private Date getDateFromNumOfDaysSince0000(int n) { - long millis = ((long) n - 719530) * 86400000; - return new Date(millis); - } - - @Override - final protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) { - try { - return getIdFromValue(new String(value, offset, len, "ISO-8859-1")); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); // never happen - } - } - - @Override - final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) { - String date = getValueFromId(id); - byte bytes[]; - try { - bytes = date.getBytes("ISO-8859-1"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); // never happen - } - System.arraycopy(bytes, 0, returnValue, offset, bytes.length); - return bytes.length; - } - - private int calcIdFromSeqNo(int seq) { - return seq < 0 ? seq : baseId + seq; - } - - private int calcSeqNoFromId(int id) { - return id - baseId; - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeUTF(pattern); - out.writeInt(baseId); - } - - @Override - public void readFields(DataInput in) throws IOException { - String pattern = in.readUTF(); - int baseId = in.readInt(); - init(pattern, baseId); - } - - @Override - public int hashCode() { - return 31 * baseId + pattern.hashCode(); - } - - @Override - public boolean equals(Object o) { - if ((o instanceof DateStrDictionary) == false) - return false; - DateStrDictionary that = (DateStrDictionary) o; - return StringUtils.equals(this.pattern, that.pattern) && this.baseId == that.baseId; - } - - @Override - public void dump(PrintStream out) { - out.println(this.toString()); - } - - @Override - public String toString() { - return "DateStrDictionary [pattern=" + pattern + ", baseId=" + baseId + "]"; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java ---------------------------------------------------------------------- diff --git a/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java b/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java deleted file mode 100644 index b54c6a3..0000000 --- a/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.kylinolap.dict; - -import java.io.PrintStream; -import java.io.UnsupportedEncodingException; - -import org.apache.hadoop.io.Writable; - -import com.kylinolap.common.util.BytesUtil; - -/** - * A bi-way dictionary that maps from dimension/column values to IDs and vice - * versa. By storing IDs instead of real values, the size of cube is - * significantly reduced. - * - * - IDs are smallest integers possible for the cardinality of a column, for the - * purpose of minimal storage space - IDs preserve ordering of values, such that - * range query can be applied to IDs directly - * - * A dictionary once built, is immutable. This allows optimal memory footprint - * by e.g. flatten the Trie structure into a byte array, replacing node pointers - * with array offsets. - * - * @author yangli9 - */ -abstract public class Dictionary implements Writable { - - public static final byte NULL = (byte) 0xff; - - // ID with all bit-1 (0xff e.g.) reserved for NULL value - public static final int NULL_ID[] = new int[] { 0, 0xff, 0xffff, 0xffffff, 0xffffff }; - - abstract public int getMinId(); - - abstract public int getMaxId(); - - /** - * @return the size of an ID in bytes, determined by the cardinality of - * column - */ - abstract public int getSizeOfId(); - - /** - * @return the (maximum) size of value in bytes, determined by the longest - * value of column - */ - abstract public int getSizeOfValue(); - - /** - * Convenient form of getIdFromValue(value, 0) - */ - final public int getIdFromValue(T value) { - return getIdFromValue(value, 0); - } - - /** - * Returns the ID integer of given value. In case of not found - if - * roundingFlag=0, throw IllegalArgumentException; - if roundingFlag<0, the - * closest smaller ID integer if exist; - if roundingFlag>0, the closest - * bigger ID integer if exist. The implementation often has cache, thus - * faster than the byte[] version getIdFromValueBytes() - * - * @throws IllegalArgumentException - * if value is not found in dictionary and rounding is off or - * failed - */ - final public int getIdFromValue(T value, int roundingFlag) { - if (isNullObjectForm(value)) - return nullId(); - else - return getIdFromValueImpl(value, roundingFlag); - } - - protected boolean isNullObjectForm(T value) { - return value == null; - } - - abstract protected int getIdFromValueImpl(T value, int roundingFlag); - - /** - * @return the value corresponds to the given ID - * @throws IllegalArgumentException - * if ID is not found in dictionary - */ - final public T getValueFromId(int id) { - if (isNullId(id)) - return null; - else - return getValueFromIdImpl(id); - } - - abstract protected T getValueFromIdImpl(int id); - - /** - * Convenient form of - * getIdFromValueBytes(value, offset, len, 0) - */ - final public int getIdFromValueBytes(byte[] value, int offset, int len) { - return getIdFromValueBytes(value, offset, len, 0); - } - - /** - * A lower level API, return ID integer from raw value bytes. In case of not - * found - if roundingFlag=0, throw IllegalArgumentException; - if - * roundingFlag<0, the closest smaller ID integer if exist; - if - * roundingFlag>0, the closest bigger ID integer if exist. Bypassing the - * cache layer, this could be significantly slower than getIdFromValue(T - * value). - * - * @throws IllegalArgumentException - * if value is not found in dictionary and rounding is off or - * failed - */ - final public int getIdFromValueBytes(byte[] value, int offset, int len, int roundingFlag) { - if (isNullByteForm(value, offset, len)) - return nullId(); - else - return getIdFromValueBytesImpl(value, offset, len, roundingFlag); - } - - protected boolean isNullByteForm(byte[] value, int offset, int len) { - return value == null; - } - - abstract protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag); - - /** - * A lower level API, get byte values from ID, return the number of bytes - * written. Bypassing the cache layer, this could be significantly slower - * than getIdFromValue(T value). - * - * @throws IllegalArgumentException - * if ID is not found in dictionary - */ - final public int getValueBytesFromId(int id, byte[] returnValue, int offset) { - if (isNullId(id)) - return 0; - else - return getValueBytesFromIdImpl(id, returnValue, offset); - } - - abstract protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset); - - abstract public void dump(PrintStream out); - - public int nullId() { - return NULL_ID[getSizeOfId()]; - } - - public boolean isNullId(int id) { - int nullId = NULL_ID[getSizeOfId()]; - return (nullId & id) == nullId; - } - - /** utility that converts a dictionary ID to string, preserving order */ - public static String dictIdToString(byte[] idBytes, int offset, int length) { - try { - return new String(idBytes, offset, length, "ISO-8859-1"); - } catch (UnsupportedEncodingException e) { - // never happen - return null; - } - } - - /** the reverse of dictIdToString(), returns integer ID */ - public static int stringToDictId(String str) { - try { - byte[] bytes = str.getBytes("ISO-8859-1"); - return BytesUtil.readUnsigned(bytes, 0, bytes.length); - } catch (UnsupportedEncodingException e) { - // never happen - return 0; - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java ---------------------------------------------------------------------- diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java deleted file mode 100644 index 16ae61c..0000000 --- a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kylinolap.dict; - -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; - -import org.apache.hadoop.hbase.util.Bytes; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; -import com.kylinolap.common.util.JsonUtil; -import com.kylinolap.dict.lookup.ReadableTable; -import com.kylinolap.dict.lookup.TableReader; -import com.kylinolap.metadata.model.schema.DataType; - -/** - * @author yangli9 - */ -@SuppressWarnings({ "rawtypes", "unchecked" }) -public class DictionaryGenerator { - - private static final Logger logger = LoggerFactory.getLogger(DictionaryGenerator.class); - - private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd" }; - - public static Dictionary buildDictionaryFromValueList(DictionaryInfo info, List values) { - info.setCardinality(values.size()); - - Dictionary dict = null; - int baseId = 0; // always 0 for now - int nSamples = 5; - ArrayList samples = new ArrayList(); - - // build dict, case by data type - DataType dataType = DataType.getInstance(info.getDataType()); - if (dataType.isDateTimeFamily()) - dict = buildDateStrDict(values, baseId, nSamples, samples); - else if (dataType.isNumberFamily()) - dict = buildNumberDict(values, baseId, nSamples, samples); - else - dict = buildStringDict(values, baseId, nSamples, samples); - - // log a few samples - StringBuilder buf = new StringBuilder(); - for (Object s : samples) { - if (buf.length() > 0) - buf.append(", "); - buf.append(s.toString()).append("=>").append(dict.getIdFromValue(s)); - } - logger.info("Dictionary value samples: " + buf.toString()); - logger.info("Dictionary cardinality " + info.getCardinality()); - - if (values.size() > 1000000) - throw new IllegalArgumentException("Too high cardinality is not suitable for dictionary! Are the values stable enough for incremental load??"); - - return dict; - } - - public static Dictionary mergeDictionaries(DictionaryInfo targetInfo, List sourceDicts) { - - HashSet dedup = new HashSet(); - - for (DictionaryInfo info : sourceDicts) { - Dictionary dict = info.getDictionaryObject(); - int minkey = dict.getMinId(); - int maxkey = dict.getMaxId(); - byte[] buffer = new byte[dict.getSizeOfValue()]; - for (int i = minkey; i <= maxkey; ++i) { - int size = dict.getValueBytesFromId(i, buffer, 0); - dedup.add(Bytes.copy(buffer, 0, size)); - } - } - - List valueList = new ArrayList(); - valueList.addAll(dedup); - - return buildDictionaryFromValueList(targetInfo, valueList); - } - - public static Dictionary buildDictionary(DictionaryInfo info, ReadableTable inpTable) throws IOException { - - // currently all data types are casted to string to build dictionary - // String dataType = info.getDataType(); - - logger.info("Building dictionary " + JsonUtil.writeValueAsString(info)); - - ArrayList values = loadColumnValues(inpTable, info.getSourceColumnIndex()); - - return buildDictionaryFromValueList(info, values); - } - - private static Dictionary buildDateStrDict(List values, int baseId, int nSamples, ArrayList samples) { - final int BAD_THRESHOLD = 2; - String matchPattern = null; - - for (String ptn : DATE_PATTERNS) { - matchPattern = ptn; // be optimistic - int badCount = 0; - SimpleDateFormat sdf = new SimpleDateFormat(ptn); - for (byte[] value : values) { - if (value.length == 0) - continue; - - String str = Bytes.toString(value); - try { - sdf.parse(str); - if (samples.size() < nSamples && samples.contains(str) == false) - samples.add(str); - } catch (ParseException e) { - logger.info("Unrecognized datetime value: " + str); - badCount++; - if (badCount > BAD_THRESHOLD) { - matchPattern = null; - break; - } - } - } - if (matchPattern != null) - return new DateStrDictionary(matchPattern, baseId); - } - throw new IllegalStateException("Unrecognized datetime value"); - } - - private static Dictionary buildStringDict(List values, int baseId, int nSamples, ArrayList samples) { - TrieDictionaryBuilder builder = new TrieDictionaryBuilder(new StringBytesConverter()); - for (byte[] value : values) { - String v = Bytes.toString(value); - builder.addValue(v); - if (samples.size() < nSamples && samples.contains(v) == false) - samples.add(v); - } - return builder.build(baseId); - } - - private static Dictionary buildNumberDict(List values, int baseId, int nSamples, ArrayList samples) { - NumberDictionaryBuilder builder = new NumberDictionaryBuilder(new StringBytesConverter()); - for (byte[] value : values) { - String v = Bytes.toString(value); - builder.addValue(v); - if (samples.size() < nSamples && samples.contains(v) == false) - samples.add(v); - } - return builder.build(baseId); - } - - static ArrayList loadColumnValues(ReadableTable inpTable, int colIndex) throws IOException { - - TableReader reader = inpTable.getReader(); - - try { - ArrayList result = Lists.newArrayList(); - HashSet dedup = new HashSet(); - - while (reader.next()) { - String[] split = reader.getRow(); - - String colValue; - // special single column file, e.g. common_indicator.txt - if (split.length == 1) { - colValue = split[0]; - } - // normal case - else { - if (split.length <= colIndex) { - throw new ArrayIndexOutOfBoundsException("Column no. " + colIndex + " not found, line split is " + Arrays.asList(split)); - } - colValue = split[colIndex]; - } - - if (dedup.contains(colValue) == false) { - dedup.add(colValue); - result.add(Bytes.toBytes(colValue)); - } - } - return result; - - } finally { - reader.close(); - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfo.java ---------------------------------------------------------------------- diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfo.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfo.java deleted file mode 100644 index 89e01b2..0000000 --- a/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfo.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.kylinolap.dict; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.kylinolap.common.persistence.ResourceStore; -import com.kylinolap.common.persistence.RootPersistentEntity; -import com.kylinolap.dict.lookup.TableSignature; - -@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class DictionaryInfo extends RootPersistentEntity { - - @JsonProperty("source_table") - private String sourceTable; - @JsonProperty("source_column") - private String sourceColumn; - @JsonProperty("source_column_index") - private int sourceColumnIndex; // 0 based - @JsonProperty("data_type") - private String dataType; - @JsonProperty("input") - private TableSignature input; - @JsonProperty("input_delimeter") - private String inputDelimeter; - @JsonProperty("dictionary_class") - private String dictionaryClass; - @JsonProperty("cardinality") - private int cardinality; - - transient Dictionary dictionaryObject; - - public DictionaryInfo() { - } - - public DictionaryInfo(String sourceTable, String sourceColumn, int sourceColumnIndex, String dataType, TableSignature input, String inputDelimeter) { - - this.updateRandomUuid(); - - this.sourceTable = sourceTable; - this.sourceColumn = sourceColumn; - this.sourceColumnIndex = sourceColumnIndex; - this.dataType = dataType; - this.input = input; - this.inputDelimeter = inputDelimeter; - } - - public DictionaryInfo(DictionaryInfo other) { - - this.updateRandomUuid(); - - this.sourceTable = other.sourceTable; - this.sourceColumn = other.sourceColumn; - this.sourceColumnIndex = other.sourceColumnIndex; - this.dataType = other.dataType; - this.input = other.input; - this.inputDelimeter = other.inputDelimeter; - } - - // ---------------------------------------------------------------------------- - - public String getResourcePath() { - return ResourceStore.DICT_RESOURCE_ROOT + "/" + sourceTable + "/" + sourceColumn + "/" + uuid + ".dict"; - } - - public String getResourceDir() { - return ResourceStore.DICT_RESOURCE_ROOT + "/" + sourceTable + "/" + sourceColumn; - } - - // ---------------------------------------------------------------------------- - - // to decide if two dictionaries are built on the same table/column, - // regardless of their signature - public boolean isDictOnSameColumn(DictionaryInfo other) { - return this.sourceTable.equalsIgnoreCase(other.sourceTable) && - this.sourceColumn.equalsIgnoreCase(other.sourceColumn) && - this.sourceColumnIndex == other.sourceColumnIndex && - this.dataType.equalsIgnoreCase(other.dataType) && - this.inputDelimeter.equalsIgnoreCase(other.inputDelimeter) && - this.dictionaryClass.equalsIgnoreCase(other.dictionaryClass); - } - - public String getSourceTable() { - return sourceTable; - } - - public void setSourceTable(String sourceTable) { - this.sourceTable = sourceTable; - } - - public String getSourceColumn() { - return sourceColumn; - } - - public void setSourceColumn(String sourceColumn) { - this.sourceColumn = sourceColumn; - } - - public int getSourceColumnIndex() { - return sourceColumnIndex; - } - - public void setSourceColumnIndex(int sourceColumnIndex) { - this.sourceColumnIndex = sourceColumnIndex; - } - - public String getDataType() { - return dataType; - } - - public void setDataType(String dataType) { - this.dataType = dataType; - } - - public TableSignature getInput() { - return input; - } - - public void setInput(TableSignature input) { - this.input = input; - } - - public String getInputDelimeter() { - return inputDelimeter; - } - - public void setInputDelimeter(String inputDelimeter) { - this.inputDelimeter = inputDelimeter; - } - - public String getDictionaryClass() { - return dictionaryClass; - } - - public void setDictionaryClass(String dictionaryClass) { - this.dictionaryClass = dictionaryClass; - } - - public Dictionary getDictionaryObject() { - return dictionaryObject; - } - - public void setDictionaryObject(Dictionary dictionaryObject) { - this.dictionaryObject = dictionaryObject; - } - - public int getCardinality() { - return cardinality; - } - - public void setCardinality(int cardinality) { - this.cardinality = cardinality; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfoSerializer.java ---------------------------------------------------------------------- diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfoSerializer.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfoSerializer.java deleted file mode 100644 index 623d25a..0000000 --- a/dictionary/src/main/java/com/kylinolap/dict/DictionaryInfoSerializer.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kylinolap.dict; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; - -import com.kylinolap.common.persistence.Serializer; -import com.kylinolap.common.util.JsonUtil; - -/** - * @author yangli9 - * - */ -public class DictionaryInfoSerializer implements Serializer { - - public static final DictionaryInfoSerializer FULL_SERIALIZER = new DictionaryInfoSerializer(false); - public static final DictionaryInfoSerializer INFO_SERIALIZER = new DictionaryInfoSerializer(true); - - private boolean infoOnly; - - public DictionaryInfoSerializer() { - this(false); - } - - public DictionaryInfoSerializer(boolean infoOnly) { - this.infoOnly = infoOnly; - } - - @Override - public void serialize(DictionaryInfo obj, DataOutputStream out) throws IOException { - String json = JsonUtil.writeValueAsIndentString(obj); - out.writeUTF(json); - - if (infoOnly == false) - obj.getDictionaryObject().write(out); - } - - @Override - public DictionaryInfo deserialize(DataInputStream in) throws IOException { - String json = in.readUTF(); - DictionaryInfo obj = JsonUtil.readValue(json, DictionaryInfo.class); - - if (infoOnly == false) { - Dictionary dict; - try { - dict = (Dictionary) Class.forName(obj.getDictionaryClass()).newInstance(); - } catch (InstantiationException e) { - throw new RuntimeException(e); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - dict.readFields(in); - obj.setDictionaryObject(dict); - } - return obj; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/dictionary/src/main/java/com/kylinolap/dict/DictionaryManager.java ---------------------------------------------------------------------- diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryManager.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryManager.java deleted file mode 100644 index 73bc9d9..0000000 --- a/dictionary/src/main/java/com/kylinolap/dict/DictionaryManager.java +++ /dev/null @@ -1,345 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.kylinolap.dict; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.commons.compress.utils.IOUtils; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.kylinolap.common.KylinConfig; -import com.kylinolap.common.persistence.ResourceStore; -import com.kylinolap.common.util.HadoopUtil; -import com.kylinolap.dict.lookup.FileTable; -import com.kylinolap.dict.lookup.HiveTable; -import com.kylinolap.dict.lookup.ReadableTable; -import com.kylinolap.dict.lookup.TableSignature; -import com.kylinolap.metadata.MetadataManager; -import com.kylinolap.metadata.model.cube.CubeDesc; -import com.kylinolap.metadata.model.cube.TblColRef; - -public class DictionaryManager { - - private static final Logger logger = LoggerFactory.getLogger(DictionaryManager.class); - - private static final DictionaryInfo NONE_INDICATOR = new DictionaryInfo(); - - // static cached instances - private static final ConcurrentHashMap SERVICE_CACHE = new ConcurrentHashMap(); - - public static DictionaryManager getInstance(KylinConfig config) { - DictionaryManager r = SERVICE_CACHE.get(config); - if (r == null) { - r = new DictionaryManager(config); - SERVICE_CACHE.put(config, r); - if (SERVICE_CACHE.size() > 1) { - logger.warn("More than one singleton exist"); - } - } - return r; - } - - public static void removeInstance(KylinConfig config) { - SERVICE_CACHE.remove(config); - } - - // ============================================================================ - - private KylinConfig config; - private ConcurrentHashMap dictCache; // resource - // path ==> - // DictionaryInfo - - private DictionaryManager(KylinConfig config) { - this.config = config; - dictCache = new ConcurrentHashMap(); - } - - public Dictionary getDictionary(String resourcePath) throws IOException { - DictionaryInfo dictInfo = getDictionaryInfo(resourcePath); - return dictInfo == null ? null : dictInfo.getDictionaryObject(); - } - - public DictionaryInfo getDictionaryInfo(String resourcePath) throws IOException { - DictionaryInfo dictInfo = dictCache.get(resourcePath); - if (dictInfo == null) { - dictInfo = load(resourcePath, true); - if (dictInfo == null) - dictInfo = NONE_INDICATOR; - dictCache.put(resourcePath, dictInfo); - } - return dictInfo == NONE_INDICATOR ? null : dictInfo; - } - - public DictionaryInfo trySaveNewDict(Dictionary newDict, DictionaryInfo newDictInfo) throws IOException { - - String dupDict = checkDupByContent(newDictInfo, newDict); - if (dupDict != null) { - logger.info("Identical dictionary content " + newDict + ", reuse existing dictionary at " + dupDict); - return getDictionaryInfo(dupDict); - } - - newDictInfo.setDictionaryObject(newDict); - newDictInfo.setDictionaryClass(newDict.getClass().getName()); - - save(newDictInfo); - dictCache.put(newDictInfo.getResourcePath(), newDictInfo); - - return newDictInfo; - } - - public DictionaryInfo mergeDictionary(List dicts) throws IOException { - DictionaryInfo firstDictInfo = null; - int totalSize = 0; - for (DictionaryInfo info : dicts) { - // check - if (firstDictInfo == null) { - firstDictInfo = info; - } else { - if (!firstDictInfo.isDictOnSameColumn(info)) { - throw new IllegalArgumentException("Merging dictionaries are not structurally equal(regardless of signature)."); - } - } - totalSize += info.getInput().getSize(); - } - - if (firstDictInfo == null) { - throw new IllegalArgumentException("DictionaryManager.mergeDictionary input cannot be null"); - } - - DictionaryInfo newDictInfo = new DictionaryInfo(firstDictInfo); - TableSignature signature = newDictInfo.getInput(); - signature.setSize(totalSize); - signature.setLastModifiedTime(System.currentTimeMillis()); - signature.setPath("merged_with_no_original_path"); - - String dupDict = checkDupByInfo(newDictInfo); - if (dupDict != null) { - logger.info("Identical dictionary input " + newDictInfo.getInput() + ", reuse existing dictionary at " + dupDict); - return getDictionaryInfo(dupDict); - } - - Dictionary newDict = DictionaryGenerator.mergeDictionaries(newDictInfo, dicts); - - return trySaveNewDict(newDict, newDictInfo); - } - - public DictionaryInfo buildDictionary(CubeDesc cube, TblColRef col, String factColumnsPath) throws IOException { - - Object[] tmp = decideSourceData(cube, col, factColumnsPath); - String srcTable = (String) tmp[0]; - String srcCol = (String) tmp[1]; - int srcColIdx = (Integer) tmp[2]; - ReadableTable inpTable = (ReadableTable) tmp[3]; - - DictionaryInfo dictInfo = new DictionaryInfo(srcTable, srcCol, srcColIdx, col.getDatatype(), inpTable.getSignature(), inpTable.getColumnDelimeter()); - - String dupDict = checkDupByInfo(dictInfo); - if (dupDict != null) { - logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupDict); - return getDictionaryInfo(dupDict); - } - - Dictionary dict = DictionaryGenerator.buildDictionary(dictInfo, inpTable); - - return trySaveNewDict(dict, dictInfo); - } - - /** - * Get column origin - * - * @return 1. source table name - * 2. column name - * 3. column cardinal in source table - * 4. ReadableTable object - */ - public Object[] decideSourceData(CubeDesc cube, TblColRef col, String factColumnsPath) throws IOException { - String srcTable; - String srcCol; - int srcColIdx; - ReadableTable table; - MetadataManager metaMgr = MetadataManager.getInstance(config); - - // case of full table (dict on fact table) - if (cube == null) { - srcTable = col.getTable(); - srcCol = col.getName(); - srcColIdx = col.getColumn().getZeroBasedIndex(); - int nColumns = metaMgr.getTableDesc(col.getTable()).getColumnCount(); - table = new FileTable(factColumnsPath + "/" + col.getName(), nColumns); - return new Object[] { srcTable, srcCol, srcColIdx, table }; - } - - // Decide source data of dictionary: - // 1. If 'useDict' specifies pre-defined data set, use that - // 2. Otherwise find a lookup table to scan through - - // Note FK on fact table is supported by scan the related PK on lookup - // table - - String useDict = cube.getRowkey().getDictionary(col); - - // normal case, source from lookup table - if ("true".equals(useDict) || "string".equals(useDict) || "number".equals(useDict) || "any".equals(useDict)) { - // FK on fact table, use PK from lookup instead - if (cube.isFactTable(col.getTable())) { - TblColRef pkCol = cube.findPKByFK(col); - if (pkCol != null) - col = pkCol; // scan the counterparty PK on lookup table - // instead - } - srcTable = col.getTable(); - srcCol = col.getName(); - srcColIdx = col.getColumn().getZeroBasedIndex(); - if (cube.isFactTable(col.getTable())) { - table = new FileTable(factColumnsPath + "/" + col.getName(), -1); - } else { - table = new HiveTable(metaMgr, col.getTable()); - } - } - // otherwise could refer to a data set, e.g. common_indicators.txt - // (LEGACY PATH, since distinct values are collected from fact table) - else { - String dictDataSetPath = unpackDataSet(this.config.getTempHDFSDir(), useDict); - if (dictDataSetPath == null) - throw new IllegalArgumentException("Unknown dictionary data set '" + useDict + "', referred from " + col); - srcTable = "PREDEFINED"; - srcCol = useDict; - srcColIdx = 0; - table = new FileTable(dictDataSetPath, -1); - } - - return new Object[] { srcTable, srcCol, srcColIdx, table }; - } - - private String unpackDataSet(String tempHDFSDir, String dataSetName) throws IOException { - - InputStream in = this.getClass().getResourceAsStream("/com/kylinolap/dict/" + dataSetName + ".txt"); - if (in == null) // data set resource not found - return null; - - ByteArrayOutputStream buf = new ByteArrayOutputStream(); - IOUtils.copy(in, buf); - in.close(); - byte[] bytes = buf.toByteArray(); - - Path tmpDataSetPath = new Path(tempHDFSDir + "/dict/temp_dataset/" + dataSetName + "_" + bytes.length + ".txt"); - - FileSystem fs = HadoopUtil.getFileSystem(tempHDFSDir); - boolean writtenNewFile = false; - if (fs.exists(tmpDataSetPath) == false || fs.getFileStatus(tmpDataSetPath).getLen() != bytes.length) { - fs.mkdirs(tmpDataSetPath.getParent()); - FSDataOutputStream out = fs.create(tmpDataSetPath); - IOUtils.copy(new ByteArrayInputStream(bytes), out); - out.close(); - writtenNewFile = true; - } - - String qualifiedPath = tmpDataSetPath.makeQualified(fs.getUri(), new Path("/")).toString(); - if (writtenNewFile) - logger.info("Dictionary temp data set file written to " + qualifiedPath); - return qualifiedPath; - } - - private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException { - ResourceStore store = MetadataManager.getInstance(config).getStore(); - ArrayList existings = store.listResources(dictInfo.getResourceDir()); - if (existings == null) - return null; - - TableSignature input = dictInfo.getInput(); - for (String existing : existings) { - DictionaryInfo existingInfo = load(existing, false); // skip cache, - // direct - // load from - // store - if (input.equals(existingInfo.getInput())) - return existing; - } - - return null; - } - - private String checkDupByContent(DictionaryInfo dictInfo, Dictionary dict) throws IOException { - ResourceStore store = MetadataManager.getInstance(config).getStore(); - ArrayList existings = store.listResources(dictInfo.getResourceDir()); - if (existings == null) - return null; - - for (String existing : existings) { - logger.info("Checking dup dict :" + existing); - DictionaryInfo existingInfo = load(existing, true); // skip cache, - // direct load - // from store - if(existingInfo == null) - logger.info("existingInfo is null"); - - if (existingInfo != null && dict.equals(existingInfo.getDictionaryObject())) - return existing; - } - - return null; - } - - public void removeDictionary(String resourcePath) throws IOException { - ResourceStore store = MetadataManager.getInstance(config).getStore(); - store.deleteResource(resourcePath); - dictCache.remove(resourcePath); - } - - public void removeDictionaries(String srcTable, String srcCol) throws IOException { - DictionaryInfo info = new DictionaryInfo(); - info.setSourceTable(srcTable); - info.setSourceColumn(srcCol); - - ResourceStore store = MetadataManager.getInstance(config).getStore(); - ArrayList existings = store.listResources(info.getResourceDir()); - if (existings == null) - return; - - for (String existing : existings) - removeDictionary(existing); - } - - void save(DictionaryInfo dict) throws IOException { - ResourceStore store = MetadataManager.getInstance(config).getStore(); - String path = dict.getResourcePath(); - logger.info("Saving dictionary at " + path); - store.putResource(path, dict, DictionaryInfoSerializer.FULL_SERIALIZER); - } - - DictionaryInfo load(String resourcePath, boolean loadDictObj) throws IOException { - ResourceStore store = MetadataManager.getInstance(config).getStore(); - - DictionaryInfo info = store.getResource(resourcePath, DictionaryInfo.class, loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : DictionaryInfoSerializer.INFO_SERIALIZER); - - if (loadDictObj) - logger.debug("Loaded dictionary at " + resourcePath); - - return info; - } - -}