http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/.settings/org.eclipse.jdt.core.prefs ---------------------------------------------------------------------- diff --git a/invertedindex/.settings/org.eclipse.jdt.core.prefs b/invertedindex/.settings/org.eclipse.jdt.core.prefs deleted file mode 100644 index 5aaaf1e..0000000 --- a/invertedindex/.settings/org.eclipse.jdt.core.prefs +++ /dev/null @@ -1,386 +0,0 @@ -eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled -org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore -org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull -org.eclipse.jdt.core.compiler.annotation.nonnull.secondary= -org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault -org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary= -org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable -org.eclipse.jdt.core.compiler.annotation.nullable.secondary= -org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled -org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 -org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.7 -org.eclipse.jdt.core.compiler.debug.lineNumber=generate -org.eclipse.jdt.core.compiler.debug.localVariable=generate -org.eclipse.jdt.core.compiler.debug.sourceFile=generate -org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning -org.eclipse.jdt.core.compiler.problem.assertIdentifier=error -org.eclipse.jdt.core.compiler.problem.autoboxing=ignore -org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning -org.eclipse.jdt.core.compiler.problem.deadCode=warning -org.eclipse.jdt.core.compiler.problem.deprecation=warning -org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled -org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled -org.eclipse.jdt.core.compiler.problem.discouragedReference=ignore -org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore -org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore -org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore -org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled -org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore -org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning -org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning -org.eclipse.jdt.core.compiler.problem.forbiddenReference=ignore -org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning -org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled -org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning -org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning -org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore -org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore -org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning -org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore -org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore -org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled -org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore -org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore -org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled -org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning -org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore -org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning -org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning -org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore -org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning -org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning -org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error -org.eclipse.jdt.core.compiler.problem.nullReference=warning -org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error -org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning -org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning -org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore -org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning -org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore -org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore -org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore -org.eclipse.jdt.core.compiler.problem.rawTypeReference=ignore -org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning -org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore -org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore -org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore -org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore -org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore -org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled -org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning -org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled -org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled -org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled -org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore -org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning -org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled -org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=ignore -org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning -org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore -org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning -org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore -org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore -org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled -org.eclipse.jdt.core.compiler.problem.unusedExceptionParameter=ignore -org.eclipse.jdt.core.compiler.problem.unusedImport=warning -org.eclipse.jdt.core.compiler.problem.unusedLabel=warning -org.eclipse.jdt.core.compiler.problem.unusedLocal=warning -org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore -org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore -org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled -org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled -org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled -org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning -org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore -org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning -org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning -org.eclipse.jdt.core.compiler.source=1.7 -org.eclipse.jdt.core.formatter.align_type_members_on_columns=false -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16 -org.eclipse.jdt.core.formatter.alignment_for_assignment=0 -org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16 -org.eclipse.jdt.core.formatter.alignment_for_compact_if=16 -org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80 -org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0 -org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16 -org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0 -org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16 -org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80 -org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16 -org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16 -org.eclipse.jdt.core.formatter.blank_lines_after_imports=1 -org.eclipse.jdt.core.formatter.blank_lines_after_package=1 -org.eclipse.jdt.core.formatter.blank_lines_before_field=0 -org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0 -org.eclipse.jdt.core.formatter.blank_lines_before_imports=1 -org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1 -org.eclipse.jdt.core.formatter.blank_lines_before_method=1 -org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1 -org.eclipse.jdt.core.formatter.blank_lines_before_package=0 -org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1 -org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1 -org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line -org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false -org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false -org.eclipse.jdt.core.formatter.comment.format_block_comments=false -org.eclipse.jdt.core.formatter.comment.format_header=false -org.eclipse.jdt.core.formatter.comment.format_html=true -org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=false -org.eclipse.jdt.core.formatter.comment.format_line_comments=false -org.eclipse.jdt.core.formatter.comment.format_source_code=true -org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true -org.eclipse.jdt.core.formatter.comment.indent_root_tags=true -org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert -org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert -org.eclipse.jdt.core.formatter.comment.line_length=80 -org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true -org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true -org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false -org.eclipse.jdt.core.formatter.compact_else_if=true -org.eclipse.jdt.core.formatter.continuation_indentation=2 -org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2 -org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off -org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on -org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false -org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=true -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true -org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true -org.eclipse.jdt.core.formatter.indent_empty_lines=false -org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true -org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true -org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true -org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false -org.eclipse.jdt.core.formatter.indentation.size=4 -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert -org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert -org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert -org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert -org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert -org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert -org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert -org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert -org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert -org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert -org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert -org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert -org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert -org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.join_lines_in_comments=true -org.eclipse.jdt.core.formatter.join_wrapped_lines=true -org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false -org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false -org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false -org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false -org.eclipse.jdt.core.formatter.lineSplit=999 -org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false -org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false -org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0 -org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1 -org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true -org.eclipse.jdt.core.formatter.tabulation.char=space -org.eclipse.jdt.core.formatter.tabulation.size=4 -org.eclipse.jdt.core.formatter.use_on_off_tags=false -org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false -org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true -org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true -org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/.settings/org.eclipse.jdt.ui.prefs ---------------------------------------------------------------------- diff --git a/invertedindex/.settings/org.eclipse.jdt.ui.prefs b/invertedindex/.settings/org.eclipse.jdt.ui.prefs deleted file mode 100644 index d521bab..0000000 --- a/invertedindex/.settings/org.eclipse.jdt.ui.prefs +++ /dev/null @@ -1,7 +0,0 @@ -eclipse.preferences.version=1 -formatter_profile=_Space Indent & Long Lines -formatter_settings_version=12 -org.eclipse.jdt.ui.ignorelowercasenames=true -org.eclipse.jdt.ui.importorder=java;javax;org;com; -org.eclipse.jdt.ui.ondemandthreshold=99 -org.eclipse.jdt.ui.staticondemandthreshold=99 http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/pom.xml ---------------------------------------------------------------------- diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml deleted file mode 100644 index 11fa401..0000000 --- a/invertedindex/pom.xml +++ /dev/null @@ -1,103 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <parent> - <artifactId>kylin</artifactId> - <groupId>org.apache.kylin</groupId> - <version>1.5.3-SNAPSHOT</version> - - </parent> - <modelVersion>4.0.0</modelVersion> - <artifactId>kylin-invertedindex</artifactId> - <packaging>jar</packaging> - <name>Kylin:InvertedIndex</name> - - <dependencies> - - <!--Kylin Jar --> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-common</artifactId> - <version>${project.parent.version}</version> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-metadata</artifactId> - <version>${project.parent.version}</version> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-dictionary</artifactId> - <version>${project.parent.version}</version> - </dependency> - - <dependency> - <groupId>com.n3twork.druid</groupId> - <artifactId>extendedset</artifactId> - </dependency> - <dependency> - <groupId>com.ning</groupId> - <artifactId>compress-lzf</artifactId> - </dependency> - - <!-- Env & Test --> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-common</artifactId> - <type>test-jar</type> - <scope>test</scope> - <version>${project.parent.version}</version> - </dependency> - - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-jobclient</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-hadoop2-compat</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-client</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-server</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.mrunit</groupId> - <artifactId>mrunit</artifactId> - <classifier>hadoop2</classifier> - <scope>test</scope> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - -</project> http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/IICapabilityChecker.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IICapabilityChecker.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IICapabilityChecker.java deleted file mode 100644 index 4ee8f50..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IICapabilityChecker.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.metadata.model.FunctionDesc; -import org.apache.kylin.metadata.model.JoinDesc; -import org.apache.kylin.metadata.model.LookupDesc; -import org.apache.kylin.metadata.model.TableDesc; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.realization.CapabilityResult; -import org.apache.kylin.metadata.realization.SQLDigest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Sets; - -/** - */ -public class IICapabilityChecker { - private static final Logger logger = LoggerFactory.getLogger(IICapabilityChecker.class); - - public static CapabilityResult check(IIInstance ii, SQLDigest digest) { - CapabilityResult result = new CapabilityResult(); - result.capable = false; - - // match fact table - if (!digest.factTable.equalsIgnoreCase(ii.getFactTable())) { - logger.info("Exclude II " + ii.getName() + " because fact table unmatch"); - return result; - } - - // match joins - boolean matchJoin = isMatchedWithJoins(digest.joinDescs, ii); - if (!matchJoin) { - logger.info("Exclude II " + ii.getName() + " because unmatched joins"); - return result; - } - - // dimensions & measures - Collection<TblColRef> dimensionColumns = getDimensionColumns(digest); - Collection<FunctionDesc> aggrFunctions = digest.aggregations; - Collection<TblColRef> unmatchedDimensions = unmatchedDimensions(dimensionColumns, ii); - Collection<FunctionDesc> unmatchedAggregations = unmatchedAggregations(aggrFunctions, ii); - - // try dimension-as-measure - if (!unmatchedAggregations.isEmpty()) { - tryDimensionAsMeasures(unmatchedAggregations, digest, ii, result); - } - - if (!unmatchedDimensions.isEmpty()) { - logger.info("Exclude ii " + ii.getName() + " because unmatched dimensions"); - return result; - } - - if (!unmatchedAggregations.isEmpty()) { - logger.info("Exclude ii " + ii.getName() + " because unmatched aggregations"); - return result; - } - - // cost will be minded by caller - result.capable = true; - return result; - } - - private static boolean isMatchedWithJoins(Collection<JoinDesc> joins, IIInstance iiInstance) { - IIDesc iiDesc = iiInstance.getDescriptor(); - List<TableDesc> tables = iiDesc.listTables(); - - List<JoinDesc> cubeJoins = new ArrayList<JoinDesc>(tables.size()); - for (TableDesc tableDesc : tables) { - JoinDesc join = null; - for (LookupDesc lookup : iiDesc.getModel().getLookups()) { - if (lookup.getTable().equalsIgnoreCase(tableDesc.getIdentity())) { - join = lookup.getJoin(); - cubeJoins.add(join); - break; - } - } - } - - for (JoinDesc j : joins) { - // optiq engine can't decide which one is fk or pk - String pTable = j.getPrimaryKeyColumns()[0].getTable(); - String factTable = iiDesc.getModel().getFactTable(); - if (factTable.equals(pTable)) { - j.swapPKFK(); - } - - // check primary key, all PK column should refer to same tale, the Fact Table of iiInstance. - // Using first column's table name to check. - String fTable = j.getForeignKeyColumns()[0].getTable(); - if (!factTable.equals(fTable)) { - logger.info("Fact Table" + factTable + " not matched in join: " + j + " on ii " + iiInstance.getName()); - return false; - } - - // The hashcode() function of JoinDesc has been overwritten, - // which takes into consideration: pk,fk,jointype - if (!cubeJoins.contains(j)) { - logger.info("Query joins don't match on ii " + iiInstance.getName()); - return false; - } - } - return true; - } - - private static Collection<TblColRef> getDimensionColumns(SQLDigest sqlDigest) { - Collection<TblColRef> groupByColumns = sqlDigest.groupbyColumns; - Collection<TblColRef> filterColumns = sqlDigest.filterColumns; - - Collection<TblColRef> dimensionColumns = new HashSet<TblColRef>(); - dimensionColumns.addAll(groupByColumns); - dimensionColumns.addAll(filterColumns); - return dimensionColumns; - } - - private static Set<TblColRef> unmatchedDimensions(Collection<TblColRef> dimensionColumns, IIInstance ii) { - HashSet<TblColRef> result = Sets.newHashSet(dimensionColumns); - result.removeAll(ii.getDescriptor().listAllDimensions()); - return result; - } - - private static Set<FunctionDesc> unmatchedAggregations(Collection<FunctionDesc> aggregations, IIInstance ii) { - HashSet<FunctionDesc> result = Sets.newHashSet(aggregations); - result.removeAll(ii.getDescriptor().listAllFunctions()); - return result; - } - - private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, IIInstance ii, CapabilityResult result) { - IIDesc iiDesc = ii.getDescriptor(); - Collection<FunctionDesc> iiFuncs = iiDesc.listAllFunctions(); - - Iterator<FunctionDesc> it = unmatchedAggregations.iterator(); - while (it.hasNext()) { - FunctionDesc functionDesc = it.next(); - - if (iiFuncs.contains(functionDesc)) { - it.remove(); - continue; - } - - // let calcite handle count - if (functionDesc.isCount()) { - it.remove(); - continue; - } - - // calcite can do aggregation from columns on-the-fly - List<TblColRef> neededCols = functionDesc.getParameter().getColRefs(); - if (neededCols.size() > 0 && iiDesc.listAllDimensions().containsAll(neededCols)) { - result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc)); - it.remove(); - continue; - } - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java deleted file mode 100644 index 917fe46..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.commons.lang3.StringUtils; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.JsonSerializer; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.Serializer; -import org.apache.kylin.common.restclient.Broadcaster; -import org.apache.kylin.common.restclient.CaseInsensitiveStringCache; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.metadata.MetadataConstants; -import org.apache.kylin.metadata.MetadataManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * copied from CubeDescManager - * - * @author honma - */ -public class IIDescManager { - - private static final Logger logger = LoggerFactory.getLogger(IIDescManager.class); - - private static final Serializer<IIDesc> II_DESC_SERIALIZER = new JsonSerializer<IIDesc>(IIDesc.class); - - // static cached instances - private static final ConcurrentHashMap<KylinConfig, IIDescManager> CACHE = new ConcurrentHashMap<KylinConfig, IIDescManager>(); - - // ============================================================================ - - private KylinConfig config; - // name ==> IIDesc - private CaseInsensitiveStringCache<IIDesc> iiDescMap; - - public static IIDescManager getInstance(KylinConfig config) { - IIDescManager r = CACHE.get(config); - if (r != null) { - return r; - } - - synchronized (IIDescManager.class) { - r = CACHE.get(config); - if (r != null) { - return r; - } - try { - r = new IIDescManager(config); - CACHE.put(config, r); - if (CACHE.size() > 1) { - logger.warn("More than one singleton exist"); - } - return r; - } catch (IOException e) { - throw new IllegalStateException("Failed to init IIDescManager from " + config, e); - } - } - } - - public static void clearCache() { - CACHE.clear(); - } - - private IIDescManager(KylinConfig config) throws IOException { - logger.info("Initializing IIDescManager with config " + config); - this.config = config; - this.iiDescMap = new CaseInsensitiveStringCache<IIDesc>(config, Broadcaster.TYPE.INVERTED_INDEX_DESC); - reloadAllIIDesc(); - } - - public List<IIDesc> listAllDesc() { - return new ArrayList<IIDesc>(iiDescMap.values()); - } - - public IIDesc getIIDesc(String name) { - return iiDescMap.get(name); - } - - /** - * Reload IIDesc from resource store It will be triggered by an desc update - * event. - * - * @param name - * @throws IOException - */ - public IIDesc reloadIIDescLocal(String name) throws IOException { - - // Save Source - String path = IIDesc.getIIDescResourcePath(name); - - // Reload the IIDesc - IIDesc ndesc = loadIIDesc(path); - - // Here replace the old one - iiDescMap.putLocal(ndesc.getName(), ndesc); - return ndesc; - } - - private IIDesc loadIIDesc(String path) throws IOException { - ResourceStore store = getStore(); - logger.info("Loading IIDesc " + store.getReadableResourcePath(path)); - - IIDesc ndesc = store.getResource(path, IIDesc.class, II_DESC_SERIALIZER); - - if (StringUtils.isBlank(ndesc.getName())) { - throw new IllegalStateException("IIDesc name must not be blank"); - } - - ndesc.init(getMetadataManager()); - - return ndesc; - } - - /** - * Create a new IIDesc - * - * @param iiDesc - * @return - * @throws IOException - */ - public IIDesc createIIDesc(IIDesc iiDesc) throws IOException { - if (iiDesc.getUuid() == null || iiDesc.getName() == null) - throw new IllegalArgumentException(); - - if (iiDescMap.containsKey(iiDesc.getName())) - throw new IllegalArgumentException("IIDesc '" + iiDesc.getName() + "' already exists"); - - iiDesc.init(getMetadataManager()); - - // Check base validation - // Semantic validation - // TODO - - iiDesc.setSignature(iiDesc.calculateSignature()); - - String path = iiDesc.getResourcePath(); - getStore().putResource(path, iiDesc, II_DESC_SERIALIZER); - iiDescMap.put(iiDesc.getName(), iiDesc); - - return iiDesc; - } - - // remove iiDesc - public void removeIIDesc(IIDesc iiDesc) throws IOException { - String path = iiDesc.getResourcePath(); - getStore().deleteResource(path); - iiDescMap.remove(iiDesc.getName()); - } - - public void removeIIDescLocal(String name) throws IOException { - iiDescMap.removeLocal(name); - } - - private void reloadAllIIDesc() throws IOException { - ResourceStore store = getStore(); - logger.info("Reloading all II desc from folder " + store.getReadableResourcePath(ResourceStore.II_DESC_RESOURCE_ROOT)); - - iiDescMap.clear(); - - List<String> paths = store.collectResourceRecursively(ResourceStore.II_DESC_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX); - for (String path : paths) { - logger.info("loading II Desc from path" + path); - IIDesc desc; - try { - desc = loadIIDesc(path); - } catch (Exception e) { - logger.error("Error loading II desc " + path, e); - continue; - } - if (!path.equals(desc.getResourcePath())) { - logger.error("Skip suspicious desc at " + path + ", " + desc + " should be at " + desc.getResourcePath()); - continue; - } - if (iiDescMap.containsKey(desc.getName())) { - logger.error("Dup IIDesc name '" + desc.getName() + "' on path " + path); - continue; - } - - iiDescMap.putLocal(desc.getName(), desc); - } - - logger.debug("Loaded " + iiDescMap.size() + " II desc(s)"); - } - - /** - * Update IIDesc with the input. Broadcast the event into cluster - * - * @param desc - * @return - * @throws IOException - */ - public IIDesc updateIIDesc(IIDesc desc) throws IOException { - // Validate IIDesc - if (desc.getUuid() == null || desc.getName() == null) { - throw new IllegalArgumentException(); - } - String name = desc.getName(); - if (!iiDescMap.containsKey(name)) { - throw new IllegalArgumentException("IIDesc '" + name + "' does not exist."); - } - - desc.init(getMetadataManager()); - - // TODO: Semantic validation - - desc.setSignature(desc.calculateSignature()); - - // Save Source - String path = desc.getResourcePath(); - getStore().putResource(path, desc, II_DESC_SERIALIZER); - - // Reload the IIDesc - IIDesc ndesc = loadIIDesc(path); - // Here replace the old one - iiDescMap.put(ndesc.getName(), desc); - - return ndesc; - } - - private MetadataManager getMetadataManager() { - return MetadataManager.getInstance(config); - } - - private ResourceStore getStore() { - return ResourceStore.getStore(this.config); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java deleted file mode 100644 index 9b56c88..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java +++ /dev/null @@ -1,369 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.RootPersistentEntity; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.metadata.model.DataModelDesc; -import org.apache.kylin.metadata.model.IBuildable; -import org.apache.kylin.metadata.model.IStorageAware; -import org.apache.kylin.metadata.model.LookupDesc; -import org.apache.kylin.metadata.model.MeasureDesc; -import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.realization.CapabilityResult; -import org.apache.kylin.metadata.realization.IRealization; -import org.apache.kylin.metadata.realization.RealizationStatusEnum; -import org.apache.kylin.metadata.realization.RealizationType; -import org.apache.kylin.metadata.realization.SQLDigest; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonManagedReference; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * @author honma - */ -@SuppressWarnings("serial") -@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class IIInstance extends RootPersistentEntity implements IRealization, IBuildable { - - public static IIInstance create(String iiName, String projectName, IIDesc iiDesc) { - IIInstance iii = new IIInstance(); - - iii.setConfig(iiDesc.getConfig()); - iii.setName(iiName); - iii.setDescName(iiDesc.getName()); - iii.setCreateTimeUTC(System.currentTimeMillis()); - iii.setStatus(RealizationStatusEnum.DISABLED); - iii.updateRandomUuid(); - - return iii; - } - - @JsonIgnore - private KylinConfig config; - @JsonProperty("name") - private String name; - @JsonProperty("owner") - private String owner; - @JsonProperty("descriptor") - private String descName; - // Mark cube priority for query - @JsonProperty("cost") - private int cost = 50; - @JsonProperty("status") - private RealizationStatusEnum status; - - @JsonManagedReference - @JsonProperty("segments") - private List<IISegment> segments = new ArrayList<IISegment>(); - - @JsonProperty("create_time_utc") - private long createTimeUTC; - - private static final int COST_WEIGHT_LOOKUP_TABLE = 1; - private static final int COST_WEIGHT_INNER_JOIN = 2; - - public long getAllocatedEndDate() { - if (null == segments || segments.size() == 0) { - return 0; - } - - Collections.sort(segments); - - return segments.get(segments.size() - 1).getDateRangeEnd(); - } - - public long getAllocatedStartDate() { - if (null == segments || segments.size() == 0) { - return 0; - } - - Collections.sort(segments); - - return segments.get(0).getDateRangeStart(); - } - - public IIDesc getDescriptor() { - return IIDescManager.getInstance(config).getIIDesc(descName); - } - - @Override - public DataModelDesc getDataModelDesc() { - return this.getDescriptor().getModel(); - } - - public boolean isReady() { - return getStatus() == RealizationStatusEnum.READY; - } - - public String getResourcePath() { - return concatResourcePath(name); - } - - public static String concatResourcePath(String cubeName) { - return ResourceStore.II_RESOURCE_ROOT + "/" + cubeName + ".json"; - } - - @Override - public String toString() { - return getCanonicalName(); - } - - // ============================================================================ - - @JsonProperty("size_kb") - public long getSizeKB() { - long sizeKb = 0L; - - for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { - sizeKb += cubeSegment.getSizeKB(); - } - - return sizeKb; - } - - @JsonProperty("input_records_count") - public long getInputRecordCount() { - long sizeRecordCount = 0L; - - for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { - sizeRecordCount += cubeSegment.getInputRecords(); - } - - return sizeRecordCount; - } - - @JsonProperty("input_records_size") - public long getInputRecordSize() { - long sizeRecordSize = 0L; - - for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { - sizeRecordSize += cubeSegment.getInputRecordsSize(); - } - - return sizeRecordSize; - } - - public KylinConfig getConfig() { - return config; - } - - public void setConfig(KylinConfig config) { - this.config = config; - } - - @Override - public String getCanonicalName() { - return getType() + "[name=" + name + "]"; - } - - @Override - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getOwner() { - return owner; - } - - public void setOwner(String owner) { - this.owner = owner; - } - - public String getDescName() { - return descName; - } - - public void setDescName(String descName) { - this.descName = descName; - } - - public int getCost() { - return cost; - } - - public RealizationStatusEnum getStatus() { - return status; - } - - public void setStatus(RealizationStatusEnum status) { - this.status = status; - } - - public IISegment getFirstSegment() { - if (this.segments == null || this.segments.size() == 0) { - return null; - } else { - return this.segments.get(0); - } - } - - public List<IISegment> getSegments() { - return segments; - } - - public List<IISegment> getSegments(SegmentStatusEnum status) { - List<IISegment> result = new ArrayList<IISegment>(); - - for (IISegment segment : segments) { - if (segment.getStatus() == status) { - result.add(segment); - } - } - - return result; - } - - public IISegment getSegment(String name, SegmentStatusEnum status) { - for (IISegment segment : segments) { - if ((null != segment.getName() && segment.getName().equals(name)) && segment.getStatus() == status) { - return segment; - } - } - - return null; - } - - public void setSegments(List<IISegment> segments) { - this.segments = segments; - } - - public long getCreateTimeUTC() { - return createTimeUTC; - } - - public void setCreateTimeUTC(long createTimeUTC) { - this.createTimeUTC = createTimeUTC; - } - - @Override - public CapabilityResult isCapable(SQLDigest digest) { - CapabilityResult result = IICapabilityChecker.check(this, digest); - if (result.capable) { - result.cost = getCost(digest); - } else { - result.cost = -1; - } - return result; - } - - private int getCost(SQLDigest digest) { - - int calculatedCost = cost; - for (LookupDesc lookupDesc : this.getDescriptor().getModel().getLookups()) { - // more tables, more cost - calculatedCost += COST_WEIGHT_LOOKUP_TABLE; - if ("inner".equals(lookupDesc.getJoin().getType())) { - // inner join cost is bigger than left join, as it will filter some records - calculatedCost += COST_WEIGHT_INNER_JOIN; - } - } - return calculatedCost; - } - - @Override - public RealizationType getType() { - return RealizationType.INVERTED_INDEX; - } - - @Override - public List<TblColRef> getAllColumns() { - return getDescriptor().listAllColumns(); - } - - @Override - public String getFactTable() { - return getDescriptor().getFactTableName(); - } - - @Override - public List<MeasureDesc> getMeasures() { - return getDescriptor().getMeasures(); - } - - @Override - public long getDateRangeStart() { - List<IISegment> readySegs = getSegments(SegmentStatusEnum.READY); - - long startTime = Long.MAX_VALUE; - for (IISegment seg : readySegs) { - if (seg.getDateRangeStart() < startTime) - startTime = seg.getDateRangeStart(); - } - - return startTime; - } - - @Override - public long getDateRangeEnd() { - - List<IISegment> readySegs = getSegments(SegmentStatusEnum.READY); - - long endTime = 0; - for (IISegment seg : readySegs) { - if (seg.getDateRangeEnd() > endTime) - endTime = seg.getDateRangeEnd(); - } - - return endTime; - } - - @Override - public String getModelName() { - return this.getDescriptor().getModelName(); - } - - @Override - public List<TblColRef> getAllDimensions() { - return getDescriptor().listAllDimensions(); - } - - public void setCost(int cost) { - this.cost = cost; - } - - @Override - public int getStorageType() { - return IStorageAware.ID_HBASE; - } - - @Override - public int getEngineType() { - return getDescriptor().getEngineType(); - } - - @Override - public int getSourceType() { - return getDataModelDesc().getFactTableDesc().getSourceType(); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java deleted file mode 100644 index e869328..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.commons.lang3.StringUtils; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.JsonSerializer; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.Serializer; -import org.apache.kylin.common.restclient.Broadcaster; -import org.apache.kylin.common.restclient.CaseInsensitiveStringCache; -import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.apache.kylin.metadata.project.ProjectInstance; -import org.apache.kylin.metadata.project.ProjectManager; -import org.apache.kylin.metadata.realization.IRealization; -import org.apache.kylin.metadata.realization.IRealizationConstants; -import org.apache.kylin.metadata.realization.IRealizationProvider; -import org.apache.kylin.metadata.realization.RealizationType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; - -/** - * @author honma - */ -public class IIManager implements IRealizationProvider { - - private static String ALPHA_NUM = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - private static int HBASE_TABLE_LENGTH = 10; - - private static final Serializer<IIInstance> II_SERIALIZER = new JsonSerializer<IIInstance>(IIInstance.class); - - private static final Logger logger = LoggerFactory.getLogger(IIManager.class); - - // static cached instances - private static final ConcurrentHashMap<KylinConfig, IIManager> CACHE = new ConcurrentHashMap<KylinConfig, IIManager>(); - - public static IIManager getInstance(KylinConfig config) { - IIManager r = CACHE.get(config); - if (r != null) { - return r; - } - - synchronized (IIManager.class) { - r = CACHE.get(config); - if (r != null) { - return r; - } - try { - r = new IIManager(config); - CACHE.put(config, r); - if (CACHE.size() > 1) { - logger.warn("More than one singleton exist"); - } - return r; - } catch (IOException e) { - throw new IllegalStateException("Failed to init IIManager from " + config, e); - } - } - } - - public static void clearCache() { - CACHE.clear(); - } - - // ============================================================================ - - private KylinConfig config; - // ii name ==> IIInstance - private CaseInsensitiveStringCache<IIInstance> iiMap; - - // for generation hbase table name of a new segment - private Multimap<String, String> usedStorageLocation = HashMultimap.create(); - - private IIManager(KylinConfig config) throws IOException { - logger.info("Initializing IIManager with config " + config); - this.config = config; - this.iiMap = new CaseInsensitiveStringCache<IIInstance>(config, Broadcaster.TYPE.INVERTED_INDEX); - loadAllIIInstance(); - } - - public List<IIInstance> listAllIIs() { - return new ArrayList<IIInstance>(iiMap.values()); - } - - public IIInstance getII(String iiName) { - return iiMap.get(iiName); - } - - public List<IIInstance> getIIsByDesc(String descName) { - - List<IIInstance> list = listAllIIs(); - List<IIInstance> result = new ArrayList<IIInstance>(); - Iterator<IIInstance> it = list.iterator(); - while (it.hasNext()) { - IIInstance ci = it.next(); - if (descName.equalsIgnoreCase(ci.getDescName())) { - result.add(ci); - } - } - return result; - } - - public IIInstance createII(IIInstance ii) throws IOException { - - if (this.getII(ii.getName()) != null) - throw new IllegalArgumentException("The II name '" + ii.getName() + "' already exists."); - - this.updateII(ii); - - // FIXME need to pass in project name - String projectName = ProjectInstance.DEFAULT_PROJECT_NAME; - ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.INVERTED_INDEX, ii.getName(), projectName, ii.getOwner()); - return ii; - } - - public void reloadIILocal(String iiName) { - try { - reloadIILocalAt(IIInstance.concatResourcePath(iiName)); - } catch (IOException e) { - logger.error(e.getLocalizedMessage(), e); - } - } - - public IIInstance dropII(String iiName, boolean deleteDesc) throws IOException { - logger.info("Dropping II '" + iiName + "'"); - - IIInstance ii = getII(iiName); - - if (deleteDesc && ii.getDescriptor() != null) { - IIDescManager.getInstance(config).removeIIDesc(ii.getDescriptor()); - } - - removeII(ii); - ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.INVERTED_INDEX, iiName); - - return ii; - } - - private void removeII(IIInstance ii) throws IOException { - getStore().deleteResource(ii.getResourcePath()); - iiMap.remove(ii.getName()); - } - - public void removeIILocal(String name) { - iiMap.removeLocal(name); - usedStorageLocation.removeAll(name.toUpperCase()); - } - - public void updateII(IIInstance ii) throws IOException { - logger.info("Updating II instance : " + ii.getName()); - getStore().putResource(ii.getResourcePath(), ii, II_SERIALIZER); - iiMap.put(ii.getName(), ii); - - //this is a duplicate call to take care of scenarios where REST cache service unavailable - ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).clearL2Cache(); - } - - /** - * @param IIInstance - * @param startDate (pass 0 if full build) - * @param endDate (pass 0 if full build) - * @return - */ - public IISegment buildSegment(IIInstance IIInstance, long startDate, long endDate) { - IISegment segment = new IISegment(); - String incrementalSegName = IISegment.getSegmentName(startDate, endDate); - segment.setUuid(UUID.randomUUID().toString()); - segment.setName(incrementalSegName); - segment.setCreateTimeUTC(System.currentTimeMillis()); - segment.setDateRangeStart(startDate); - segment.setDateRangeEnd(endDate); - segment.setStatus(SegmentStatusEnum.NEW); - segment.setStorageLocationIdentifier(generateStorageLocation()); - - segment.setIIInstance(IIInstance); - - return segment; - } - - private String generateStorageLocation() { - while (true) { - StringBuilder sb = new StringBuilder(IRealizationConstants.IIHbaseStorageLocationPrefix); - for (int i = 0; i < HBASE_TABLE_LENGTH; i++) { - int idx = (int) (Math.random() * ALPHA_NUM.length()); - sb.append(ALPHA_NUM.charAt(idx)); - } - if (usedStorageLocation.containsValue(sb.toString())) { - continue; - } else { - return sb.toString(); - } - } - } - - private void loadAllIIInstance() throws IOException { - ResourceStore store = getStore(); - List<String> paths = store.collectResourceRecursively(ResourceStore.II_RESOURCE_ROOT, ".json"); - - logger.debug("Loading II from folder " + store.getReadableResourcePath(ResourceStore.II_RESOURCE_ROOT)); - - for (String path : paths) { - reloadIILocalAt(path); - } - - logger.debug("Loaded " + paths.size() + " II(s)"); - } - - private synchronized IIInstance reloadIILocalAt(String path) throws IOException { - ResourceStore store = getStore(); - logger.debug("Loading IIInstance " + store.getReadableResourcePath(path)); - - IIInstance ii = null; - try { - ii = store.getResource(path, IIInstance.class, II_SERIALIZER); - ii.setConfig(config); - - if (StringUtils.isBlank(ii.getName())) - throw new IllegalStateException("IIInstance name must not be blank"); - - iiMap.putLocal(ii.getName(), ii); - - for (IISegment segment : ii.getSegments()) { - usedStorageLocation.put(ii.getName().toUpperCase(), segment.getStorageLocationIdentifier()); - } - - return ii; - } catch (Exception e) { - logger.error("Error during load ii instance " + path, e); - return null; - } - } - - private ResourceStore getStore() { - return ResourceStore.getStore(this.config); - } - - @Override - public RealizationType getRealizationType() { - return RealizationType.INVERTED_INDEX; - } - - @Override - public IRealization getRealization(String name) { - return getII(name); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java deleted file mode 100644 index 34fd67c..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex; - -import java.text.SimpleDateFormat; -import java.util.List; -import java.util.TimeZone; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.invertedindex.index.TableRecordInfo; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc; -import org.apache.kylin.metadata.model.IJoinedFlatTableDesc; -import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.realization.IRealization; -import org.apache.kylin.metadata.realization.IRealizationSegment; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonBackReference; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Objects; - -/** - * @author honma - */ - -// TODO: remove segment concept for II, append old hbase table -@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class IISegment implements Comparable<IISegment>, IRealizationSegment { - - @JsonBackReference - private IIInstance iiInstance; - @JsonProperty("uuid") - private String uuid; - @JsonProperty("name") - private String name; - @JsonProperty("storage_location_identifier") - private String storageLocationIdentifier; // HTable name - @JsonProperty("date_range_start") - private long dateRangeStart; - @JsonProperty("date_range_end") - private long dateRangeEnd; - @JsonProperty("status") - private SegmentStatusEnum status; - @JsonProperty("size_kb") - private long sizeKB; - @JsonProperty("input_records") - private long inputRecords; - @JsonProperty("input_records_size") - private long inputRecordsSize; - @JsonProperty("last_build_time") - private long lastBuildTime; - @JsonProperty("last_build_job_id") - private String lastBuildJobID; - - @JsonProperty("create_time_utc") - private long createTimeUTC; - - @JsonProperty("binary_signature") - private String binarySignature; // a hash of schema and dictionary ID, - // used for sanity check - - private transient TableRecordInfo tableRecordInfo; - - /** - * @param startDate - * @param endDate - * @return if(startDate == 0 && endDate == 0), returns "FULL_BUILD", else - * returns "yyyyMMddHHmmss_yyyyMMddHHmmss" - */ - public static String getSegmentName(long startDate, long endDate) { - if (startDate == 0 && endDate == 0) { - return "FULL_BUILD"; - } - - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss"); - dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); - - return dateFormat.format(startDate) + "_" + dateFormat.format(endDate); - } - - public IIDesc getIIDesc() { - return getIIInstance().getDescriptor(); - } - - // ============================================================================ - - @Override - public KylinConfig getConfig() { - return iiInstance.getConfig(); - } - - @Override - public String getUuid() { - return uuid; - } - - public void setUuid(String id) { - this.uuid = id; - } - - @Override - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public long getDateRangeStart() { - return dateRangeStart; - } - - public void setDateRangeStart(long dateRangeStart) { - this.dateRangeStart = dateRangeStart; - } - - public long getDateRangeEnd() { - return dateRangeEnd; - } - - public void setDateRangeEnd(long dateRangeEnd) { - this.dateRangeEnd = dateRangeEnd; - } - - public SegmentStatusEnum getStatus() { - return status; - } - - public void setStatus(SegmentStatusEnum status) { - this.status = status; - } - - public long getSizeKB() { - return sizeKB; - } - - public void setSizeKB(long sizeKB) { - this.sizeKB = sizeKB; - } - - public long getInputRecords() { - return inputRecords; - } - - public void setInputRecords(long inputRecords) { - this.inputRecords = inputRecords; - } - - public long getInputRecordsSize() { - return inputRecordsSize; - } - - public void setInputRecordsSize(long inputRecordsSize) { - this.inputRecordsSize = inputRecordsSize; - } - - public long getLastBuildTime() { - return lastBuildTime; - } - - public void setLastBuildTime(long lastBuildTime) { - this.lastBuildTime = lastBuildTime; - } - - public String getLastBuildJobID() { - return lastBuildJobID; - } - - public void setLastBuildJobID(String lastBuildJobID) { - this.lastBuildJobID = lastBuildJobID; - } - - public String getBinarySignature() { - return binarySignature; - } - - public void setBinarySignature(String binarySignature) { - this.binarySignature = binarySignature; - } - - public IIInstance getIIInstance() { - return iiInstance; - } - - public void setIIInstance(IIInstance iiInstance) { - this.iiInstance = iiInstance; - } - - @Override - public String getStorageLocationIdentifier() { - return storageLocationIdentifier; - } - - /** - * @param storageLocationIdentifier the storageLocationIdentifier to set - */ - public void setStorageLocationIdentifier(String storageLocationIdentifier) { - this.storageLocationIdentifier = storageLocationIdentifier; - } - - @Override - public int compareTo(IISegment other) { - if (this.dateRangeEnd < other.dateRangeEnd) { - return -1; - } else if (this.dateRangeEnd > other.dateRangeEnd) { - return 1; - } else { - return 0; - } - } - - private TableRecordInfo getTableRecordInfo() { - if (tableRecordInfo == null) - tableRecordInfo = new TableRecordInfo(this); - return tableRecordInfo; - } - - public List<TblColRef> getColumns() { - return this.getTableRecordInfo().getColumns(); - } - - @Override - public String toString() { - return Objects.toStringHelper(this).add("uuid", uuid).add("create_time_utc:", createTimeUTC).add("name", name).add("last_build_job_id", lastBuildJobID).add("status", status).toString(); - } - - public long getCreateTimeUTC() { - return createTimeUTC; - } - - public void setCreateTimeUTC(long createTimeUTC) { - this.createTimeUTC = createTimeUTC; - } - - @Override - public int getEngineType() { - return 0; - } - - @Override - public int getSourceType() { - return 0; - } - - @Override - public int getStorageType() { - return 0; - } - - @Override - public IRealization getRealization() { - return iiInstance; - } - - @Override - public IJoinedFlatTableDesc getJoinedFlatTableDesc() { - return new IIJoinedFlatTableDesc(this.getIIDesc()); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BatchSliceMaker.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BatchSliceMaker.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BatchSliceMaker.java deleted file mode 100644 index 952cb8f..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BatchSliceMaker.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * - * contributor license agreements. See the NOTICE file distributed with - * - * this work for additional information regarding copyright ownership. - * - * The ASF licenses this file to You under the Apache License, Version 2.0 - * - * (the "License"); you may not use this file except in compliance with - * - * the License. You may obtain a copy of the License at - * - * - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * - * - * Unless required by applicable law or agreed to in writing, software - * - * distributed under the License is distributed on an "AS IS" BASIS, - * - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and - * - * limitations under the License. - * - * / - */ - -package org.apache.kylin.invertedindex.index; - -import java.util.List; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.kylin.invertedindex.model.IIDesc; - -import com.google.common.base.Preconditions; - -/** - */ -public class BatchSliceMaker { - - private final int nColumns; - private final int nRecordsCap; - private final short shard; - private final IIDesc desc; - - private long sliceTimestamp; - - transient ImmutableBytesWritable temp = new ImmutableBytesWritable(); - - public BatchSliceMaker(IIDesc desc, short shard) { - this.desc = desc; - this.nColumns = desc.listAllColumns().size(); - this.nRecordsCap = Math.max(1, desc.getSliceSize()); - - this.shard = shard; - this.sliceTimestamp = Long.MIN_VALUE; - } - - public Slice makeSlice(TableRecordInfoDigest digest, List<TableRecord> records) { - Preconditions.checkArgument(records != null && !records.isEmpty(), "records cannot be empty"); - Preconditions.checkArgument(records.size() <= nRecordsCap, "batch count cannot exceed " + nRecordsCap); - sliceTimestamp = increaseSliceTimestamp(records.get(0).getTimestamp()); - ColumnValueContainer[] containers = new ColumnValueContainer[nColumns]; - for (int i : desc.getValueColumns()) { - containers[i] = new CompressedValueContainer(digest, i, nRecordsCap); - } - for (int i : desc.getMetricsColumns()) { - containers[i] = new CompressedValueContainer(digest, i, nRecordsCap); - } - for (TableRecord record : records) { - for (int i = 0; i < nColumns; i++) { - record.getValueBytes(i, temp); - containers[i].append(temp); - } - } - return new Slice(digest, shard, sliceTimestamp, containers); - - } - - private long increaseSliceTimestamp(long timestamp) { - if (timestamp <= sliceTimestamp) { - return sliceTimestamp + 1; // ensure slice timestamp increases - } else { - return timestamp; - } - } - -}