madlib-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From iyerr3 <...@git.apache.org>
Subject [GitHub] madlib pull request #288: Jira:1239: Converts features from multiple columns...
Date Wed, 04 Jul 2018 00:38:46 GMT
Github user iyerr3 commented on a diff in the pull request:

    https://github.com/apache/madlib/pull/288#discussion_r199983644
  
    --- Diff: src/ports/postgres/modules/cols_vec/cols2vec.py_in ---
    @@ -0,0 +1,110 @@
    +"""
    +@file cols2vec.py_in
    +
    +@brief Utility to convert Columns to array
    +
    +"""
    +
    +import plpy
    +from utilities.control import MinWarning
    +from utilities.utilities import split_quoted_delimited_str
    +from utilities.utilities import _string_to_array
    +from utilities.validate_args import columns_exist_in_table
    +from utilities.validate_args import is_var_valid
    +from utilities.validate_args import get_cols
    +from utilities.validate_args import quote_ident
    +from utilities.utilities import py_list_to_sql_string
    +
    +
    +m4_changequote(`<!', `!>')
    +
    +
    +def validate_cols2vec_args(source_table, output_table,
    +                           list_of_features, list_of_features_to_exclude, cols_to_output,
**kwargs):
    +    """
    +        Function to validate input parameters
    +    """
    +    if list_of_features.strip() != '*':
    +        if not (list_of_features and list_of_features.strip()):
    +            plpy.error("Features to include is empty")
    +
    +    if list_of_features.strip() != '*':
    +        if not columns_exist_in_table(
    +                source_table, split_quoted_delimited_str(list_of_features)):
    +            plpy.error(
    +                "Invalid columns to list_of_features ({0})".format(list_of_features))
    +
    +    if cols_to_output and cols_to_output.strip() != '*':
    +        if not columns_exist_in_table(
    +                source_table, _string_to_array(cols_to_output)):
    +            plpy.error("Invalid columns to output list ({0})".
    +                       format(cols_to_output))
    +
    +
    +def cols2vec(schema_madlib, source_table, output_table, list_of_features,
    +             list_of_features_to_exclude=None, cols_to_output=None, **kwargs):
    +    """
    +    Args:
    +        @param schema_madlib:               Name of MADlib schema
    +        @param model:                       Name of table containing the tree model
    +        @param source_table:                Name of table containing prediction data
    +        @param output_table:                Name of table to output the results
    +        @param list_of_features:            Comma-separated string of column names or
    +                                            expressions to put into feature array.
    +                                            Can also be a '*' implying all columns
    +                                            are to be put into feature array.
    +        @param list_of_features_to_exclude: Comma-separated string of column names
    +                                            to exclude from the feature array
    +        @param cols_to_output:              Comma-separated string of column names
    +                                            from the source table to keep in the output
table,
    +                                            in addition to the feature array.
    +
    +    Returns:
    +        None
    +
    +    """
    +
    +    with MinWarning('warning'):
    +        validate_cols2vec_args(source_table, output_table, list_of_features,
    +                               list_of_features_to_exclude, cols_to_output, **kwargs)
    +
    +        all_cols = ''
    +        feature_cols = ''
    +        if list_of_features.strip() == '*':
    +            all_cols = get_cols(source_table, schema_madlib)
    +            all_col_set = set(list(all_cols))
    +            exclude_set = set(split_quoted_delimited_str(
    +                list_of_features_to_exclude))
    +            feature_set = all_col_set - exclude_set
    +            feature_cols = py_list_to_sql_string(
    +                list(feature_set), "text", False)
    +            filtered_list_of_features = ",".join(
    +                feat for feat in list(feature_set))
    +        else:
    +            feature_list = split_quoted_delimited_str(list_of_features)
    +            feature_exclude = split_quoted_delimited_str(
    +                list_of_features_to_exclude)
    +            return_set = set(feature_list) - set(feature_exclude)
    +            feature_cols = py_list_to_sql_string(
    +                list(return_set), "text", False)
    +            filtered_list_of_features = ",".join(
    +                feat for feat in feature_list if feat in return_set)
    +
    +        output_cols = ''
    +        if cols_to_output:
    +            output_cols_list = [', '.join(get_cols(source_table, schema_madlib)) if col
== '*' else col
    +                                for col in split_quoted_delimited_str(cols_to_output)]
    +            output_cols = ', '.join(output_cols_list) + ","
    +
    +        plpy.execute("""
    +    		CREATE TABLE {output_table} AS
    +    		select {output_cols}
    +            array[{filtered_list_of_features}] as feature_vector
    +    		from {source_table}
    +    		""".format(**locals()))
    +
    +        plpy.execute("""
    --- End diff --
    
    I haven't understood the need for the summary table. Is it just to record the features
combined in the array? Can we provide that as an output of the function? Creating a table
just to record that one parameter seems unnecessary. 


---

Mime
View raw message