From dev-return-6213-archive-asf-public=cust-asf.ponee.io@madlib.apache.org Fri Oct 9 13:24:00 2020 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mxout1-ec2-va.apache.org (mxout1-ec2-va.apache.org [3.227.148.255]) by mx-eu-01.ponee.io (Postfix) with ESMTPS id DD034180642 for ; Fri, 9 Oct 2020 15:23:59 +0200 (CEST) Received: from mail.apache.org (mailroute1-lw-us.apache.org [207.244.88.153]) by mxout1-ec2-va.apache.org (ASF Mail Server at mxout1-ec2-va.apache.org) with SMTP id 269BE45BA3 for ; Fri, 9 Oct 2020 13:23:59 +0000 (UTC) Received: (qmail 2706 invoked by uid 500); 9 Oct 2020 13:23:58 -0000 Mailing-List: contact dev-help@madlib.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@madlib.apache.org Delivered-To: mailing list dev@madlib.apache.org Received: (qmail 2686 invoked by uid 99); 9 Oct 2020 13:23:58 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 09 Oct 2020 13:23:58 +0000 From: =?utf-8?q?GitBox?= To: dev@madlib.apache.org Subject: =?utf-8?q?=5BGitHub=5D_=5Bmadlib=5D_fmcquillan99_commented_on_pull_request_?= =?utf-8?q?=23519=3A_DL=3A_Add_a_helper_function_to_load_custom_top_n_accura?= =?utf-8?q?cy_functions?= Message-ID: <160224983854.32230.11012729243682211909.asfpy@gitbox.apache.org> Date: Fri, 09 Oct 2020 13:23:58 -0000 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit In-Reply-To: References: fmcquillan99 commented on pull request #519: URL: https://github.com/apache/madlib/pull/519#issuecomment-705841751 -- (0) can we please rename this to : -- -- load_top_k_accuracy_function( -- object table, -- k -- ) -- -- because this is the terminology that Keras uses -- (1) multi-model, top k default DROP TABLE IF EXISTS mst_table, mst_table_summary; SELECT madlib.load_model_selection_table('model_arch_library', -- model architecture table 'mst_table', -- model selection table output ARRAY[1,2], -- model ids from model architecture table ARRAY[ -- compile params $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['top_k_categorical_accuracy']$$, $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['top_k_categorical_accuracy']$$, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['top_k_categorical_accuracy']$$ ], ARRAY[ -- fit params $$batch_size=4,epochs=1$$, $$batch_size=8,epochs=1$$ ] ); SELECT * FROM mst_table ORDER BY mst_key; ``` mst_key | model_id | compile_params | fit_params ---------+----------+---------------------------------------------------------------------------------------------------+----------------------- 1 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['top_k_categorical_accuracy'] | batch_size=4,epochs=1 2 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['top_k_categorical_accuracy'] | batch_size=8,epochs=1 3 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['top_k_categorical_accuracy'] | batch_size=4,epochs=1 4 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['top_k_categorical_accuracy'] | batch_size=8,epochs=1 5 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['top_k_categorical_accuracy'] | batch_size=4,epochs=1 6 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['top_k_categorical_accuracy'] | batch_size=8,epochs=1 7 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['top_k_categorical_accuracy'] | batch_size=4,epochs=1 8 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['top_k_categorical_accuracy'] | batch_size=8,epochs=1 9 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['top_k_categorical_accuracy'] | batch_size=4,epochs=1 10 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['top_k_categorical_accuracy'] | batch_size=8,epochs=1 11 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['top_k_categorical_accuracy'] | batch_size=4,epochs=1 12 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['top_k_categorical_accuracy'] | batch_size=8,epochs=1 (12 rows) ``` DROP TABLE IF EXISTS iris_multi_model, iris_multi_model_summary, iris_multi_model_info; SELECT madlib.madlib_keras_fit_multiple_model('iris_train_packed', -- source_table 'iris_multi_model', -- model_output_table 'mst_table', -- model_selection_table 2, -- num_iterations FALSE -- use gpus ); SELECT * FROM iris_multi_model_info; ``` -[ RECORD 1 ]-------------+--------------------------------------------- source_table | iris_train_packed validation_table | model | iris_multi_model model_info | iris_multi_model_info dependent_varname | class_text independent_varname | attributes model_arch_table | model_arch_library model_selection_table | mst_table object_table | num_iterations | 2 metrics_compute_frequency | 2 warm_start | f name | description | start_training_time | 2020-10-08 21:00:11.766483 end_training_time | 2020-10-08 21:00:36.933537 madlib_version | 1.18.0-dev num_classes | 3 class_values | {Iris-setosa,Iris-versicolor,Iris-virginica} dependent_vartype | character varying normalizing_const | 1 metrics_iters | {2} Time: 8.248 ms madlib=# SELECT * FROM iris_multi_model_info; -[ RECORD 1 ]------------+-------------------------------------------------------------------------------------------------- mst_key | 2 model_id | 1 compile_params | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['top_k_categorical_accuracy'] fit_params | batch_size=8,epochs=1 model_type | madlib_keras model_size | 0.7900390625 metrics_elapsed_time | {23.0940079689026} metrics_type | {top_k_categorical_accuracy} loss_type | categorical_crossentropy training_metrics_final | 1 training_loss_final | 0.981531918048859 training_metrics | {1} training_loss | {0.981531918048859} validation_metrics_final | validation_loss_final | validation_metrics | validation_loss | -[ RECORD 2 ]------------+-------------------------------------------------------------------------------------------------- mst_key | 10 model_id | 2 compile_params | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['top_k_categorical_accuracy'] fit_params | batch_size=8,epochs=1 model_type | madlib_keras model_size | 1.2197265625 metrics_elapsed_time | {23.4646620750427} metrics_type | {top_k_categorical_accuracy} loss_type | categorical_crossentropy training_metrics_final | 1 training_loss_final | 0.49222993850708 training_metrics | {1} training_loss | {0.49222993850708} validation_metrics_final | validation_loss_final | validation_metrics | validation_loss | etc ``` -- OK -- (2) multi-model, top k custom function DROP TABLE IF EXISTS custom_function_table; SELECT madlib.load_top_n_accuracy_function('custom_function_table', 3); ``` INFO: Keras Custom Function: Created new custom function table custom_function_table. CONTEXT: PL/Python function "load_custom_function" SQL statement " SELECT madlib.load_custom_function('custom_function_table', madlib.top_k_categorical_acc_pickled(3, 'top_3_accuracy'), 'top_3_accuracy', 'returns top_3_accuracy'); " PL/Python function "load_custom_function" INFO: Keras Custom Function: Added function top_3_accuracy to custom_function_table table CONTEXT: PL/Python function "load_custom_function" SQL statement " SELECT madlib.load_custom_function('custom_function_table', madlib.top_k_categorical_acc_pickled(3, 'top_3_accuracy'), 'top_3_accuracy', 'returns top_3_accuracy'); " PL/Python function "load_custom_function" load_top_n_accuracy_function ------------------------------ (1 row) ``` SELECT id, name, description FROM custom_function_table ORDER BY id; ``` id | name | description ----+----------------+------------------------ 1 | top_3_accuracy | returns top_3_accuracy (1 row) ``` -- results OK but please remove verbose output -- (3) multi-model, run fit() with custom function DROP TABLE IF EXISTS mst_table, mst_table_summary; SELECT madlib.load_model_selection_table('model_arch_library', -- model architecture table 'mst_table', -- model selection table output ARRAY[1,2], -- model ids from model architecture table ARRAY[ -- compile params $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['top_3_accuracy']$$, $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['top_3_accuracy']$$, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['top_3_accuracy']$$ ], ARRAY[ -- fit params $$batch_size=4,epochs=1$$, $$batch_size=8,epochs=1$$ ], 'custom_function_table' -- custom table ); SELECT * FROM mst_table ORDER BY mst_key; ``` mst_key | model_id | compile_params | fit_params ---------+----------+---------------------------------------------------------------------------------------+----------------------- 1 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['top_3_accuracy'] | batch_size=4,epochs=1 2 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['top_3_accuracy'] | batch_size=8,epochs=1 3 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['top_3_accuracy'] | batch_size=4,epochs=1 4 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['top_3_accuracy'] | batch_size=8,epochs=1 5 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['top_3_accuracy'] | batch_size=4,epochs=1 6 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['top_3_accuracy'] | batch_size=8,epochs=1 7 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['top_3_accuracy'] | batch_size=4,epochs=1 8 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['top_3_accuracy'] | batch_size=8,epochs=1 9 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['top_3_accuracy'] | batch_size=4,epochs=1 10 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['top_3_accuracy'] | batch_size=8,epochs=1 11 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['top_3_accuracy'] | batch_size=4,epochs=1 12 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['top_3_accuracy'] | batch_size=8,epochs=1 (12 rows) ``` DROP TABLE IF EXISTS iris_multi_model, iris_multi_model_summary, iris_multi_model_info; SELECT madlib.madlib_keras_fit_multiple_model('iris_train_packed', -- source_table 'iris_multi_model', -- model_output_table 'mst_table', -- model_selection_table 2, -- num_iterations FALSE, -- use gpus NULL, NULL ); SELECT * FROM iris_multi_model_summary; ``` source_table | iris_train_packed validation_table | model | iris_multi_model model_info | iris_multi_model_info dependent_varname | class_text independent_varname | attributes model_arch_table | model_arch_library model_selection_table | mst_table object_table | custom_function_table num_iterations | 2 metrics_compute_frequency | 2 warm_start | f name | description | start_training_time | 2020-10-08 21:22:27.219617 end_training_time | 2020-10-08 21:22:52.310825 madlib_version | 1.18.0-dev num_classes | 3 class_values | {Iris-setosa,Iris-versicolor,Iris-virginica} dependent_vartype | character varying normalizing_const | 1 metrics_iters | {2} ``` SELECT * FROM iris_multi_model_info; ``` -[ RECORD 1 ]------------+-------------------------------------------------------------------------------------- mst_key | 2 model_id | 1 compile_params | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['top_3_accuracy'] fit_params | batch_size=8,epochs=1 model_type | madlib_keras model_size | 0.7900390625 metrics_elapsed_time | {22.8225800991058} metrics_type | {top_3_accuracy} loss_type | categorical_crossentropy training_metrics_final | 1 training_loss_final | 0.488488465547562 training_metrics | {1} training_loss | {0.488488465547562} validation_metrics_final | validation_loss_final | validation_metrics | validation_loss | -[ RECORD 2 ]------------+-------------------------------------------------------------------------------------- mst_key | 10 model_id | 2 compile_params | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['top_3_accuracy'] fit_params | batch_size=8,epochs=1 model_type | madlib_keras model_size | 1.2197265625 metrics_elapsed_time | {23.2352600097656} metrics_type | {top_3_accuracy} loss_type | categorical_crossentropy training_metrics_final | 1 training_loss_final | 0.713700234889984 training_metrics | {1} training_loss | {0.713700234889984} validation_metrics_final | validation_loss_final | validation_metrics | validation_loss | etc ``` -- OK -- (4) single-model, run fit() with custom function -- did you test with madlib.madlib_keras_fit() ? ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: users@infra.apache.org