ctakes-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dlig...@apache.org
Subject svn commit: r1761469 - in /ctakes/trunk/ctakes-temporal/scripts/nn: predict.py train_and_package.py
Date Mon, 19 Sep 2016 15:40:48 GMT
Author: dligach
Date: Mon Sep 19 15:40:48 2016
New Revision: 1761469

URL: http://svn.apache.org/viewvc?rev=1761469&view=rev
Log:
cleaned up chen's scripts a bit more

Modified:
    ctakes/trunk/ctakes-temporal/scripts/nn/predict.py
    ctakes/trunk/ctakes-temporal/scripts/nn/train_and_package.py

Modified: ctakes/trunk/ctakes-temporal/scripts/nn/predict.py
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/scripts/nn/predict.py?rev=1761469&r1=1761468&r2=1761469&view=diff
==============================================================================
--- ctakes/trunk/ctakes-temporal/scripts/nn/predict.py (original)
+++ ctakes/trunk/ctakes-temporal/scripts/nn/predict.py Mon Sep 19 15:40:48 2016
@@ -21,12 +21,11 @@ def main(args):
         2:'CONTAINS-1'
     }
 
-    ## Load models and weights:
-    #outcomes = ctk_io.get_outcome_array(working_dir)
-    model_dir = "/Users/Dima/Loyola/Workspaces/cTakes/ctakes/ctakes-temporal/target/eval/thyme/train_and_test/event-time"
+    ctakes_root = '/Users/Dima/Loyola/Workspaces/cTakes/ctakes/'
+    target_dir = 'ctakes-temporal/target/eval/thyme/train_and_test/event-time/'
+    model_dir = ctakes_root + target_dir
     maxlen   = pickle.load(open(os.path.join(model_dir, "maxlen.p"), "rb"))
     alphabet = pickle.load(open(os.path.join(model_dir, "alphabet.p"), "rb"))
-    #print("Outcomes array is %s" % (outcomes) )
     model = model_from_json(open(os.path.join(model_dir, "model_0.json")).read())
     model.load_weights(os.path.join(model_dir, "model_0.h5"))
 
@@ -36,26 +35,25 @@ def main(args):
             if not line:
                 break
 
-            ## Convert the line of Strings to lists of indices
             feats=[]
             for unigram in line.rstrip().split():
                 if(alphabet.has_key(unigram)):
                     feats.append(alphabet[unigram])
                 else:
                     feats.append(alphabet["none"])
-            if(len(feats)> maxlen):
+                    
+            if(len(feats) > maxlen):
                 feats=feats[0:maxlen]
             test_x = pad_sequences([feats], maxlen=maxlen)
-            #feats = np.reshape(feats, (1, 6, input_dims / 6))
-            #feats = np.reshape(feats, (1, input_dims))
 
             X_dup = []
             X_dup.append(test_x)
             X_dup.append(test_x)
             X_dup.append(test_x)
+            X_dup.append(test_x)
 
             out = model.predict(X_dup, batch_size=50)[0]
-            # print("Out is %s and decision is %d" % (out, out.argmax()))
+
         except KeyboardInterrupt:
             sys.stderr.write("Caught keyboard interrupt\n")
             break
@@ -65,12 +63,10 @@ def main(args):
             break
 
         out_str = int2label[out.argmax()]
-
         print(out_str)
         sys.stdout.flush()
 
     sys.exit(0)
 
-
 if __name__ == "__main__":
     main(sys.argv[1:])

Modified: ctakes/trunk/ctakes-temporal/scripts/nn/train_and_package.py
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/scripts/nn/train_and_package.py?rev=1761469&r1=1761468&r2=1761469&view=diff
==============================================================================
--- ctakes/trunk/ctakes-temporal/scripts/nn/train_and_package.py (original)
+++ ctakes/trunk/ctakes-temporal/scripts/nn/train_and_package.py Mon Sep 19 15:40:48 2016
@@ -1,18 +1,13 @@
 #!/usr/bin/env python
 
 import sklearn as sk
-
 import numpy as np
 np.random.seed(1337)
-
 import et_cleartk_io as ctk_io
 import nn_models
-
 import sys
 import os.path
-
 import dataset
-
 import keras as k
 from keras.utils.np_utils import to_categorical
 from keras.optimizers import RMSprop
@@ -22,28 +17,20 @@ from keras.layers import Merge
 from keras.layers.core import Dense, Dropout, Activation, Flatten
 from keras.layers.convolutional import Convolution1D, MaxPooling1D
 from keras.layers.embeddings import Embedding
-
 import pickle
 
 def main(args):
     if len(args) < 1:
         sys.stderr.write("Error - one required argument: <data directory>\n")
         sys.exit(-1)
-
+        
     working_dir = args[0]
-
-    #read in data file
-#    print("Reading data...")
-    #Y, X = ctk_io.read_liblinear(working_dir) # ('data_testing/multitask_assertion/train_and_test')
     data_file = os.path.join(working_dir, 'training-data.liblinear')
 
-    # learn alphabet from training and test data
-    dataset1 = dataset.DatasetProvider([data_file])
+    # learn alphabet from training data
+    data_set = dataset.DatasetProvider([data_file])
     # now load training examples and labels
-    train_x, train_y = dataset1.load(data_file)
-
-    init_vectors = None #used for pre-trained embeddings
-    
+    train_x, train_y = data_set.load(data_file)
     # turn x and y into numpy array among other things
     maxlen = max([len(seq) for seq in train_x])
     outcomes = set(train_y)
@@ -53,58 +40,55 @@ def main(args):
     train_y = to_categorical(np.array(train_y), classes)
 
     pickle.dump(maxlen, open(os.path.join(working_dir, 'maxlen.p'),"wb"))
-    pickle.dump(dataset1.alphabet, open(os.path.join(working_dir, 'alphabet.p'),"wb"))
-    #test_x = pad_sequences(test_x, maxlen=maxlen)
-    #test_y = to_categorical(np.array(test_y), classes)
+    pickle.dump(data_set.alphabet, open(os.path.join(working_dir, 'alphabet.p'),"wb"))
 
     print 'train_x shape:', train_x.shape
     print 'train_y shape:', train_y.shape
 
     branches = [] # models to be merged
     train_xs = [] # train x for each branch
-    #test_xs = []  # test x for each branch
 
-    filtlens = "3,4,5"
-    for filter_len in filtlens.split(','):
+    for filter_len in '2,3,4,5'.split(','):
+      
         branch = Sequential()
-        branch.add(Embedding(len(dataset1.alphabet),
-                         300,
-                         input_length=maxlen,
-                         weights=init_vectors))
+        branch.add(Embedding(len(data_set.alphabet),
+                             300,
+                             input_length=maxlen,
+                             weights=None))
         branch.add(Convolution1D(nb_filter=200,
-                             filter_length=int(filter_len),
-                             border_mode='valid',
-                             activation='relu',
-                             subsample_length=1))
+                                 filter_length=int(filter_len),
+                                 border_mode='valid',
+                                 activation='relu',
+                                 subsample_length=1))
         branch.add(MaxPooling1D(pool_length=2))
         branch.add(Flatten())
 
         branches.append(branch)
         train_xs.append(train_x)
-        #test_xs.append(test_x)
+
     model = Sequential()
     model.add(Merge(branches, mode='concat'))
 
-    model.add(Dense(250))#cfg.getint('cnn', 'hidden')))
-    model.add(Dropout(0.25))#cfg.getfloat('cnn', 'dropout')))
+    model.add(Dense(300))
+    model.add(Dropout(0.25))
     model.add(Activation('relu'))
 
-    model.add(Dropout(0.25))#cfg.getfloat('cnn', 'dropout')))
+    model.add(Dropout(0.25))
     model.add(Dense(classes))
     model.add(Activation('softmax'))
 
-    optimizer = RMSprop(lr=0.0001,#cfg.getfloat('cnn', 'learnrt'),
-                      rho=0.9, epsilon=1e-08)
+    optimizer = RMSprop(lr=0.0001,
+                        rho=0.9, epsilon=1e-08)
     model.compile(loss='categorical_crossentropy',
-                optimizer=optimizer,
-                metrics=['accuracy'])
+                  optimizer=optimizer,
+                  metrics=['accuracy'])
     model.fit(train_xs,
-            train_y,
-            nb_epoch=3,#cfg.getint('cnn', 'epochs'),
-            batch_size=50,#cfg.getint('cnn', 'batches'),
-            verbose=1,
-            validation_split=0.1,
-            class_weight=None)
+              train_y,
+              nb_epoch=3,
+              batch_size=50,
+              verbose=1,
+              validation_split=0.1,
+              class_weight=None)
 
     model.summary()
 
@@ -114,4 +98,4 @@ def main(args):
     sys.exit(0)
 
 if __name__ == "__main__":
-    main(sys.argv[1:])
\ No newline at end of file
+    main(sys.argv[1:])



Mime
View raw message