systemml-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Niketan Pansare (JIRA)" <j...@apache.org>
Subject [jira] [Updated] (SYSTEMML-1583) Implement converter in Python to convert caffemodel in SystemML format
Date Thu, 04 May 2017 20:15:04 GMT

     [ https://issues.apache.org/jira/browse/SYSTEMML-1583?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Niketan Pansare updated SYSTEMML-1583:
--------------------------------------
    Description: 
Ideally, this converter shouldnot require the caffe to be installed. Please see http://stackoverflow.com/questions/37572948/extracting-weights-from-caffemodel-without-caffe-installed-in-python

An example code to convert a caffe model to csv if caffe is installed:
{code}
import caffe
import numpy as np
#net = caffe.Net('/home/biuser/nike/barista/VGG_ILSVRC_19_layers_train_val.prototxt', caffe.TEST)
net = caffe.Net('/home/biuser/VGG_trained_models/VGG_ILSVRC_19_layers_deploy.prototxt', '/home/biuser/VGG_trained_models/VGG_ILSVRC_19_layers.caffemodel',
caffe.TEST)
#surgery.transplant(net, base_net)
for l in [ "conv1_1", "conv1_2", "conv2_1", "conv2_2", "conv3_1", "conv3_2", "conv3_3", "conv3_4",
"conv4_1", "conv4_2", "conv4_3", "conv4_4", "conv5_1", "conv5_2", "conv5_3", "conv5_4", "fc6",
"fc7", "fc8" ]:
        w = net.params[l][0].data
        w = w.reshape(w.shape[0], -1)
        b = net.params[l][1].data
        b = b.reshape(b.shape[0], -1)
        # You may have to reshape it for fc layers
        np.savetxt("VGG_trained_models/" + l + "_weight.csv", w, delimiter=",")
        np.savetxt("VGG_trained_models/" + l + "_bias.csv", b, delimiter=",")
{code}

Here is an example pyspark script to test this JIRA:
{code}
from systemml.mllearn import Caffe2DML
from pyspark.sql import SQLContext
import numpy as np
import urllib, os, scipy.ndimage
from PIL import Image
import systemml as sml

# ImageNet specific parameters
img_shape = (3, 224, 224)

# Downloads a jpg image, resizes it to 224 and return as numpy array in N X CHW format
url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/58/MountainLion.jpg/312px-MountainLion.jpg'
outFile = 'test.jpg'
urllib.urlretrieve(url, outFile)
input_image = sml.convertImageToNumPyArr(Image.open(outFile), img_shape=img_shape)

# Download the ResNet network
import urllib
urllib.urlretrieve('https://raw.githubusercontent.com/niketanpansare/model_zoo/master/caffe/vision/resnet/ilsvrc12/ResNet_50_network.proto',
'ResNet_50_network.proto')
urllib.urlretrieve('https://raw.githubusercontent.com/niketanpansare/model_zoo/master/caffe/vision/resnet/ilsvrc12/ResNet_50_solver.proto',
'ResNet_50_solver.proto')

home_dir = os.path.expanduser('~')

# let's assume that this function is implemented as saveAsBinaryBlock(inputCaffeModel, outputDir)
resnet_pretrained_weight_dir = os.path.join(home_dir, 'model_zoo', 'caffe', 'vision', 'resnet',
'ilsvrc12', 'ResNet_50_pretrained_weights')
urllib.urlretrieve(('https://deepdetect.com/models/resnet/ResNet-50-model.caffemodel', 'ResNet-50-model.caffemodel')
#######################################################################
# To be implemented as part of this JIRA
sml.saveAsBinaryBlock('ResNet-50-model.caffemodel', resnet_pretrained_weight_dir)
#######################################################################
resnet = Caffe2DML(sqlCtx, solver='ResNet_50_solver.proto', weights=resnet_pretrained_weight_dir,
input_shape=img_shape)
resnet.predict(input_image)
# This should return array(['cougar, puma, catamount, mountain lion, painter, panther, Felis
'], dtype='|S64')
{code}

  was:
Ideally, this converter shouldnot require the caffe to be installed. Please see http://stackoverflow.com/questions/37572948/extracting-weights-from-caffemodel-without-caffe-installed-in-python

An example code to convert a caffe model to csv if caffe is installed:
{code}
import caffe
import numpy as np
#net = caffe.Net('/home/biuser/nike/barista/VGG_ILSVRC_19_layers_train_val.prototxt', caffe.TEST)
net = caffe.Net('/home/biuser/VGG_trained_models/VGG_ILSVRC_19_layers_deploy.prototxt', '/home/biuser/VGG_trained_models/VGG_ILSVRC_19_layers.caffemodel',
caffe.TEST)
#surgery.transplant(net, base_net)
for l in [ "conv1_1", "conv1_2", "conv2_1", "conv2_2", "conv3_1", "conv3_2", "conv3_3", "conv3_4",
"conv4_1", "conv4_2", "conv4_3", "conv4_4", "conv5_1", "conv5_2", "conv5_3", "conv5_4", "fc6",
"fc7", "fc8" ]:
        w = net.params[l][0].data
        w = w.reshape(w.shape[0], -1)
        b = net.params[l][1].data
        b = b.reshape(b.shape[0], -1)
        # You may have to reshape it for fc layers
        np.savetxt("VGG_trained_models/" + l + "_weight.csv", w, delimiter=",")
        np.savetxt("VGG_trained_models/" + l + "_bias.csv", b, delimiter=",")
{code}

Here is an example pyspark script to test this JIRA:
{code}
from systemml.mllearn import Caffe2DML
from pyspark.sql import SQLContext
import numpy as np
import urllib, os, scipy.ndimage
from PIL import Image
import systemml as sml

# ImageNet specific parameters
img_shape = (3, 224, 224)

# Downloads a jpg image, resizes it to 224 and return as numpy array in N X CHW format
url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/58/MountainLion.jpg/312px-MountainLion.jpg'
outFile = 'test.jpg'
urllib.urlretrieve(url, outFile)
input_image = sml.convertImageToNumPyArr(Image.open(outFile), img_shape=img_shape)

# Download the ResNet network
import urllib
urllib.urlretrieve('https://raw.githubusercontent.com/niketanpansare/model_zoo/master/caffe/vision/resnet/ilsvrc12/ResNet_50_network.proto',
'ResNet_50_network.proto')
urllib.urlretrieve('https://raw.githubusercontent.com/niketanpansare/model_zoo/master/caffe/vision/resnet/ilsvrc12/ResNet_50_solver.proto',
'ResNet_50_solver.proto')

home_dir = os.path.expanduser('~')

# let's assume that this function is implemented as saveAsBinaryBlock(inputCaffeModel, outputDir)
resnet_pretrained_weight_dir = os.path.join(home_dir, 'model_zoo', 'caffe', 'vision', 'resnet',
'ilsvrc12', 'ResNet_50_pretrained_weights')
urllib.urlretrieve(('https://deepdetect.com/models/resnet/ResNet-50-model.caffemodel', 'ResNet-50-model.caffemodel')
sml.saveAsBinaryBlock('ResNet-50-model.caffemodel', resnet_pretrained_weight_dir)
resnet = Caffe2DML(sqlCtx, solver='ResNet_50_solver.proto', weights=resnet_pretrained_weight_dir,
input_shape=img_shape)
resnet.predict(input_image)
# This should return array(['cougar, puma, catamount, mountain lion, painter, panther, Felis
'], dtype='|S64')
{code}


> Implement converter in Python to convert caffemodel in SystemML format
> ----------------------------------------------------------------------
>
>                 Key: SYSTEMML-1583
>                 URL: https://issues.apache.org/jira/browse/SYSTEMML-1583
>             Project: SystemML
>          Issue Type: Sub-task
>            Reporter: Niketan Pansare
>            Assignee: Arvind Surve
>
> Ideally, this converter shouldnot require the caffe to be installed. Please see http://stackoverflow.com/questions/37572948/extracting-weights-from-caffemodel-without-caffe-installed-in-python
> An example code to convert a caffe model to csv if caffe is installed:
> {code}
> import caffe
> import numpy as np
> #net = caffe.Net('/home/biuser/nike/barista/VGG_ILSVRC_19_layers_train_val.prototxt',
caffe.TEST)
> net = caffe.Net('/home/biuser/VGG_trained_models/VGG_ILSVRC_19_layers_deploy.prototxt',
'/home/biuser/VGG_trained_models/VGG_ILSVRC_19_layers.caffemodel', caffe.TEST)
> #surgery.transplant(net, base_net)
> for l in [ "conv1_1", "conv1_2", "conv2_1", "conv2_2", "conv3_1", "conv3_2", "conv3_3",
"conv3_4", "conv4_1", "conv4_2", "conv4_3", "conv4_4", "conv5_1", "conv5_2", "conv5_3", "conv5_4",
"fc6", "fc7", "fc8" ]:
>         w = net.params[l][0].data
>         w = w.reshape(w.shape[0], -1)
>         b = net.params[l][1].data
>         b = b.reshape(b.shape[0], -1)
>         # You may have to reshape it for fc layers
>         np.savetxt("VGG_trained_models/" + l + "_weight.csv", w, delimiter=",")
>         np.savetxt("VGG_trained_models/" + l + "_bias.csv", b, delimiter=",")
> {code}
> Here is an example pyspark script to test this JIRA:
> {code}
> from systemml.mllearn import Caffe2DML
> from pyspark.sql import SQLContext
> import numpy as np
> import urllib, os, scipy.ndimage
> from PIL import Image
> import systemml as sml
> # ImageNet specific parameters
> img_shape = (3, 224, 224)
> # Downloads a jpg image, resizes it to 224 and return as numpy array in N X CHW format
> url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/58/MountainLion.jpg/312px-MountainLion.jpg'
> outFile = 'test.jpg'
> urllib.urlretrieve(url, outFile)
> input_image = sml.convertImageToNumPyArr(Image.open(outFile), img_shape=img_shape)
> # Download the ResNet network
> import urllib
> urllib.urlretrieve('https://raw.githubusercontent.com/niketanpansare/model_zoo/master/caffe/vision/resnet/ilsvrc12/ResNet_50_network.proto',
'ResNet_50_network.proto')
> urllib.urlretrieve('https://raw.githubusercontent.com/niketanpansare/model_zoo/master/caffe/vision/resnet/ilsvrc12/ResNet_50_solver.proto',
'ResNet_50_solver.proto')
> home_dir = os.path.expanduser('~')
> # let's assume that this function is implemented as saveAsBinaryBlock(inputCaffeModel,
outputDir)
> resnet_pretrained_weight_dir = os.path.join(home_dir, 'model_zoo', 'caffe', 'vision',
'resnet', 'ilsvrc12', 'ResNet_50_pretrained_weights')
> urllib.urlretrieve(('https://deepdetect.com/models/resnet/ResNet-50-model.caffemodel',
'ResNet-50-model.caffemodel')
> #######################################################################
> # To be implemented as part of this JIRA
> sml.saveAsBinaryBlock('ResNet-50-model.caffemodel', resnet_pretrained_weight_dir)
> #######################################################################
> resnet = Caffe2DML(sqlCtx, solver='ResNet_50_solver.proto', weights=resnet_pretrained_weight_dir,
input_shape=img_shape)
> resnet.predict(input_image)
> # This should return array(['cougar, puma, catamount, mountain lion, painter, panther,
Felis '], dtype='|S64')
> {code}



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Mime
View raw message