Merge pull request #9 from kuke/convert_fit_a_line

Enable to load parameters & convert fit_a_line model

Author: Yibing Liu

convert.py

@@ -15,65 +15,118 @@
 import os
 import argparse
 
-from onnx import helper
+from onnx import helper, checker
 import paddle.fluid as fluid
 
-import ops
-from variables import paddle_variable_to_onnx_tensor
+import fluid_onnx.ops as ops
+from fluid_onnx.variables import paddle_variable_to_onnx_tensor
+from fluid_onnx.variables import PADDLE_TO_ONNX_DTYPE
 
 
-def convert(dirname):
+def parse_args():
+ # Read arguments: path to model.
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--fluid_model", required=True, help="Input PaddlePaddle Fluid model.")
+ parser.add_argument(
+ "--onnx_model", required=False, help="The path to save ONNX model.")
+ args = parser.parse_args()
+ return args
+
+
+def print_arguments(args):
+ print('----------- Configuration Arguments -----------')
+ for arg, value in sorted(vars(args).iteritems()):
+ print('%s: %s' % (arg, value))
+ print('------------------------------------------------')
+
+
+def convert(args):
  # Read the model files.
  place = fluid.CPUPlace()
  exe = fluid.Executor(place)
 
  inference_scope = fluid.core.Scope()
  with fluid.scope_guard(inference_scope):
  [inference_program, feed_target_names,
- fetch_targets] = fluid.io.load_inference_model(dirname, exe)
+ fetch_targets] = fluid.io.load_inference_model(args.fluid_model, exe)
 
  # Using blocks in programs, create nodes using:
  onnx_nodes = []
- all_inputs = []
- for block in inference_program.blocks:
- all_inputs += [
- paddle_variable_to_onnx_tensor(v, block) for v in block.vars
- if v not in ['feed', 'fetch']
- ]
 
+ # Load parameters
+ global_block = inference_program.global_block()
+ for var_name in global_block.vars:
+ var = global_block.var(var_name)
+ if var_name not in ['feed', 'fetch'] and var.persistable:
+ param = fluid.executor.fetch_var(var_name, inference_scope)
+ param_node = helper.make_node(
+ 'Constant',
+ inputs=[],
+ outputs=[var_name],
+ value=helper.make_tensor(
+ name=var_name,
+ dims=var.shape,
+ data_type=PADDLE_TO_ONNX_DTYPE[var.dtype],
+ vals=param.flatten().tolist()))
+ onnx_nodes.append(param_node)
+
+ # Create inputs
+ inputs = [
+ paddle_variable_to_onnx_tensor(v, global_block)
+ for v in feed_target_names
+ ]
+
+ # Create outputs
+ fetch_target_names = [
+ fetch_target.name for fetch_target in fetch_targets
+ ]
+ outputs = [
+ paddle_variable_to_onnx_tensor(v, global_block)
+ for v in fetch_target_names
+ ]
+
+ # Create nodes
+ for block in inference_program.blocks:
  for op in block.ops:
- if op.type in ops.PADDLE_TO_ONNX:
- # TODO(varunarora): Attributes.
- # TODO(varunarora): Use the modifier function to make the
- # transformation.
- node_proto = helper.make_node(
- ops.PADDLE_TO_ONNX[op.type][0], op.input_arg_names,
- op.output_arg_names)
+ if op.type in ops.node_maker:
+ # TODO(kuke): deal with the corner case that vars in 
+ #  different blocks have the same name
+ node_proto = ops.node_maker[op.type](
+  inputs=op.input_arg_names,
+ attrs=op.attr_names,
+ outputs=op.output_arg_names)
 
  onnx_nodes.append(node_proto)
  else:
- # Not valid to skip any op, so after all edge cases have
- # been accounted for, this exception raising to be
- # re-enabled.
- # raise NameError(op.type)
- pass
+ if op.type not in ['feed', 'fetch']:
+ raise NotImplementedError("OP[%s] is not supported in "
+ "the converter!" % op.type)
+
+ # Make graph
+ model_name = os.path.basename(args.fluid_model.strip('/')).split('.')[0]
+ onnx_graph = helper.make_graph(onnx_nodes, model_name, inputs, outputs)
 
- # Nodes, name of graph, inputs, outputs.
- if dirname[-1] == '/':
- dirname = dirname[:-1]
- graph = helper.make_graph(onnx_nodes,
- os.path.basename(dirname).split('.')[0],
- all_inputs, [])
+ # Make model
+ onnx_model = helper.make_model(onnx_graph, producer_name='PaddlePaddle')
 
- print graph
+ # Model check
+ checker.check_model(onnx_model)
 
- # TODO(varunarora): Plug in parameters.
+ # Output readable model
+ print("The converted model is:\n{}".format(onnx_model))
+
+ # Save converted model
+ if args.onnx_model is not None:
+ try:
+ with open(args.onnx_model, 'wb') as f:
+ f.write(onnx_model.SerializeToString())
+ print("Saved converted model to path: %s" % args.onnx_model)
+ except (IOError), e:
+ print("Invalid ONNX model saving path: %s" % args.onnx_model)
 
 
 if __name__ == "__main__":
- # Read arguments: path to model.
- parser = argparse.ArgumentParser()
- parser.add_argument(
- "--modeldir", required=True, help="Input PaddlePaddle model")
- args = parser.parse_args()
- convert(args.modeldir)
+ args = parse_args()
+ print_arguments(args)
+ convert(args)

fluid_onnx/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

ops.py renamed to fluid_onnx/ops.py

@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from onnx.helper import make_node
 """
 Priority of ops (uniques) to figure out support for.
 
@@ -53,8 +55,8 @@ def abs_op():
  pass
 
 
-def add_op():
- pass
+def add_op(inputs, attrs, outputs):
+ return make_node('Add', inputs=inputs, outputs=outputs, broadcast=1)
 
 
 def and_op():
@@ -222,8 +224,8 @@ def lppool_op():
  pass
 
 
-def matmul_op():
- pass
+def matmul_op(inputs, attrs, outputs):
+ return make_node('MatMul', inputs=inputs, outputs=outputs)
 
 
 def max_op():
@@ -445,10 +447,10 @@ def xor_op():
 # ONNX Ops that use multiple Paddle ops are keyed by '<op1>,<op2>' fed into the
 # modifier.
 
-PADDLE_TO_ONNX = {
+node_maker = {
  # Paddle op name : (ONNX op name, modifier)
  'abs': ('Abs', abs_op),
- 'elementwise_add': ('Add', add_op),
+ 'elementwise_add': add_op,
 
  # '': 'And', # ?
  # 'ArgMax', NEEDS ATTENTION.
@@ -496,7 +498,7 @@ def xor_op():
  '': 'MaxRoiPool',
  'mean': ('Mean', mean_op),
  '': 'Min',
- 'mul': ('Mul', mul_op),
+ 'mul': matmul_op,
  ',': 'Neg',
  '': 'Not',
  '': 'Or',

variables.py renamed to fluid_onnx/variables.py

@@ -12,16 +12,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import numpy as np
 from onnx import helper, onnx_pb2, TensorProto
 import paddle.fluid.core as core
 
 
 def paddle_variable_to_onnx_tensor(paddle_var_name, block):
  # TODO(varunarora): Need to do this only in the case of VarType.LOD_TENSOR.
  paddle_var = block.var(paddle_var_name)
- return helper.make_tensor_value_info(paddle_var_name,
- PADDLE_TO_ONNX_DTYPE[paddle_var.dtype],
- paddle_var.shape)
+ shape = paddle_onnx_shape(paddle_var.shape)
+ return helper.make_tensor_value_info(
+ paddle_var_name, PADDLE_TO_ONNX_DTYPE[paddle_var.dtype], shape)
+
+
+def paddle_onnx_shape(paddle_shape):
+ """ Convert shape info from paddle to onnx
+ """
+
+ onnx_shape = np.array(list(paddle_shape))
+ onnx_shape[onnx_shape < 0] = 0
+ return tuple(onnx_shape)
 
 
 PADDLE_TO_ONNX_DTYPE = {