Skip to content

Commit e2ea1f4

Browse files
reyoungreyoung
committed
Generate python protobufs for paddle.v2.framework
Python should be able to manipulate Protobuf message because: 1. Python's `create_op_creation_methods` take the `OpProto` array to generate all `op_creation_methods` in RunTime. 2. All `op_creation_methods` will create an `OpDesc` and pass it to Paddle C++ method `CreateOp` and return the Op handle. Here is the list of what is added in this commit: * Add `protobuf_generate_python` if it is not defined. * Before cmake 3.4, `protobuf_generate_python` is not defined. Just copy the implementation of that function in `protobuf.cmake` * Add `py_proto_compile` function in `cmake/generic.cmake`. * It follows bazel's API interface. * https://github.com/pubref/rules_protobuf#rules * Add an empty package named `paddle.v2.framework`, all python code of `paddle::framework` will be in that package. * Generate protobuf's python module `__init__.py` by `touch` while compiling. * Change setup.py.in, make `paddle.v2.framework.proto` uses the generated protobuf pythons.
1 parent 4d899a8 commit e2ea1f4

File tree

8 files changed

+109
-4
lines changed

8 files changed

+109
-4
lines changed

‎cmake/external/protobuf.cmake‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,65 @@ INCLUDE(ExternalProject)
1717
FIND_PACKAGE(Protobuf QUIET)
1818
SET(PROTOBUF_FOUND "OFF")
1919

20+
if(NOT COMMAND protobuf_generate_python) # before cmake 3.4, protobuf_genrerate_python is not defined.
21+
function(protobuf_generate_python SRCS)
22+
# shameless copy from https://github.com/Kitware/CMake/blob/master/Modules/FindProtobuf.cmake
23+
if(NOT ARGN)
24+
message(SEND_ERROR "Error: PROTOBUF_GENERATE_PYTHON() called without any proto files")
25+
return()
26+
endif()
27+
28+
if(PROTOBUF_GENERATE_CPP_APPEND_PATH)
29+
# Create an include path for each file specified
30+
foreach(FIL ${ARGN})
31+
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
32+
get_filename_component(ABS_PATH ${ABS_FIL} PATH)
33+
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
34+
if(${_contains_already} EQUAL -1)
35+
list(APPEND _protobuf_include_path -I ${ABS_PATH})
36+
endif()
37+
endforeach()
38+
else()
39+
set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
40+
endif()
41+
42+
if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS)
43+
set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}")
44+
endif()
45+
46+
if(DEFINED Protobuf_IMPORT_DIRS)
47+
foreach(DIR ${Protobuf_IMPORT_DIRS})
48+
get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
49+
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
50+
if(${_contains_already} EQUAL -1)
51+
list(APPEND _protobuf_include_path -I ${ABS_PATH})
52+
endif()
53+
endforeach()
54+
endif()
55+
56+
set(${SRCS})
57+
foreach(FIL ${ARGN})
58+
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
59+
get_filename_component(FIL_WE ${FIL} NAME_WE)
60+
if(NOT PROTOBUF_GENERATE_CPP_APPEND_PATH)
61+
get_filename_component(FIL_DIR ${FIL} DIRECTORY)
62+
if(FIL_DIR)
63+
set(FIL_WE "${FIL_DIR}/${FIL_WE}")
64+
endif()
65+
endif()
66+
67+
list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py")
68+
add_custom_command(
69+
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py"
70+
COMMAND ${Protobuf_PROTOC_EXECUTABLE} --python_out ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
71+
DEPENDS ${ABS_FIL} ${Protobuf_PROTOC_EXECUTABLE}
72+
COMMENT "Running Python protocol buffer compiler on ${FIL}"
73+
VERBATIM )
74+
endforeach()
75+
76+
set(${SRCS} ${${SRCS}} PARENT_SCOPE)
77+
endfunction()
78+
endif()
2079

2180
# Print and set the protobuf library information,
2281
# finish this cmake process and exit from this file.

‎cmake/generic.cmake‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,3 +335,12 @@ function(proto_library TARGET_NAME)
335335
protobuf_generate_cpp(proto_srcs proto_hdrs ${proto_library_SRCS})
336336
cc_library(${TARGET_NAME} SRCS ${proto_srcs} DEPS ${proto_library_DEPS} protobuf)
337337
endfunction()
338+
339+
function(py_proto_compile TARGET_NAME)
340+
set(oneValueArgs "")
341+
set(multiValueArgs SRCS)
342+
cmake_parse_arguments(py_proto_compile "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
343+
set(py_srcs)
344+
protobuf_generate_python(py_srcs ${py_proto_compile_SRCS})
345+
add_custom_target(${TARGET_NAME} ALL DEPENDS ${py_srcs})
346+
endfunction()

‎paddle/framework/CMakeLists.txt‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ cc_test(enforce_test SRCS enforce_test.cc)
99
proto_library(attr_type SRCS attr_type.proto)
1010
proto_library(op_proto SRCS op_proto.proto DEPS attr_type)
1111
cc_test(op_proto_test SRCS op_proto_test.cc DEPS op_proto protobuf)
12-
1312
proto_library(op_desc SRCS op_desc.proto DEPS attr_type)
1413
cc_test(op_desc_test SRCS op_desc_test.cc DEPS op_desc protobuf)
14+
py_proto_compile(framework_py_proto SRCS attr_type.proto op_proto.proto op_desc.proto)
15+
# Generate an empty __init__.py to make framework_py_proto as a valid python module.
16+
add_custom_target(framework_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
17+
add_dependencies(framework_py_proto framework_py_proto_init)

‎python/CMakeLists.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in
2929
add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp
3030
COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
3131
COMMAND ${CMAKE_COMMAND} -E touch ${OUTPUT_DIR}/.timestamp
32-
DEPENDS gen_proto_py ${PY_FILES} ${external_project_dependencies} ${COPY_PADDLE_MASTER})
32+
DEPENDS gen_proto_py framework_py_proto ${PY_FILES} ${external_project_dependencies} ${COPY_PADDLE_MASTER})
3333

3434
add_custom_target(paddle_python ALL DEPENDS
3535
${OUTPUT_DIR}/.timestamp)
@@ -43,6 +43,7 @@ if (WITH_TESTING)
4343
add_subdirectory(paddle/v2/tests)
4444
add_subdirectory(paddle/v2/reader/tests)
4545
add_subdirectory(paddle/v2/plot/tests)
46+
add_subdirectory(paddle/v2/framework/tests)
4647
endif()
4748
endif()
4849
install(DIRECTORY ${PADDLE_PYTHON_PACKAGE_DIR}

‎python/paddle/v2/framework/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
__all__ = ['proto']

‎python/paddle/v2/framework/tests/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
add_python_test(test_framework test_protobuf.py)

‎python/paddle/v2/framework/tests/test_protobuf.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
import paddle.v2.framework.proto.op_proto_pb2
2+
import paddle.v2.framework.proto.attr_type_pb2
3+
import unittest
4+
5+
6+
class TestFrameworkProto(unittest.TestCase):
7+
def test_all(self):
8+
op_proto_lib = paddle.v2.framework.proto.op_proto_pb2
9+
attr_type_lib = paddle.v2.framework.proto.attr_type_pb2
10+
op_proto = op_proto_lib.OpProto()
11+
ipt0 = op_proto.inputs.add()
12+
ipt0.name = "a"
13+
ipt0.comment = "the input of cosine op"
14+
ipt1 = op_proto.inputs.add()
15+
ipt1.name = "b"
16+
ipt1.comment = "the other input of cosine op"
17+
opt = op_proto.outputs.add()
18+
opt.name = "output"
19+
opt.comment = "the output of cosine op"
20+
op_proto.comment = "cosine op, output = scale*cos(a, b)"
21+
attr = op_proto.attrs.add()
22+
attr.name = "scale"
23+
attr.comment = "scale of cosine op"
24+
attr.type = attr_type_lib.FLOAT
25+
op_proto.type = "cos"
26+
self.assertTrue(op_proto.IsInitialized())

‎python/setup.py.in‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ packages=['paddle',
99
'paddle.v2.dataset',
1010
'paddle.v2.reader',
1111
'paddle.v2.master',
12-
'paddle.v2.plot']
12+
'paddle.v2.plot',
13+
'paddle.v2.framework',
14+
'paddle.v2.framework.proto']
1315

1416
setup_requires=["requests",
1517
"numpy",
@@ -29,6 +31,9 @@ setup(name='paddle',
2931
packages=packages,
3032
package_data={'paddle.v2.master': ['${paddle_master_LIB_NAME}'], },
3133
package_dir={
32-
'': '${CMAKE_CURRENT_SOURCE_DIR}'
34+
'': '${CMAKE_CURRENT_SOURCE_DIR}',
35+
# The paddle.v2.framework.proto will be generated while compiling.
36+
# So that package points to other directory.
37+
'paddle.v2.framework.proto': '${CMAKE_BINARY_DIR}/paddle/framework'
3338
},
3439
)

0 commit comments

Comments
 (0)