From 62a7e5cf010a01601f8af4ddbe142a570752d4ed Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sun, 11 Dec 2016 11:16:13 -0800 Subject: [PATCH 1/7] Initialize Bazel WORKSPACE and a testing packages --- .gitignore | 3 +++ WORKSPACE | 8 ++++++++ third_party/protobuf_test/BUILD | 17 +++++++++++++++++ third_party/protobuf_test/README.md | 1 + third_party/protobuf_test/example.proto | 7 +++++++ third_party/protobuf_test/example_lib.cc | 6 ++++++ third_party/protobuf_test/example_lib.h | 7 +++++++ 7 files changed, 49 insertions(+) create mode 100644 WORKSPACE create mode 100644 third_party/protobuf_test/BUILD create mode 100644 third_party/protobuf_test/README.md create mode 100644 third_party/protobuf_test/example.proto create mode 100644 third_party/protobuf_test/example_lib.cc create mode 100644 third_party/protobuf_test/example_lib.h diff --git a/.gitignore b/.gitignore index 35bed0accd..1c9730a5ad 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ build/ .pydevproject Makefile .test_env/ + +*~ +bazel-* diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000000..06495690ab --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,8 @@ +git_repository( + name = "org_pubref_rules_protobuf", + remote = "https://github.com/pubref/rules_protobuf", + tag = "v0.7.1", +) + +load("@org_pubref_rules_protobuf//cpp:rules.bzl", "cpp_proto_repositories") +cpp_proto_repositories() diff --git a/third_party/protobuf_test/BUILD b/third_party/protobuf_test/BUILD new file mode 100644 index 0000000000..7c5b1c6994 --- /dev/null +++ b/third_party/protobuf_test/BUILD @@ -0,0 +1,17 @@ +licenses(["notice"]) # Apache 2.0 + +load("@org_pubref_rules_protobuf//cpp:rules.bzl", "cpp_proto_library") + +cpp_proto_library( + name = "example_proto", + protos = [ + "example.proto" + ], +) + +cc_library( + name = "example_lib", + srcs = ["example_lib.cc"], + hdrs = ["example_lib.h"], + deps = [":example_proto"], +) diff --git a/third_party/protobuf_test/README.md b/third_party/protobuf_test/README.md new file mode 100644 index 0000000000..e8bdeee6fe --- /dev/null +++ b/third_party/protobuf_test/README.md @@ -0,0 +1 @@ +This package tests that Bazel can build protobuf related rules. diff --git a/third_party/protobuf_test/example.proto b/third_party/protobuf_test/example.proto new file mode 100644 index 0000000000..57c52d3521 --- /dev/null +++ b/third_party/protobuf_test/example.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package protos; + +message Greeting { + string name = 1; +} diff --git a/third_party/protobuf_test/example_lib.cc b/third_party/protobuf_test/example_lib.cc new file mode 100644 index 0000000000..8d55ed66de --- /dev/null +++ b/third_party/protobuf_test/example_lib.cc @@ -0,0 +1,6 @@ +#include "third_party/protobuf_test/example_lib.h" +#include + +std::string get_greet(const ::protos::Greeting& who) { + return "Hello " + who.name(); +} diff --git a/third_party/protobuf_test/example_lib.h b/third_party/protobuf_test/example_lib.h new file mode 100644 index 0000000000..eaf4dd4cea --- /dev/null +++ b/third_party/protobuf_test/example_lib.h @@ -0,0 +1,7 @@ +#pragma once + +#include "third_party/protobuf_test/example.pb.h" + +#include + +std::string get_greet(const ::protos::Greeting &who); From e74ae54a594ca2a68c936b570076de186bc87ae9 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sun, 11 Dec 2016 11:37:28 -0800 Subject: [PATCH 2/7] Update development document --- .../build_and_install/docker_install_en.rst | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/doc/getstarted/build_and_install/docker_install_en.rst b/doc/getstarted/build_and_install/docker_install_en.rst index 1ab6fc6a72..a3fa8bccbf 100644 --- a/doc/getstarted/build_and_install/docker_install_en.rst +++ b/doc/getstarted/build_and_install/docker_install_en.rst @@ -104,3 +104,69 @@ container: Then we can direct our Web browser to the HTML version of source code at http://localhost:8088/paddle/ + + +Development Using Docker +------------------------ + +Develpers can work on PaddlePaddle using Docker. This allows +developers to work on different platforms -- Linux, Mac OS X, and +Windows -- in a consistent way. + +The general development workflow with Docker and Bazel is as follows: + +1. Get the source code of Paddle: + + .. code-block:: bash + + git clone --recursive https://github.com/paddlepaddle/paddle + + +1. Build a development Docker image `paddle:dev` from the source code. + This image contains all the development tools and dependencies of + PaddlePaddle. + + + .. code-block:: bash + + cd paddle + docker build -t paddle:dev -f paddle/scripts/docker/Dockerfile . + + +1. Run the image as a container and mounting local source code + directory into the container. This allows us to change the code on + the host and build it within the container. + + .. code-block:: bash + + docker run \ + -d # run the container in background mode \ + --name paddle # we can run a nginx container to serve documents \ + -p 2022:22 # so we can SSH into this container \ + -v $PWD:/paddle # mount the source code \ + -v $HOME/.cache/bazel:/root/.cache/bazel # mount Bazel cache \ + paddle:dev + +1. SSH into the container: + + .. code-block:: bash + + ssh root@localhost -p 2022 + +1. We can edit the source code in the container or on this host. Then + we can build using cmake + + .. code-block:: bash + + cd /paddle # where paddle source code has been mounted into the container + mkdir -p build + cd build + cmake .. + make -j `nproc` + + or Bazel in the container: + + .. code-block:: bash + + cd /paddle + bazel build ... From 06d780070439761da6947a3f8b3c2342cf52f8cf Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sun, 11 Dec 2016 12:02:04 -0800 Subject: [PATCH 3/7] Correct indentation in .rst file --- .../build_and_install/docker_install_en.rst | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/doc/getstarted/build_and_install/docker_install_en.rst b/doc/getstarted/build_and_install/docker_install_en.rst index a3fa8bccbf..95b6954789 100644 --- a/doc/getstarted/build_and_install/docker_install_en.rst +++ b/doc/getstarted/build_and_install/docker_install_en.rst @@ -119,7 +119,7 @@ The general development workflow with Docker and Bazel is as follows: .. code-block:: bash - git clone --recursive https://github.com/paddlepaddle/paddle + git clone --recursive https://github.com/paddlepaddle/paddle 1. Build a development Docker image `paddle:dev` from the source code. @@ -129,8 +129,8 @@ The general development workflow with Docker and Bazel is as follows: .. code-block:: bash - cd paddle - docker build -t paddle:dev -f paddle/scripts/docker/Dockerfile . + cd paddle + docker build -t paddle:dev -f paddle/scripts/docker/Dockerfile . 1. Run the image as a container and mounting local source code @@ -139,34 +139,34 @@ The general development workflow with Docker and Bazel is as follows: .. code-block:: bash - docker run \ - -d # run the container in background mode \ - --name paddle # we can run a nginx container to serve documents \ - -p 2022:22 # so we can SSH into this container \ - -v $PWD:/paddle # mount the source code \ - -v $HOME/.cache/bazel:/root/.cache/bazel # mount Bazel cache \ - paddle:dev + docker run \ + -d # run the container in background mode \ + --name paddle # we can run a nginx container to serve documents \ + -p 2022:22 # so we can SSH into this container \ + -v $PWD:/paddle # mount the source code \ + -v $HOME/.cache/bazel:/root/.cache/bazel # mount Bazel cache \ + paddle:dev 1. SSH into the container: .. code-block:: bash - ssh root@localhost -p 2022 + ssh root@localhost -p 2022 1. We can edit the source code in the container or on this host. Then we can build using cmake .. code-block:: bash - cd /paddle # where paddle source code has been mounted into the container - mkdir -p build - cd build - cmake .. - make -j `nproc` + cd /paddle # where paddle source code has been mounted into the container + mkdir -p build + cd build + cmake .. + make -j `nproc` or Bazel in the container: .. code-block:: bash - cd /paddle - bazel build ... + cd /paddle + bazel build ... From 8013d17dfae690d6b0408fb6b0d8e4eceee30f2d Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sun, 11 Dec 2016 12:03:05 -0800 Subject: [PATCH 4/7] Correct enumeration in lists --- doc/getstarted/build_and_install/docker_install_en.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/getstarted/build_and_install/docker_install_en.rst b/doc/getstarted/build_and_install/docker_install_en.rst index 95b6954789..f6f1bbab42 100644 --- a/doc/getstarted/build_and_install/docker_install_en.rst +++ b/doc/getstarted/build_and_install/docker_install_en.rst @@ -122,7 +122,7 @@ The general development workflow with Docker and Bazel is as follows: git clone --recursive https://github.com/paddlepaddle/paddle -1. Build a development Docker image `paddle:dev` from the source code. +2. Build a development Docker image `paddle:dev` from the source code. This image contains all the development tools and dependencies of PaddlePaddle. @@ -133,7 +133,7 @@ The general development workflow with Docker and Bazel is as follows: docker build -t paddle:dev -f paddle/scripts/docker/Dockerfile . -1. Run the image as a container and mounting local source code +3. Run the image as a container and mounting local source code directory into the container. This allows us to change the code on the host and build it within the container. @@ -147,13 +147,13 @@ The general development workflow with Docker and Bazel is as follows: -v $HOME/.cache/bazel:/root/.cache/bazel # mount Bazel cache \ paddle:dev -1. SSH into the container: +4. SSH into the container: .. code-block:: bash ssh root@localhost -p 2022 -1. We can edit the source code in the container or on this host. Then +5. We can edit the source code in the container or on this host. Then we can build using cmake .. code-block:: bash From 9b94773a7f546f5d1a825c3a6f07420a5ced6404 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sun, 11 Dec 2016 12:39:02 -0800 Subject: [PATCH 5/7] Add gtest --- WORKSPACE | 14 ++++++++++++++ .../build_and_install/docker_install_en.rst | 5 +++-- third_party/gtest.BUILD | 14 ++++++++++++++ third_party/protobuf_test/BUILD | 11 +++++++++++ third_party/protobuf_test/example.proto | 2 +- third_party/protobuf_test/example_lib.cc | 9 +++++++-- third_party/protobuf_test/example_lib.h | 8 +++++++- third_party/protobuf_test/example_lib_test.cc | 15 +++++++++++++++ 8 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 third_party/gtest.BUILD create mode 100644 third_party/protobuf_test/example_lib_test.cc diff --git a/WORKSPACE b/WORKSPACE index 06495690ab..38e1628d11 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,8 +1,22 @@ +# External dependency to grpc-enabled Google-styleprotobuf bulding +# rules. This method comes from +# https://github.com/pubref/rules_protobuf#usage. git_repository( name = "org_pubref_rules_protobuf", remote = "https://github.com/pubref/rules_protobuf", tag = "v0.7.1", ) +# External dependency to gtest 1.7.0. This method comes from +# https://www.bazel.io/versions/master/docs/tutorial/cpp.html. +new_http_archive( + name = "gtest", + url = "https://github.com/google/googletest/archive/release-1.7.0.zip", + sha256 = "b58cb7547a28b2c718d1e38aee18a3659c9e3ff52440297e965f5edffe34b6d0", + build_file = "third_party/gtest.BUILD", + strip_prefix = "googletest-release-1.7.0", +) + + load("@org_pubref_rules_protobuf//cpp:rules.bzl", "cpp_proto_repositories") cpp_proto_repositories() diff --git a/doc/getstarted/build_and_install/docker_install_en.rst b/doc/getstarted/build_and_install/docker_install_en.rst index f6f1bbab42..feb027ccbb 100644 --- a/doc/getstarted/build_and_install/docker_install_en.rst +++ b/doc/getstarted/build_and_install/docker_install_en.rst @@ -161,12 +161,13 @@ The general development workflow with Docker and Bazel is as follows: cd /paddle # where paddle source code has been mounted into the container mkdir -p build cd build - cmake .. + cmake -DWITH_TESTING=ON .. make -j `nproc` + CTEST_OUTPUT_ON_FAILURE=1 ctest or Bazel in the container: .. code-block:: bash cd /paddle - bazel build ... + bazel test ... diff --git a/third_party/gtest.BUILD b/third_party/gtest.BUILD new file mode 100644 index 0000000000..3e68a1d879 --- /dev/null +++ b/third_party/gtest.BUILD @@ -0,0 +1,14 @@ +cc_library( + name = "main", + srcs = glob( + ["src/*.cc"], + exclude = ["src/gtest-all.cc"] + ), + hdrs = glob([ + "include/**/*.h", + "src/*.h" + ]), + copts = ["-Iexternal/gtest/include"], + linkopts = ["-pthread"], + visibility = ["//visibility:public"], +) diff --git a/third_party/protobuf_test/BUILD b/third_party/protobuf_test/BUILD index 7c5b1c6994..6208b87082 100644 --- a/third_party/protobuf_test/BUILD +++ b/third_party/protobuf_test/BUILD @@ -7,6 +7,7 @@ cpp_proto_library( protos = [ "example.proto" ], + with_grpc = True, ) cc_library( @@ -15,3 +16,13 @@ cc_library( hdrs = ["example_lib.h"], deps = [":example_proto"], ) + +cc_test( + name = "example_lib_test", + srcs = ["example_lib_test.cc"], + copts = ["-Iexternal/gtest/include"], + deps =[ + "@gtest//:main", + ":example_lib", + ], +) diff --git a/third_party/protobuf_test/example.proto b/third_party/protobuf_test/example.proto index 57c52d3521..6a7eada9c1 100644 --- a/third_party/protobuf_test/example.proto +++ b/third_party/protobuf_test/example.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package protos; +package third_party.protobuf_test; message Greeting { string name = 1; diff --git a/third_party/protobuf_test/example_lib.cc b/third_party/protobuf_test/example_lib.cc index 8d55ed66de..56341a0124 100644 --- a/third_party/protobuf_test/example_lib.cc +++ b/third_party/protobuf_test/example_lib.cc @@ -1,6 +1,11 @@ #include "third_party/protobuf_test/example_lib.h" -#include -std::string get_greet(const ::protos::Greeting& who) { +namespace third_party { +namespace protobuf_test { + +std::string get_greet(const Greeting& who) { return "Hello " + who.name(); } + +} // namespace protobuf_test +} // namespace thrid_party diff --git a/third_party/protobuf_test/example_lib.h b/third_party/protobuf_test/example_lib.h index eaf4dd4cea..516326e812 100644 --- a/third_party/protobuf_test/example_lib.h +++ b/third_party/protobuf_test/example_lib.h @@ -4,4 +4,10 @@ #include -std::string get_greet(const ::protos::Greeting &who); +namespace third_party { +namespace protobuf_test { + +std::string get_greet(const Greeting &who); + +} // namespace protobuf_test +} // namespace third_party diff --git a/third_party/protobuf_test/example_lib_test.cc b/third_party/protobuf_test/example_lib_test.cc new file mode 100644 index 0000000000..6229f56e60 --- /dev/null +++ b/third_party/protobuf_test/example_lib_test.cc @@ -0,0 +1,15 @@ +#include "third_party/protobuf_test/example_lib.h" + +#include "gtest/gtest.h" + +namespace third_party { +namespace protobuf_test { + +TEST(ProtobufTest, GetGreet) { + Greeting g; + g.set_name("Paddle"); + EXPECT_EQ("Hello Paddle", get_greet(g)); +} + +} // namespace protobuf_test +} // namespace third_party From f722ee21acdc190f456a64173647c38bffabbdb8 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Mon, 12 Dec 2016 04:19:34 +0000 Subject: [PATCH 6/7] Do not use https://github.com/pubref/rules_protobuf, but use https://github.com/google/protobuf/ as external dependency for protobuf --- .pre-commit-config.yaml | 3 ++- WORKSPACE | 17 ++++++----------- third_party/protobuf_test/BUILD | 11 +++++------ 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 942669c41f..641ac583b5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,8 @@ - repo: https://github.com/reyoung/mirrors-yapf.git sha: v0.13.2 hooks: - - id: yapf + - id: yapf + files: (.*\.(py|bzl)|BUILD|.*\.BUILD)$ # Bazel BUILD files follow Python syntax. - repo: https://github.com/pre-commit/pre-commit-hooks sha: 7539d8bd1a00a3c1bfd34cdb606d3a6372e83469 hooks: diff --git a/WORKSPACE b/WORKSPACE index 38e1628d11..d6ae2af8eb 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,10 +1,9 @@ -# External dependency to grpc-enabled Google-styleprotobuf bulding -# rules. This method comes from -# https://github.com/pubref/rules_protobuf#usage. -git_repository( - name = "org_pubref_rules_protobuf", - remote = "https://github.com/pubref/rules_protobuf", - tag = "v0.7.1", +# External dependency to Google protobuf. +http_archive( + name = "protobuf", + url = "http://github.com/google/protobuf/archive/v3.1.0.tar.gz", + sha256 = "0a0ae63cbffc274efb573bdde9a253e3f32e458c41261df51c5dbc5ad541e8f7", + strip_prefix = "protobuf-3.1.0", ) # External dependency to gtest 1.7.0. This method comes from @@ -16,7 +15,3 @@ new_http_archive( build_file = "third_party/gtest.BUILD", strip_prefix = "googletest-release-1.7.0", ) - - -load("@org_pubref_rules_protobuf//cpp:rules.bzl", "cpp_proto_repositories") -cpp_proto_repositories() diff --git a/third_party/protobuf_test/BUILD b/third_party/protobuf_test/BUILD index 6208b87082..46f769da5f 100644 --- a/third_party/protobuf_test/BUILD +++ b/third_party/protobuf_test/BUILD @@ -1,13 +1,12 @@ licenses(["notice"]) # Apache 2.0 -load("@org_pubref_rules_protobuf//cpp:rules.bzl", "cpp_proto_library") +load("@protobuf//:protobuf.bzl", "cc_proto_library") -cpp_proto_library( +cc_proto_library( name = "example_proto", - protos = [ - "example.proto" - ], - with_grpc = True, + srcs = ["example.proto"], + protoc = "@protobuf//:protoc", + default_runtime = "@protobuf//:protobuf", ) cc_library( From 7e7b2bb7138a481c2004ae9e27ce15cc5e3c2adc Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sun, 11 Dec 2016 20:26:22 -0800 Subject: [PATCH 7/7] Make Git pre-commit hook auto-format WORKSPACE as well. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 641ac583b5..b9902a863d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ sha: v0.13.2 hooks: - id: yapf - files: (.*\.(py|bzl)|BUILD|.*\.BUILD)$ # Bazel BUILD files follow Python syntax. + files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$ # Bazel BUILD files follow Python syntax. - repo: https://github.com/pre-commit/pre-commit-hooks sha: 7539d8bd1a00a3c1bfd34cdb606d3a6372e83469 hooks: