From ab4eeea61ed94e58d2de001958a9f823070b31f8 Mon Sep 17 00:00:00 2001 From: Nicolas Oliver Date: Mon, 10 Jul 2017 07:28:18 -0700 Subject: [PATCH] travis: improve build process and expand build matrix Make a build heriarchy to reduce images size. Add Android Things builds to build matrix. Drop gcc-4 builds, use gcc-5 and gcc-6 instead. Add SonarQube static analysis scans. Signed-off-by: Nicolas Oliver --- .travis.yml | 38 ++++++-- Dockerfile | 118 ------------------------- bacnet/libbacnet.pc | 9 -- docker-compose.yaml | 104 +++++++++++++++------- docker/Dockerfile.android | 67 +++++++++++++++ docker/Dockerfile.base | 68 +++++++++++++++ docker/Dockerfile.java | 15 ++++ docker/Dockerfile.node | 17 ++++ docker/Dockerfile.python | 16 ++++ docker/Dockerfile.sonar | 41 +++++++++ docs/building.md | 176 +++++++++++++++++++++++++++++++++++++- scripts/build-android.sh | 125 +++++++++++++++++++++++++++ scripts/run-cmake.sh | 20 +++++ scripts/sonar-scan.sh | 76 ++++++++++++++++ 14 files changed, 722 insertions(+), 168 deletions(-) delete mode 100644 Dockerfile delete mode 100644 bacnet/libbacnet.pc create mode 100644 docker/Dockerfile.android create mode 100644 docker/Dockerfile.base create mode 100644 docker/Dockerfile.java create mode 100644 docker/Dockerfile.node create mode 100644 docker/Dockerfile.python create mode 100644 docker/Dockerfile.sonar create mode 100755 scripts/build-android.sh create mode 100755 scripts/run-cmake.sh create mode 100755 scripts/sonar-scan.sh diff --git a/.travis.yml b/.travis.yml index 67742b89..b713b563 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,11 @@ +dist: trusty sudo: required language: cpp compiler: - clang - - gcc-4.8 - gcc - - gcc-6 env: - TARGET=doc @@ -15,6 +14,33 @@ env: - TARGET=node4 - TARGET=node5 - TARGET=node6 + - TARGET=android + - TARGET=sonar-scan + +matrix: + exclude: + - compiler: clang + env: TARGET=java + - compiler: gcc + env: TARGET=android + - compiler: clang + env: TARGET=sonar-scan + include: + - compiler: gcc + env: GCC_VERSION=6 TARGET=doc + - compiler: gcc + env: GCC_VERSION=6 TARGET=python + - compiler: gcc + env: GCC_VERSION=6 TARGET=java + - compiler: gcc + env: GCC_VERSION=6 TARGET=node4 + - compiler: gcc + env: GCC_VERSION=6 TARGET=node5 + - compiler: gcc + env: GCC_VERSION=6 TARGET=node6 + allow_failures: + - compiler: gcc + env: TARGET=sonar-scan services: - docker @@ -26,12 +52,12 @@ before_install: - sudo mv docker-compose /usr/local/bin before_script: - - if [ "$CC" == "gcc-4.8" ]; then export CC=gcc-4.8; fi - - if [ "$CXX" == "g++-4.8" ]; then export CXX=g++-4.8; fi - - if [ "$CC" == "gcc-6" ]; then export CC=gcc-6; fi - - if [ "$CXX" == "g++-6" ]; then export CXX=g++-6; fi + - if [ "$CC" == "gcc" ]; then export CC=gcc-${GCC_VERSION:-5}; fi + - if [ "$CXX" == "g++" ]; then export CXX=g++-${GCC_VERSION:-5}; fi - if [ "$CC" == "clang" ]; then export CC=clang-3.8; fi - if [ "$CXX" == "clang++" ]; then export CXX=clang++-3.8; fi + - docker-compose build base + - if [ "$TARGET" == "android" ]; then docker-compose build java; fi - docker-compose build ${TARGET} script: diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 6f39c302..00000000 --- a/Dockerfile +++ /dev/null @@ -1,118 +0,0 @@ -FROM ubuntu:16.04 - -ENV DEBIAN_FRONTEND noninteractive - -# Prepare apt-get -RUN apt-get update && apt-get -y --no-install-recommends install software-properties-common - -# Add Mraa Repository -RUN add-apt-repository ppa:mraa/mraa && \ - # Update apt-get - apt-get -y update && \ - # Install apt-utils - apt-get -y --no-install-recommends install apt-utils && \ - # Main Build Dependencies - apt-get -y --no-install-recommends install git build-essential cmake swig clang-3.8 gcc-4.8 g++-4.8 libpthread-stubs0-dev pkg-config wget unzip \ - # Mraa Build Dependencies - libmraa1 libmraa-dev mraa-tools python-mraa python3-mraa libmraa-java \ - # Docs Build Dependencies - python-sphinx doxygen graphviz \ - # Python Build Dependencies - python python-dev python3 python3-dev python-pip python3-pip \ - # Java Build Dependencies - default-jre default-jdk \ - # Sensor Specific Build Dependencies - libjpeg-dev - -# Configure Compiler Environment -ARG CC -ARG CXX -ENV CC $CC -ENV CXX $CXX - -# Install GCC6 too -RUN add-apt-repository ppa:ubuntu-toolchain-r/test && \ - apt-get update && \ - apt-get -y --no-install-recommends install gcc-6 g++-6 - -# Configure Java Home -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ - -# Install libbacnet 0.3.12 -RUN wget https://downloads.sourceforge.net/project/bacnet/bacnet-stack/bacnet-stack-0.8.3/bacnet-stack-0.8.3.zip && \ - unzip -qq bacnet-stack-0.8.3.zip && cd bacnet-stack-0.8.3 && \ - BACDL_DEFINE=-DBACDL_MSTP=1 MAKE_DEFINE=-fPIC make clean all - -# Install libmodbus 3.1.4 -RUN wget http://libmodbus.org/releases/libmodbus-3.1.4.tar.gz && \ - tar xf libmodbus-3.1.4.tar.gz && cd libmodbus-3.1.4 && \ - ./configure && make -j8 && make install - -# Install openzwave -RUN apt-get update && apt-get -y --no-install-recommends install libudev-dev && \ - git clone --depth 1 https://github.com/OpenZWave/open-zwave.git && cd open-zwave && make -j8 install && \ - echo "/usr/local/lib64" >> /etc/ld.so.conf.d/openzwave.conf - -# Using a custom SWIG version -# RUN wget https://downloads.sourceforge.net/project/swig/swig/swig-3.0.10/swig-3.0.10.tar.gz && \ -## tar xf swig-3.0.10.tar.gz && cd swig-3.0.10 && \ -## apt-get update && apt-get -y --no-install-recommends install libpcre3 libpcre3-dev && \ -## ./configure --prefix=/usr/ && make && make install && cd .. - -# Node.js Build Dependencies -RUN wget -q -O - https://raw.githubusercontent.com/creationix/nvm/v0.33.2/install.sh | bash -ARG NODE_VERSION -ENV NVM_DIR /root/.nvm -RUN . $NVM_DIR/nvm.sh && nvm install $NODE_VERSION && nvm use $NODE_VERSION && \ - npm install -g node-gyp && node-gyp install - -# Install Test Dependencies -RUN pip install chardet && pip3 install chardet - -# Set Workdir -WORKDIR /usr/src/app - -# Configure Build Arguments -ARG BUILDDOC -ARG BUILDCPP -ARG BUILDFTI -ARG BUILDSWIGPYTHON -ARG BUILDSWIGNODE -ARG BUILDSWIGJAVA -ARG BUILDEXAMPLES -ARG IPK -ARG RPM -ARG NPM -ARG BUILDTESTS -ARG WERROR - -# Copy sources -COPY . . - -# Temp solution for libbacnet install -COPY bacnet/libbacnet.pc /usr/lib/pkgconfig/libbacnet.pc -RUN cp /bacnet-stack-0.8.3/lib/libbacnet.a /usr/lib/libbacnet.a -RUN mkdir /usr/include/bacnet && cp /bacnet-stack-0.8.3/include/* /usr/include/bacnet/ - -# Change Workdir to build directory -WORKDIR /usr/src/app/build - -# Run cmake -RUN . $NVM_DIR/nvm.sh && cmake \ - -DSWIG_EXECUTABLE=/usr/bin/swig \ -# -DSWIG_DIR:PATH=/usr/share/swig/3.0.10/ \ - -DBUILDDOC=$BUILDDOC \ - -DBUILDCPP=$BUILDCPP \ - -DBUILDFTI=$BUILDFTI \ - -DBUILDSWIGPYTHON=$BUILDSWIGPYTHON \ - -DBUILDSWIGNODE=$BUILDSWIGNODE \ - -DBUILDSWIGJAVA=$BUILDSWIGJAVA \ - -DBUILDEXAMPLES=$BUILDEXAMPLES \ - -DIPK=$IPK \ - -DRPM=$RPM \ - -DNPM=$NPM \ - -DBUILDTESTS=$BUILDTESTS \ - -DWERROR=$WERROR \ - .. - -CMD make diff --git a/bacnet/libbacnet.pc b/bacnet/libbacnet.pc deleted file mode 100644 index c5dd528d..00000000 --- a/bacnet/libbacnet.pc +++ /dev/null @@ -1,9 +0,0 @@ -prefix=/usr -exec_prefix=${prefix} -libdir=${exec_prefix}/lib -includedir=${prefix}/include -Name: bacnet -Description: BACnet library -Version: 0.8.3 -Libs: -L${libdir} -lbacnet -Cflags: -I${includedir}/bacnet diff --git a/docker-compose.yaml b/docker-compose.yaml index e88edcf2..510e9ec1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -2,70 +2,112 @@ version: '2.1' services: - main: + base: + image: upm-base build: context: . - dockerfile: Dockerfile + dockerfile: docker/Dockerfile.base args: - http_proxy - https_proxy - no_proxy - - BUILDDOC=${BUILDDOC:-OFF} - - BUILDCPP=${BUILDCPP:-ON} - - BUILDFTI=${BUILDFTI:-ON} - - BUILDSWIGPYTHON=${BUILDSWIGPYTHON:-OFF} - - BUILDSWIGJAVA=${BUILDSWIGJAVA:-OFF} - - BUILDSWIGNODE=${BUILDSWIGNODE:-OFF} - - BUILDEXAMPLES=${BUILDEXAMPLE:-ON} - - IPK=${IPK:-OFF} - - RPM=${RPM:-OFF} - - NPM=${NPM:-OFF} - - BUILDTESTS=${BUILDTESTS:-ON} - - CC=${CC:-clang-3.8} - - CXX=${CXX:-clang++-3.8} - - NODE_VERSION=${NODE_VERSION:-v4.4.7} - - WERROR=${WERROR:-ON} environment: - http_proxy - https_proxy - no_proxy + - BUILDDOC=${BUILDDOC:-OFF} + - BUILDCPP=${BUILDCPP:-ON} + - BUILDFTI=${BUILDFTI:-ON} + - BUILDSWIGPYTHON=${BUILDSWIGPYTHON:-OFF} + - BUILDSWIGJAVA=${BUILDSWIGJAVA:-OFF} + - BUILDSWIGNODE=${BUILDSWIGNODE:-OFF} + - BUILDEXAMPLES=${BUILDEXAMPLE:-ON} + - IPK=${IPK:-OFF} + - RPM=${RPM:-OFF} + - NPM=${NPM:-OFF} + - BUILDTESTS=${BUILDTESTS:-ON} + - CC=${CC:-clang-3.8} + - CXX=${CXX:-clang++-3.8} + - NODE_VERSION=${NODE_VERSION:-v4.4.7} + - WERROR=${WERROR:-ON} + volumes: + - .:${UPM_SRC_DIR:-/usr/src/app} doc: - extends: main + extends: base + environment: + - BUILDDOC=ON + command: bash -c "./scripts/run-cmake.sh && make -j8 -Cbuild doc" + + sonar-scan: + extends: base + image: upm-sonar build: + dockerfile: docker/Dockerfile.sonar args: - - BUILDDOC=ON - command: bash -c "make -j8 doc" + - NODE_VERSION=v5.12.0 + environment: + - BUILDSWIGPYTHON=ON + - BUILDSWIGNODE=ON + - BUILDSWIGJAVA=ON + - SONAR_TOKEN + - SONAR_ORG + - SONAR_PROJ_KEY + - TRAVIS_BRANCH + - TRAVIS_PULL_REQUEST + - TRAVIS_REPO_SLUG + - TRAVIS_PULL_REQUEST_SLUG + - GITHUB_TOKEN + command: bash -c "./scripts/run-cmake.sh && cd build && ../scripts/sonar-scan.sh" python: - extends: main + extends: base + image: upm-python build: - args: - - BUILDSWIGPYTHON=ON - command: bash -c "make -j8 && make -j8 install && ldconfig && ctest --output-on-failure" + dockerfile: docker/Dockerfile.python + environment: + - BUILDSWIGPYTHON=ON + command: bash -c "./scripts/run-cmake.sh && cd build && make -j8 && make -j8 install && ldconfig && ctest --output-on-failure" java: - extends: main + extends: base + image: upm-java build: - args: - - BUILDSWIGJAVA=ON - command: bash -c "make -j8 && make -j8 install && ldconfig && ctest --output-on-failure" + dockerfile: docker/Dockerfile.java + environment: + - BUILDSWIGJAVA=ON + command: bash -c "./scripts/run-cmake.sh && cd build && make -j8 && make -j8 install && ldconfig && ctest --output-on-failure" + + android: + extends: java + image: upm-android + build: + dockerfile: docker/Dockerfile.android + environment: + - BUILDTESTS=OFF + command: bash -c "./scripts/build-android.sh" node4: - extends: main + extends: base + image: upm-node4 build: + dockerfile: docker/Dockerfile.node args: - - BUILDSWIGNODE=ON - NODE_VERSION=v4.4.7 - command: bash -c "make -j8 && make -j8 install && ldconfig && ctest --output-on-failure -E examplenames_js" + environment: + - BUILDSWIGNODE=ON + command: bash -c "./scripts/run-cmake.sh && cd build && make -j8 && make -j8 install && ldconfig && ctest --output-on-failure -E examplenames_js" node5: extends: node4 + image: upm-node5 build: args: - NODE_VERSION=v5.12.0 + node6: extends: node4 + image: upm-node6 build: args: - NODE_VERSION=v6.11.0 diff --git a/docker/Dockerfile.android b/docker/Dockerfile.android new file mode 100644 index 00000000..e0864c75 --- /dev/null +++ b/docker/Dockerfile.android @@ -0,0 +1,67 @@ +FROM upm-java + +WORKDIR /opt + +# Install CMake 3.6.2 +RUN wget -q https://cmake.org/files/v3.6/cmake-3.6.2-Linux-x86_64.sh && \ + mkdir /opt/cmake && \ + sh /opt/cmake-3.6.2-Linux-x86_64.sh --prefix=/opt/cmake --skip-license && \ + ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake && \ + cmake --version && \ + rm -fr /opt/cmake-3.6.2-Linux-x86_64.sh + +# Android NDK build Dependencies +RUN wget -q https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip && \ + unzip -qq android-ndk-r14b-linux-x86_64.zip && \ + rm -fr android-ndk-r14b-linux-x86_64.zip +ENV NDK_HOME /opt/android-ndk-r14b +ENV NDK_MODULE_PATH /opt/android-ndk-r14b + +# Android Things library +RUN wget -q https://github.com/androidthings/native-libandroidthings/archive/0.4.1-devpreview.tar.gz && \ + tar xf 0.4.1-devpreview.tar.gz && \ + rm -fr /opt/0.4.1-devpreview.tar.gz +ENV ANDROIDTHINGS_NATIVE_LIB /opt/native-libandroidthings-0.4.1-devpreview + +# Build Mraa For Android Things +RUN git clone --depth 1 https://github.com/intel-iot-devkit/mraa.git && \ + cd mraa && \ + cmake \ + -DANDROID_COMPILER_FLAGS_CXX='-std=c++11' \ + -DANDROID_PIE=1 \ + -DANDROID_PLATFORM=android-24 \ + -DANDROID_STL_FORCE_FEATURES=on \ + -DANDROID_STL=c++_shared \ + -DANDROID_TOOLCHAIN_NAME=x86-i686 \ + -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY \ + -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \ + -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=BOTH \ + -DCMAKE_FIND_ROOT_PATH=$NDK_HOME/platforms/android-24/arch-x86/ \ + -DCMAKE_MODULE_PATH=$ANDROIDTHINGS_NATIVE_LIB \ + -DCMAKE_TOOLCHAIN_FILE=$NDK_HOME/build/cmake/android.toolchain.cmake \ + -DCMAKE_INSTALL_PREFIX=/opt/mraa/install \ + -DJAVA_AWT_INCLUDE_PATH=$JAVA_HOME/include \ + -DJAVA_AWT_LIBRARY=$JAVA_HOME/jre/lib/amd64/libjawt.so \ + -DJAVA_INCLUDE_PATH=$JAVA_HOME/include \ + -DJAVA_INCLUDE_PATH2=$JAVA_HOME/include/linux \ + -DJAVA_JVM_LIBRARY=$JAVA_HOME/jre/lib/amd64/server/libjvm.so \ + -DBUILDARCH=PERIPHERALMAN \ + -DJSONPLAT=OFF \ + -DBUILDDOC=OFF \ + -DBUILDSWIG=ON \ + -DBUILDSWIGPYTHON=OFF \ + -DBUILDSWIGNODE=OFF \ + -DBUILDSWIGJAVA=ON \ + -DENABLEEXAMPLES=OFF \ + -DINSTALLGPIOTOOL=OFF \ + -DINSTALLTOOLS=OFF \ + -DBUILDTESTS=OFF \ + -H. \ + -Bbuild && \ + make -j8 -Cbuild install +ENV MRAA_INSTALL_DIR /opt/mraa/install + +# Set Workdir +WORKDIR $UPM_SRC_DIR + +CMD bash diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base new file mode 100644 index 00000000..89872254 --- /dev/null +++ b/docker/Dockerfile.base @@ -0,0 +1,68 @@ +FROM ubuntu:16.04 + +ENV DEBIAN_FRONTEND noninteractive + +# Prepare apt-get +RUN apt-get update && apt-get -y --no-install-recommends install software-properties-common + +# Add Mraa Repository +RUN add-apt-repository ppa:mraa/mraa && \ + # Update apt-get + apt-get -y update && \ + # Install apt-utils + apt-get -y --no-install-recommends install apt-utils && \ + # Main Build Dependencies + apt-get -y --no-install-recommends install git build-essential cmake swig clang-3.8 gcc-5 g++-5 libpthread-stubs0-dev pkg-config wget unzip \ + # Mraa Build Dependencies + libmraa1 libmraa-dev mraa-tools \ + # Docs Build Dependencies + python-sphinx doxygen graphviz \ + # Sensor Specific Build Dependencies + libjpeg-dev + +# Test Build Dependencies +RUN apt-get -y --no-install-recommends install python python-dev python-pip && \ + pip install chardet + +# Install gcc-6 g++-6 +RUN add-apt-repository ppa:ubuntu-toolchain-r/test && \ + apt-get update && \ + apt-get -y --no-install-recommends install gcc-6 g++-6 + +WORKDIR /opt + +# Install libbacnet 0.3.12 +RUN wget https://downloads.sourceforge.net/project/bacnet/bacnet-stack/bacnet-stack-0.8.3/bacnet-stack-0.8.3.zip && \ + unzip -qq bacnet-stack-0.8.3.zip && cd bacnet-stack-0.8.3 && \ + BACDL_DEFINE=-DBACDL_MSTP=1 MAKE_DEFINE=-fPIC make clean all && \ + cp /opt/bacnet-stack-0.8.3/lib/libbacnet.a /usr/lib/libbacnet.a && \ + echo "prefix=/usr" >> /usr/lib/pkgconfig/libbacnet.pc && \ + echo "exec_prefix=\${prefix}" >> /usr/lib/pkgconfig/libbacnet.pc && \ + echo "libdir=\${exec_prefix}/lib" >> /usr/lib/pkgconfig/libbacnet.pc && \ + echo "includedir=\${exec_prefix}/include" >> /usr/lib/pkgconfig/libbacnet.pc && \ + echo "Name: bacnet" >> /usr/lib/pkgconfig/libbacnet.pc && \ + echo "Description: BACnet library" >> /usr/lib/pkgconfig/libbacnet.pc && \ + echo "Version: 0.8.3" >> /usr/lib/pkgconfig/libbacnet.pc && \ + echo "Libs: -L\${libdir} -lbacnet" >> /usr/lib/pkgconfig/libbacnet.pc && \ + echo "Cflags: -I\${includedir}/bacnet" >> /usr/lib/pkgconfig/libbacnet.pc && \ + mkdir /usr/include/bacnet && \ + cp /opt/bacnet-stack-0.8.3/include/* /usr/include/bacnet/ && \ + rm -fr /opt/bacnet-stack-0.8.3.zip + +# Install libmodbus 3.1.4 +RUN wget http://libmodbus.org/releases/libmodbus-3.1.4.tar.gz && \ + tar xf libmodbus-3.1.4.tar.gz && cd libmodbus-3.1.4 && \ + ./configure && make -j8 && make install && \ + rm -fr /opt/libmodbus-3.1.4.tar.gz + +# Install openzwave +RUN apt-get update && apt-get -y --no-install-recommends install libudev-dev && \ + git clone --depth 1 https://github.com/OpenZWave/open-zwave.git && cd open-zwave && make -j8 install && \ + echo "/usr/local/lib64" >> /etc/ld.so.conf.d/openzwave.conf + +# Set Workdir +ARG UPM_SRC_DIR=/usr/src/app +ENV UPM_SRC_DIR $UPM_SRC_DIR +WORKDIR $UPM_SRC_DIR + +CMD bash diff --git a/docker/Dockerfile.java b/docker/Dockerfile.java new file mode 100644 index 00000000..7f88493e --- /dev/null +++ b/docker/Dockerfile.java @@ -0,0 +1,15 @@ +FROM upm-base + +# Update apt-get +RUN apt-get -y update && \ + # Java Build Dependencies + apt-get -y --no-install-recommends install libmraa-java \ + default-jre default-jdk + +# Configure Java Home +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ + +# Set Workdir +WORKDIR $UPM_SRC_DIR + +CMD bash diff --git a/docker/Dockerfile.node b/docker/Dockerfile.node new file mode 100644 index 00000000..833eebc5 --- /dev/null +++ b/docker/Dockerfile.node @@ -0,0 +1,17 @@ +FROM upm-base + +WORKDIR /opt + +# Node.js Build Dependencies +RUN wget -q -O - https://raw.githubusercontent.com/creationix/nvm/v0.33.2/install.sh | bash +ENV NVM_DIR /root/.nvm +ARG NODE_VERSION +RUN . $NVM_DIR/nvm.sh && nvm install $NODE_VERSION && nvm use $NODE_VERSION && \ + npm install -g node-gyp && node-gyp install +ENV PATH $NVM_DIR/versions/node/$NODE_VERSION/bin:$PATH + +# Set Workdir +WORKDIR $UPM_SRC_DIR + +CMD bash + diff --git a/docker/Dockerfile.python b/docker/Dockerfile.python new file mode 100644 index 00000000..aaa99808 --- /dev/null +++ b/docker/Dockerfile.python @@ -0,0 +1,16 @@ +FROM upm-base + +# Update apt-get +RUN apt-get -y update && \ + # Python Build Dependencies + apt-get -y --no-install-recommends install python-mraa python3-mraa \ + python python-dev python3 python3-dev python-pip python3-pip + +# Install Python Test Dependencies +RUN pip install chardet && pip3 install chardet + +# Set Workdir +WORKDIR $UPM_SRC_DIR + +CMD bash + diff --git a/docker/Dockerfile.sonar b/docker/Dockerfile.sonar new file mode 100644 index 00000000..44ac71ec --- /dev/null +++ b/docker/Dockerfile.sonar @@ -0,0 +1,41 @@ +FROM upm-base + +# Update apt-get +RUN apt-get -y update && \ + # Python Build Dependencies + apt-get -y --no-install-recommends install python-mraa python3-mraa \ + python python-dev python3 python3-dev python-pip python3-pip \ + # Java Build Dependencies + libmraa-java default-jre default-jdk + +# Set JAVA_HOME +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ + +WORKDIR /opt + +# Install Python Test Dependencies +RUN pip install chardet && pip3 install chardet + +# Node.js Build Dependencies +RUN wget -q -O - https://raw.githubusercontent.com/creationix/nvm/v0.33.2/install.sh | bash +ENV NVM_DIR /root/.nvm +ARG NODE_VERSION +RUN . $NVM_DIR/nvm.sh && nvm install $NODE_VERSION && nvm use $NODE_VERSION && \ + npm install -g node-gyp && node-gyp install +ENV PATH $NVM_DIR/versions/node/$NODE_VERSION/bin:$PATH + +# Static Code Analysis Scanner +ENV SONAR_DIR /opt/sonar +ENV SONAR_VER "3.0.3.778" +WORKDIR $SONAR_DIR +RUN wget https://sonarsource.bintray.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${SONAR_VER}-linux.zip && \ + wget https://sonarqube.com/static/cpp/build-wrapper-linux-x86.zip && \ + unzip sonar-scanner-cli-${SONAR_VER}-linux.zip && \ + unzip build-wrapper-linux-x86.zip && \ + rm sonar-scanner-cli-${SONAR_VER}-linux.zip build-wrapper-linux-x86.zip +ENV PATH $SONAR_DIR/sonar-scanner-${SONAR_VER}-linux/bin:$SONAR_DIR/build-wrapper-linux-x86:$PATH + +# Set Workdir +WORKDIR $UPM_SRC_DIR + +CMD bash diff --git a/docs/building.md b/docs/building.md index ca930186..dc939427 100644 --- a/docs/building.md +++ b/docs/building.md @@ -64,10 +64,6 @@ Building with an older version of swig (swig 2.0+) requires the disabling of jav ~~~~~~~~~~~~~ -DBUILDSWIGNODE=OFF ~~~~~~~~~~~~~ -Generating python3 modules instead of python2.7 -~~~~~~~~~~~~~ --DBUILDPYTHON3=ON -~~~~~~~~~~~~~ Disabling python module building ~~~~~~~~~~~~~ -DBUILDSWIGPYTHON=OFF @@ -130,3 +126,175 @@ autotools on linux. ~~~~~~~~~~~ pkg-config --cflags --libs upm-i2clcd ~~~~~~~~~~~ + +## Building with Docker + +You can use `docker` and `docker-compose` to generate a complete build environment +for upm without having to install any other tool. + +Requirements: +* [docker](https://www.docker.com/get-docker) >= 1.12.6 +* [docker-compose](https://docs.docker.com/compose/install/) >= 1.9.0 + +**NOTE:** docker-compose is an optional requirement. It actually make running complex +docker build and run command easier. But you can just use docker to build and run. + +### Docker Images Hierarchy + +To improve build times and images sizes, upm use a build hierarchy to incrementally +create a build environment. Find below a brief description of them: + +1. **upm-base:** Provides the basic infrastructure and tools to compile C/C++ code and documentation. +2. **upm-python:** Provides the python2/python3 build tools. Depends on `upm-base`. +3. **upm-java:** Provides the Java build tools. Depends on `upm-base`. +4. **upm-android:** Provides the Android Things build tools. Depends on `upm-java`. +5. **upm-node4:** Provides the Node.js v4.4.7 build tools. Depends on `upm-base`. +6. **upm-node5:** Same as `upm-node4`, but using Node.js v5.12.0. +7. **upm-node6:** Same as `upm-node4`, but using Node.js v6.11.0. + +**NOTE:** If you want to know which tools are installed for each of the upm targets, +just take a look at the `docker/` folder. All the related Dockerfiles are stored there! + +### Building Docker Images + +**tl;dr:** Just use this commands to build the hierarchy: + +```sh +# Build the base image +$ docker-compose build base +# Build python image +$ docker-compose build python +# Build java image +$ docker-compose build java +# Build node4 image +$ docker-compose build node4 +# Build node5 image +$ docker-compose build node5 +# Build node6 image +$ docker-compose build node6 +# Build android things image +$ docker-compose build android +``` + +**docker-compose** will take a look at the `docker-compose.yaml` file in the repository +root directory, and build the requested target for you. At the end, docker-compose will +tag the image built with an `upm-` prefix. You can check them by running `docker images`. + +If you don't want to use docker-compose, you can also use `docker build` to generate every image. +For example, to create the base image, you can do: + +```sh +# From the repository root folder +$ docker build -d docker/Dockerfile.base -t upm-base . +``` + +Now, you don't actually need to build every image to start working. Let's say you +are a Python developer, and has no idea what Node.js is, just build the base and +python image! + +**NOTE:** If you work on Android Things, you will need the base, java, and android image. + +### Using Docker Images to build Upm + +**tl;dr:** Just use this commands to build upm: + +```sh +# Build upm documentation +$ docker-compose run doc +# Build upm python2 and python3 packages and run python tests +$ docker-compose run python +# Build upm java package and run java tests +$ docker-compose run java +# Build upm node4 package and run node tests +$ docker-compose run node4 +# Build upm node5 package and run node tests +$ docker-compose run node5 +# Build upm node6 package and run node tests +$ docker-compose run node6 +# Build upm for android things package +$ docker-compose run android +``` + +**docker-compose** will take a look at the `docker-compose.yaml` file in the repository +root directory, and run an specific command to build upm for the requested target. +Once the build is completed, you will have a `build/` folder in the repository root with all +the compiled code. This `build/` folder is created by using a docker volume. The `build\` +folder contents is reused each time you execute `docker-compose run [TARGET]`. +To know more about volumes in Docker, visit the [Docker Volume Documentation](https://docs.docker.com/engine/tutorials/dockervolumes/). + +You can also start an interactive session inside the docker container if you need to run some +custom build commands: + +```sh +# Start an interactive bash shell inside the container +$ docker-compose run python bash +# From now, all the commands are executed inside the container +$ cd build && cmake -DBUILDSWIGPYTHON=ON .. && make clean all +``` + +If you don't want to use docker-compose, you can also use `docker run` to build upm. +For example, to build upm for python, you can do: + +```sh +# From the repository root folder +$ docker run \ + --volume=$(pwd):/usr/src/app \ + --env BUILDSWIGPYTHON=ON \ + --env BUILDSWIGJAVA=OFF \ + --env BUILDSWIGNODE=OFF \ + upm-python \ + bash -c "./scripts/run-cmake.sh && make -Cbuild" +``` + +### Proxy considerations + +If, for some reason, you are behind a proxy, find below a list of common problems related +to proxy settings: + +**docker cannot pull images from docker.io** + + Visit [this link](https://docs.docker.com/engine/admin/systemd/#httphttps-proxy) + to configure docker daemon behind a proxy. + +**docker build fails to fetch packages from internet** + +docker-compose will automatically take `http_proxy`, `https_proxy`, and `no_proxy` +environment variables and use it as build arguments. Be sure to properly configure +this variables before building. + +docker, unlinke docker-compose, do not take the proxy settings from the environment +automatically. You need to send them as build arguments: + +```sh +# From the repository root folder +$ docker build \ + -d docker/Dockerfile.base \ + --build-arg http_proxy=$http_proxy \ + --build-arg https_proxy=$https_proxy \ + --build-arg no_proxy=$no_proxy \ + -t upm-base . +``` + +**docker run fails to access the internet** + +docker-compose will automatically take `http_proxy`, `https_proxy`, and `no_proxy` +environment variables and use it as build arguments. Be sure to properly configure +this variables before building. + +docker, unlinke docker-compose, do not take the proxy settings from the environment +automatically. You need to send them as environment arguments: + +```sh +# From the repository root folder +$ docker run \ + --volume=$(pwd):/usr/src/app \ + --env BUILDSWIG=ON \ + --env BUILDSWIGPYTHON=ON \ + --env BUILDSWIGJAVA=OFF \ + --env BUILDSWIGNODE=OFF \ + --env http_proxy=$http_proxy \ + --env https_proxy=$https_proxy \ + --env no_proxy=$no_proxy \ + upm-python \ + bash -c "./scripts/run-cmake.sh && make -Cbuild" +``` diff --git a/scripts/build-android.sh b/scripts/build-android.sh new file mode 100755 index 00000000..1352b520 --- /dev/null +++ b/scripts/build-android.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +set -e + +export PKG_CONFIG_LIBDIR="$NDK_HOME/platforms/android-24/arch-x86/usr/lib:$MRAA_INSTALL_DIR/lib/pkgconfig/" + +cmake \ + -DANDROID_COMPILER_FLAGS_CXX='-std=c++11' \ + -DANDROID_PIE=1 \ + -DANDROID_PLATFORM=android-24 \ + -DANDROID_STL_FORCE_FEATURES=ON \ + -DANDROID_STL=c++_shared \ + -DANDROID_TOOLCHAIN_NAME=x86-i686 \ + -DCMAKE_TOOLCHAIN_FILE=$NDK_HOME/build/cmake/android.toolchain.cmake \ + -DCMAKE_FIND_ROOT_PATH=$MRAA_INSTALL_DIR \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DJAVA_AWT_INCLUDE_PATH=$JAVA_HOME/include \ + -DJAVA_AWT_LIBRARY=$JAVA_HOME/jre/lib/amd64/libjawt.so \ + -DJAVA_INCLUDE_PATH=$JAVA_HOME/include \ + -DJAVA_INCLUDE_PATH2=$JAVA_HOME/include/linux \ + -DJAVA_JVM_LIBRARY=$JAVA_HOME/jre/lib/amd64/server/libjvm.so \ + -DBUILDSWIGJAVA=$BUILDSWIGJAVA \ + -DBUILDSWIGNODE=$BUILDSWIGNODE \ + -DBUILDSWIGPYTHON=$BUILDSWIGPYTHON \ + -DWERROR=$WERROR \ + -H. \ + -Bbuild + +make -j8 -Cbuild + +# Anotate the .java src from doxygen +find src/ -name "javaupm_*.i" > build/upm.i.list +# TODO: install doxy port tool +#doxyport build/upm.i.list \ +# -s src/interfaces/,src/bacnetmstp,src/bmg160,src/bma250e,src/bmm150 \ +# -m doxy/samples.mapping.txt \ +# -d build/src/ \ +# --convert-protected-to-private \ +# -o build/upm-java-files.txt \ +# -c build/compile_commands.json + +gen_and_upload() { + # Skip any directories without .java src files + if [ ! -f *.java ]; then + return + fi + + # Get the MRAA version string from the .pom file + VERSION=$(ls *.pom | perl -nle 'print $2 if /^(.*)-(\d+\.\d+\.\d+)\.pom/') + # Get the MRAA name string from the .pom file + LIB_NAME=$(ls *.pom | perl -nle 'print $1 if /^(.*)-(\d+\.\d+\.\d+)\.pom/') + + # Bundle upload directory + PKG_BUNDLE_DIR=BUNDLE_DIR + mkdir -p ${PKG_BUNDLE_DIR} + + # Create the aar directory structure + AAR_DIRECTORY="$LIB_NAME-$VERSION" + mkdir -p $AAR_DIRECTORY/jni/x86 + mkdir -p $AAR_DIRECTORY/res/values + + # Write out res/values/values.xml to the aar + echo "${values_xml/XXX_LIBRARY/$LIB_NAME}" > $AAR_DIRECTORY/res/values/values.xml + + # Write out AndroidManifest.xml to the aar + _tmp_manifest=${AndroidManifest_xml/XXX_PACKAGE/$JAVA_PKG_PREFIX_DOT} + _tmp_manifest=${_tmp_manifest/XXX_LIBRARY/$LIB_NAME} + echo "${_tmp_manifest/XXX_VERSION/$VERSION}" > $AAR_DIRECTORY/AndroidManifest.xml + + # Copy over the so's + cp *.so $AAR_DIRECTORY/jni/x86 + + # Create javadoc jar + javadoc *.java -d $LIB_NAME-javadoc + jar cf $PKG_BUNDLE_DIR/$LIB_NAME-$VERSION-javadoc.jar $LIB_NAME-javadoc/* + + # Create sources jar file for upload + jar cf $PKG_BUNDLE_DIR/$LIB_NAME-$VERSION-sources.jar *.java + + # Copy pom file + cp $LIB_NAME-$VERSION.pom $PKG_BUNDLE_DIR + + # Copy the java module jar to classes.jar + cp $LIB_NAME.jar $AAR_DIRECTORY/classes.jar + + # Create the .aar + pushd $AAR_DIRECTORY + jar cf ../$PKG_BUNDLE_DIR/$LIB_NAME-$VERSION.aar * + popd + + # Create the bundle file + pushd $PKG_BUNDLE_DIR + # Create the tar bundle + jar cvf ../$LIB_NAME-$VERSION-bundle.jar * + popd +} + +# Template for res/values/values.xml in aar +values_xml='XXX_LIBRARY' + +# Template for AndroidManifest.xml in aar +AndroidManifest_xml=' + + + +' + +# Package prefix +JAVA_PKG_PREFIX_DOT="io.mraa.at.upm" + +# Work in the UPM build directory +cd build/src + +# For each directory in build/src which contains a .pom file... +for lib_dir in `ls -d */`; do + if ls $lib_dir/upm_*.pom 1> /dev/null 2>&1; then + pushd $lib_dir + # Allow for spawning multiple if necessary (&) + gen_and_upload + popd + fi +done diff --git a/scripts/run-cmake.sh b/scripts/run-cmake.sh new file mode 100755 index 00000000..05f1e438 --- /dev/null +++ b/scripts/run-cmake.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Run cmake +cmake \ + -DSWIG_EXECUTABLE=/usr/bin/swig \ + -DBUILDDOC=$BUILDDOC \ + -DBUILDCPP=$BUILDCPP \ + -DBUILDFTI=$BUILDFTI \ + -DBUILDSWIGPYTHON=$BUILDSWIGPYTHON \ + -DBUILDSWIGNODE=$BUILDSWIGNODE \ + -DBUILDSWIGJAVA=$BUILDSWIGJAVA \ + -DBUILDEXAMPLES=$BUILDEXAMPLES \ + -DIPK=$IPK \ + -DRPM=$RPM \ + -DNPM=$NPM \ + -DBUILDTESTS=$BUILDTESTS \ + -DWERROR=$WERROR \ + -H. \ + -Bbuild + diff --git a/scripts/sonar-scan.sh b/scripts/sonar-scan.sh new file mode 100755 index 00000000..93b5150c --- /dev/null +++ b/scripts/sonar-scan.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# +# The script is used for determining options and running a static code +# analysis scan via SonarCloud. +# +# Author: Alex Tereschenko +# +# All environment variables used are passed from either Travis or docker-compose. +# See details at https://docs.sonarqube.org/display/SONAR/Analysis+Parameters. +# +# Travis ones are: +# Created by us: +# - SONAR_ORG - SonarCloud "organization", under which the project is located. +# - SONAR_PROJ_KEY - SonarCloud project key (name) to report to. +# - SONAR_TOKEN - access token for that project (must be protected in Travis). +# - GITHUB_TOKEN - GH OAuth token used by SonarCloud's GH plugin to report status in PRs. +# See details at https://docs.sonarqube.org/display/PLUG/GitHub+Plugin. Must be protected. +# Default: +# - All TRAVIS_* variables. They are described in Travis docs +# at https://docs.travis-ci.com/user/environment-variables +# +# docker-compose ones are: +# - UPM_SRC_DIR - path to upm's git clone in the Docker container. + +# Check required environment variables and exit if they are not set +UPM_SRC_DIR=${UPM_SRC_DIR:?value not provided} +SONAR_PROJ_KEY=${SONAR_PROJ_KEY:?value not provided} +SONAR_ORG=${SONAR_ORG:?value not provided} +SONAR_TOKEN=${SONAR_TOKEN:?value not provided} + +bw_output_path="${UPM_SRC_DIR}/build/bw-output" + +sonar_cmd_base="build-wrapper-linux-x86-64 --out-dir ${bw_output_path} make clean all && \ + sonar-scanner \ + --debug \ + -Dsonar.projectKey=${SONAR_PROJ_KEY} \ + -Dsonar.projectBaseDir=${UPM_SRC_DIR} \ + -Dsonar.sources=${UPM_SRC_DIR} \ + -Dsonar.inclusions='api/**/*,CMakeLists.txt,examples/**/*,imraa/**/*,include/**/*,src/**/*,tests/**/*' \ + -Dsonar.coverage.exclusions='**/*' \ + -Dsonar.cfamily.build-wrapper-output=${bw_output_path} \ + -Dsonar.host.url=https://sonarqube.com \ + -Dsonar.organization=${SONAR_ORG} \ + -Dsonar.login=${SONAR_TOKEN} \ +" + +# Some useful data for logs +echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}" +echo "TRAVIS_PULL_REQUEST: ${TRAVIS_PULL_REQUEST}" +echo "TRAVIS_PULL_REQUEST_SLUG: ${TRAVIS_PULL_REQUEST_SLUG}" +echo "TRAVIS_REPO_SLUG: ${TRAVIS_REPO_SLUG}" + +if [ "${TRAVIS_BRANCH}" == "master" -a "${TRAVIS_PULL_REQUEST}" == "false" ]; then + # Master branch push - do a full-blown scan + echo "Performing master branch push scan" + sonar_cmd="${sonar_cmd_base}" +elif [ "${TRAVIS_PULL_REQUEST}" != "false" -a "${TRAVIS_PULL_REQUEST_SLUG}" == "${TRAVIS_REPO_SLUG}" ]; then + # Internal PR - do a preview scan with report to the PR + ${GITHUB_TOKEN:?value not provided} + + echo "Performing internal pull request scan" + sonar_cmd="${sonar_cmd_base} \ + -Dsonar.analysis.mode=preview \ + -Dsonar.github.pullRequest=${TRAVIS_PULL_REQUEST} \ + -Dsonar.github.repository=${TRAVIS_REPO_SLUG} \ + -Dsonar.github.oauth=${GITHUB_TOKEN} \ + " +else + echo "Skipping the scan - external pull request or non-master branch push" + exit 0 +fi + +echo "About to run the scan, the command is:" +echo "${sonar_cmd}" + +eval "${sonar_cmd}"