From 61544c20997a8e94f6009d4b922eb994c206fa6d Mon Sep 17 00:00:00 2001 From: Artem Smirnov Date: Tue, 25 Sep 2018 19:38:02 +0300 Subject: [PATCH] image_builder: new architecture for builder --- .builder.json | 5 -- .gitignore | 1 + .../assets/clever.service | 0 .../assets/generate_ros_lib | 0 .../assets/hardware_setup.sh | 0 .../assets/init_rpi.sh | 0 .../assets/kinetic-ros-clever.rosinstall | 0 .../assets/kinetic-rosdep-clever.yaml | 0 .../assets/roscore.env | 0 .../assets/roscore.service | 0 builder/image-build.sh | 82 +++++++++++++++++++ .../init_image.sh => builder/image-init.sh | 0 .../image-network.sh | 0 .../ros_install.sh => builder/image-ros.sh | 0 .../image-software.sh | 0 builder_docker/image-build.sh | 76 ----------------- builder_scripts/builder.sh | 34 -------- 17 files changed, 83 insertions(+), 115 deletions(-) delete mode 100644 .builder.json rename {builder_scripts => builder}/assets/clever.service (100%) rename {builder_scripts => builder}/assets/generate_ros_lib (100%) rename {builder_scripts => builder}/assets/hardware_setup.sh (100%) rename {builder_scripts => builder}/assets/init_rpi.sh (100%) rename {builder_scripts => builder}/assets/kinetic-ros-clever.rosinstall (100%) rename {builder_scripts => builder}/assets/kinetic-rosdep-clever.yaml (100%) rename {builder_scripts => builder}/assets/roscore.env (100%) rename {builder_scripts => builder}/assets/roscore.service (100%) create mode 100755 builder/image-build.sh rename builder_scripts/init_image.sh => builder/image-init.sh (100%) rename builder_scripts/network_setup.sh => builder/image-network.sh (100%) rename builder_scripts/ros_install.sh => builder/image-ros.sh (100%) rename builder_scripts/software_install.sh => builder/image-software.sh (100%) delete mode 100755 builder_docker/image-build.sh delete mode 100755 builder_scripts/builder.sh diff --git a/.builder.json b/.builder.json deleted file mode 100644 index 7815362b..00000000 --- a/.builder.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "main_script":"/builder_scripts/builder.sh", - "scripts_dir":"/builder_scripts", - "source_image": "https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2017-12-01/2017-11-29-raspbian-stretch-lite.zip" -} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 78497847..533c8130 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc *.DS_Store +/image \ No newline at end of file diff --git a/builder_scripts/assets/clever.service b/builder/assets/clever.service similarity index 100% rename from builder_scripts/assets/clever.service rename to builder/assets/clever.service diff --git a/builder_scripts/assets/generate_ros_lib b/builder/assets/generate_ros_lib similarity index 100% rename from builder_scripts/assets/generate_ros_lib rename to builder/assets/generate_ros_lib diff --git a/builder_scripts/assets/hardware_setup.sh b/builder/assets/hardware_setup.sh similarity index 100% rename from builder_scripts/assets/hardware_setup.sh rename to builder/assets/hardware_setup.sh diff --git a/builder_scripts/assets/init_rpi.sh b/builder/assets/init_rpi.sh similarity index 100% rename from builder_scripts/assets/init_rpi.sh rename to builder/assets/init_rpi.sh diff --git a/builder_scripts/assets/kinetic-ros-clever.rosinstall b/builder/assets/kinetic-ros-clever.rosinstall similarity index 100% rename from builder_scripts/assets/kinetic-ros-clever.rosinstall rename to builder/assets/kinetic-ros-clever.rosinstall diff --git a/builder_scripts/assets/kinetic-rosdep-clever.yaml b/builder/assets/kinetic-rosdep-clever.yaml similarity index 100% rename from builder_scripts/assets/kinetic-rosdep-clever.yaml rename to builder/assets/kinetic-rosdep-clever.yaml diff --git a/builder_scripts/assets/roscore.env b/builder/assets/roscore.env similarity index 100% rename from builder_scripts/assets/roscore.env rename to builder/assets/roscore.env diff --git a/builder_scripts/assets/roscore.service b/builder/assets/roscore.service similarity index 100% rename from builder_scripts/assets/roscore.service rename to builder/assets/roscore.service diff --git a/builder/image-build.sh b/builder/image-build.sh new file mode 100755 index 00000000..b8dba799 --- /dev/null +++ b/builder/image-build.sh @@ -0,0 +1,82 @@ +@@ -0,0 +1,81 @@ +#! /usr/bin/env bash + +# +# Script for build the image. Used builder script of the target repo +# For build: docker run --privileged -it --rm -v /dev:/dev -v $(pwd):/builder/repo smirart/builder +# +# Copyright (C) 2018 Copter Express Technologies +# +# Author: Artem Smirnov +# +# Distributed under MIT License (available at https://opensource.org/licenses/MIT). +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# + +set -e # Exit immidiately on non-zero result + +SOURCE_IMAGE="https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2017-12-01/2017-11-29-raspbian-stretch-lite.zip" + +export DEBIAN_FRONTEND=${DEBIAN_FRONTEND:='noninteractive'} +export LANG=${LANG:='C.UTF-8'} +export LC_ALL=${LC_ALL:='C.UTF-8'} + +echo "BUILDER_DIR (pwd): $(pwd)" + +BUILDER_DIR="$(pwd)" +REPO_DIR="$(pwd)/repo" +SCRIPTS_DIR="${REPO_DIR}/builder" +IMAGES_DIR="${REPO_DIR}/images" + +[[ ! -d "${SCRIPTS_DIR}" ]] && (echo -e "Directory ${SCRIPTS_DIR} doesn't exist"; exit 1) +[[ ! -d ${IMAGES_DIR} ]] && mkdir ${IMAGES_DIR} && echo -e "Directory ${IMAGES_DIR} was created successful" + +IMAGE_VERSION="${TRAVIS_TAG:=$(cd ${REPO_DIR}; git log --format=%h -1)}" +REPO_NAME=$(basename -s '.git' $(git remote --verbose | grep origin | grep fetch | cut -f2 | cut -d' ' -f1)) +IMAGE_NAME="${REPO_NAME}_${IMAGE_VERSION}.img" +IMAGE_PATH="${IMAGES_DIR}/${IMAGE_NAME}" + +get_image() { + # TEMPLATE: get_image + local BUILD_DIR=$(dirname $1) + local RPI_ZIP_NAME=$(basename $2) + local RPI_IMAGE_NAME=$(echo ${RPI_ZIP_NAME} | sed 's/zip/img/') + + if [ ! -e "${BUILD_DIR}/${RPI_ZIP_NAME}" ]; then + echo -e "Downloading original Linux distribution" \ + && wget -nv -O ${BUILD_DIR}/${RPI_ZIP_NAME} $2 \ + && echo -e "Downloading complete" "SUCCESS" \ + || (echo -e "Downloading was failed!" "ERROR"; exit 1) + else echo -e "Linux distribution already donwloaded"; fi + + echo -e "Unzipping Linux distribution image" \ + && unzip -p ${BUILD_DIR}/${RPI_ZIP_NAME} ${RPI_IMAGE_NAME} > $1 \ + && echo -e "Unzipping complete" "SUCCESS" \ + || (echo -e "Unzipping was failed!" "ERROR"; exit 1) +} + +get_image ${IMAGE_PATH} ${SOURCE_IMAGE} + +# Make free space +./image-resize.sh ${IMAGE_PATH} max '7G' + +./image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/init_rpi.sh' '/root/' +./image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/hardware_setup.sh' '/root/' + +./image-chroot.sh ${IMAGE_PATH} exec ${SCRIPTS_DIR}'/image-init.sh' ${CLEVER_VERSION} $(jq '.source_image' -r ${TARGET_CONFIG}) +./image-chroot.sh ${IMAGE_PATH} exec ${SCRIPTS_DIR}'/image-software.sh' +./image-chroot.sh ${IMAGE_PATH} exec ${SCRIPTS_DIR}'/image-network.sh' + +# If RPi then use a one thread to build a ROS package on RPi, else use all +[[ $(arch) == 'armv7l' ]] && NUMBER_THREADS=1 || NUMBER_THREADS=$(nproc --all) + +./image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/clever.service' '/root/' +./image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/roscore.env' '/root/' +./image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/roscore.service' '/root/' + +./image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/kinetic-rosdep-clever.yaml' '/etc/ros/rosdep/' +# ./image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/kinetic-ros-clever.rosinstall' '/home/pi/ros_catkin_ws/' +./image-chroot.sh ${IMAGE_PATH} exec ${SCRIPTS_DIR}'/image-ros.sh' ${TARGET_REPO} ${TARGET_REF} False False ${NUMBER_THREADS} + +./image-resize.sh ${IMAGE_PATH} diff --git a/builder_scripts/init_image.sh b/builder/image-init.sh similarity index 100% rename from builder_scripts/init_image.sh rename to builder/image-init.sh diff --git a/builder_scripts/network_setup.sh b/builder/image-network.sh similarity index 100% rename from builder_scripts/network_setup.sh rename to builder/image-network.sh diff --git a/builder_scripts/ros_install.sh b/builder/image-ros.sh similarity index 100% rename from builder_scripts/ros_install.sh rename to builder/image-ros.sh diff --git a/builder_scripts/software_install.sh b/builder/image-software.sh similarity index 100% rename from builder_scripts/software_install.sh rename to builder/image-software.sh diff --git a/builder_docker/image-build.sh b/builder_docker/image-build.sh deleted file mode 100755 index 04c723f7..00000000 --- a/builder_docker/image-build.sh +++ /dev/null @@ -1,76 +0,0 @@ -#! /usr/bin/env bash - -# -# Script for build the image. Used builder script of the target repo -# Copyright (C) 2018 Copter Express Technologies -# -# Author: Artem Smirnov -# -# Distributed under MIT License (available at https://opensource.org/licenses/MIT). -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# - -set -e # Exit immidiately on non-zero result - -export DEBIAN_FRONTEND=${DEBIAN_FRONTEND:='noninteractive'} -export LANG=${LANG:='C.UTF-8'} -export LC_ALL=${LC_ALL:='C.UTF-8'} - -export TARGET_REPO=${TARGET_REPO:='https://github.com/urpylka/clever.git'} -export TARGET_REF=${TARGET_REF:='master'} -export TARGET_CONFIG=${TARGET_CONFIG:='/.builder.json'} - -echo "================================================================================" -echo "TARGET_REPO: $TARGET_REPO" -echo "TARGET_REF: $TARGET_REF" -echo "TARGET_CONFIG: $TARGET_CONFIG" -echo "================================================================================" - -get_image() { - # TEMPLATE: get_image - local BUILD_DIR=$(dirname $1) - local RPI_ZIP_NAME=$(basename $2) - local RPI_IMAGE_NAME=$(echo ${RPI_ZIP_NAME} | sed 's/zip/img/') - - if [ ! -e "${BUILD_DIR}/${RPI_ZIP_NAME}" ]; then - echo -e "Downloading original Linux distribution" \ - && wget -nv -O ${BUILD_DIR}/${RPI_ZIP_NAME} $2 \ - && echo -e "Downloading complete" "SUCCESS" \ - || (echo -e "Downloading was failed!" "ERROR"; exit 1) - else echo -e "Linux distribution already donwloaded"; fi - - echo -e "Unzipping Linux distribution image" \ - && unzip -p ${BUILD_DIR}/${RPI_ZIP_NAME} ${RPI_IMAGE_NAME} > $1 \ - && echo -e "Unzipping complete" "SUCCESS" \ - || (echo -e "Unzipping was failed!" "ERROR"; exit 1) -} - -# TODO: The repository can be already downloaded, use the TARGET_REPO also as unix path. -REPO_DIR=$(mktemp -d --suffix=.builder_repo) -git clone ${TARGET_REPO} --single-branch --branch ${TARGET_REF} --depth 1 ${REPO_DIR} &> /dev/null \ -|| (echo 'Error: Could not clone repo!'; return 1) -[[ -f ${REPO_DIR}${TARGET_CONFIG} ]] && export TARGET_CONFIG=${REPO_DIR}${TARGET_CONFIG} \ -|| (echo "Error: TARGET_CONFIG doesn't exist!"; return 1) - -CUR_DIR="$(pwd)" && cd ${REPO_DIR} -#TARGET_COMMIT=$(git show-ref --hash origin/master | cut -c1-7) -cd ${CUR_DIR} && unset CUR_DIR - -#export IMAGE_VERSION="${TARGET_REF}_${TARGET_COMMIT}" -export IMAGE_VERSION="${TARGET_REF}" -export IMAGE_PATH="$(pwd)/image/$(basename -s '.git' ${TARGET_REPO})_${IMAGE_VERSION}.img" - -get_image ${IMAGE_PATH} $(jq '.source_image' -r ${TARGET_CONFIG}) - -REGISTER=':arm:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/qemu-arm-static:' -if [[ $(arch) != 'armv7l' ]]; then - mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc 2> /dev/null || true - echo ${REGISTER} > /proc/sys/fs/binfmt_misc/register 2> /dev/null || true - ./image-chroot.sh ${IMAGE_PATH} copy './qemu-arm-resin' '/usr/bin/qemu-arm-static' -fi - -export IMAGE_BUILDER="$(dirname $(readlink -e "$0"))" -export SCRIPTS_DIR="${REPO_DIR}$(jq '.scripts_dir' -r ${TARGET_CONFIG})" - -${REPO_DIR}$(jq '.main_script' -r ${TARGET_CONFIG}) diff --git a/builder_scripts/builder.sh b/builder_scripts/builder.sh deleted file mode 100755 index a1c1063d..00000000 --- a/builder_scripts/builder.sh +++ /dev/null @@ -1,34 +0,0 @@ -#! /usr/bin/env bash - -# -# Script for image configure -# @urpylka Artem Smirnov -# - -# Exit immidiately on non-zero result -set -e - -cd ${IMAGE_BUILDER} - -# Make free space -./image-resize.sh ${IMAGE_PATH} max '7G' - -./image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/init_rpi.sh' '/root/' -./image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/hardware_setup.sh' '/root/' - -./image-chroot.sh ${IMAGE_PATH} exec ${SCRIPTS_DIR}'/init_image.sh' ${CLEVER_VERSION} $(jq '.source_image' -r ${TARGET_CONFIG}) -./image-chroot.sh ${IMAGE_PATH} exec ${SCRIPTS_DIR}'/software_install.sh' -./image-chroot.sh ${IMAGE_PATH} exec ${SCRIPTS_DIR}'/network_setup.sh' - -# If RPi then use a one thread to build a ROS package on RPi, else use all -[[ $(arch) == 'armv7l' ]] && NUMBER_THREADS=1 || NUMBER_THREADS=$(nproc --all) - -./image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/clever.service' '/root/' -./image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/roscore.env' '/root/' -./image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/roscore.service' '/root/' - -./image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/kinetic-rosdep-clever.yaml' '/etc/ros/rosdep/' -# ./image-chroot.sh ${IMAGE_PATH} copy ${SCRIPTS_DIR}'/assets/kinetic-ros-clever.rosinstall' '/home/pi/ros_catkin_ws/' -./image-chroot.sh ${IMAGE_PATH} exec ${SCRIPTS_DIR}'/ros_install.sh' ${TARGET_REPO} ${TARGET_REF} False False ${NUMBER_THREADS} - -./image-resize.sh ${IMAGE_PATH}