From ccef57f3113b724d951835be4e69bd8c7da53a5a Mon Sep 17 00:00:00 2001 From: Artem Smirnov Date: Mon, 24 Sep 2018 16:56:39 +0300 Subject: [PATCH] image_builder: new structure of builder --- .travis.yml | 39 ++-- builder_docker/autosizer.sh | 134 ------------ builder_docker/echo_stamp.sh | 26 --- builder_docker/get_image.sh | 32 --- builder_docker/{build.sh => image-build.sh} | 36 +++- builder_docker/image-chroot.sh | 184 ++++++++++++++++ .../{publish_image.sh => image-publish.sh} | 43 ++-- builder_docker/image-resize.sh | 203 ++++++++++++++++++ builder_docker/image_config.sh | 168 --------------- builder_docker/resize_fs.sh | 57 ----- builder_docker/yadisk.py | 13 +- builder_scripts/builder.sh | 26 +-- 12 files changed, 486 insertions(+), 475 deletions(-) delete mode 100755 builder_docker/autosizer.sh delete mode 100755 builder_docker/echo_stamp.sh delete mode 100755 builder_docker/get_image.sh rename builder_docker/{build.sh => image-build.sh} (58%) create mode 100755 builder_docker/image-chroot.sh rename builder_docker/{publish_image.sh => image-publish.sh} (54%) create mode 100755 builder_docker/image-resize.sh delete mode 100755 builder_docker/image_config.sh delete mode 100755 builder_docker/resize_fs.sh diff --git a/.travis.yml b/.travis.yml index b5230ce2..470a05e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,31 +1,42 @@ +# More info there +# https://github.com/travis-ci/travis-ci/issues/6893 +# https://docs.travis-ci.com/user/customizing-the-build/ +# https://docs.travis-ci.com/user/deployment/releases +# https://docs.travis-ci.com/user/environment-variables/ + sudo: required language: generic services: - docker -branches: - only: - - /^v\d+\.\d+(\.\d+)?(-\S*)?$/ +# branches: +# only: +# - master +# - /^v\d+\.\d+(\.\d+)?(-\S*)?$/ env: global: - - DOCKER=smirart/builder:latest - - TARGET_REPO=https://github.com/${TRAVIS_REPO_SLUG}.git - - IMAGE_VERSION="${TRAVIS_BRANCH}_$(echo ${TRAVIS_COMMIT} | cut -c1-7)" - - IMAGE_PATH="$(pwd)/image/$(basename -s '.git' ${TARGET_REPO})_${IMAGE_VERSION}.img" - - IMAGE_NAME="$(basename ${IMAGE_PATH})" - - BUILD_DIR="$(dirname ${IMAGE_PATH})" + - DOCKER = smirart/builder:latest + - TARGET_REPO = https://github.com/${TRAVIS_REPO_SLUG}.git + # - TARGET_REF = ${TRAVIS_BRANCH} + - TARGET_REF = ${TRAVIS_TAG} + # - IMAGE_VERSION = "${TRAVIS_BRANCH}_$(echo ${TRAVIS_COMMIT} | cut -c1-7)" + - IMAGE_VERSION = ${TRAVIS_TAG} + - IMAGE_PATH = "$(pwd)/image/$(basename -s '.git' ${TARGET_REPO})_${IMAGE_VERSION}.img" + - IMAGE_NAME = "$(basename ${IMAGE_PATH})" + - BUILD_DIR = "$(dirname ${IMAGE_PATH})" git: depth: 1 before_script: - docker pull ${DOCKER} script: - - docker run --privileged --rm -v /dev:/dev -v ${BUILD_DIR}:/builder/image -e "TARGET_REPO=${TARGET_REPO}" -e "TARGET_REF=${TRAVIS_BRANCH}" ${DOCKER} -#after_failure: -# - skip_deploy: true + - [[ -z TARGET_REF ]] || exit 1 + - docker run --privileged --rm -v /dev:/dev -v ${BUILD_DIR}:/builder/image -e "TARGET_REPO=${TARGET_REPO}" -e "TARGET_REF=${TARGET_REF}" ${DOCKER} +# after_failure: +# - skip_deploy: true before_deploy: # Set up git user name and tag this commit - git config --local user.name "urpylka" - git config --local user.email "urpylka@gmail.com" - #- git tag "$(date +'%Y%m%d%H%M%S')-$(git log --format=%h -1)" + # - git tag "$(date +'%Y%m%d%H%M%S')-$(git log --format=%h -1)" - sudo chmod -R 777 ${BUILD_DIR} - cd ${BUILD_DIR} && zip ${IMAGE_NAME}.zip ${IMAGE_NAME} deploy: @@ -34,4 +45,4 @@ deploy: file: ${IMAGE_PATH}.zip skip_cleanup: true on: - tags: true \ No newline at end of file + tags: true diff --git a/builder_docker/autosizer.sh b/builder_docker/autosizer.sh deleted file mode 100755 index 2e8298fa..00000000 --- a/builder_docker/autosizer.sh +++ /dev/null @@ -1,134 +0,0 @@ -#! /usr/bin/env bash - -# -# Script for image configure -# @urpylka Artem Smirnov -# - -# Exit immidiately on non-zero result -set -e - -echo_bold() { - # TEMPLATE: echo_bold - # TYPE: SUCCESS, ERROR, INFO - - # More info there https://www.shellhacks.com/ru/bash-colors/ - - TEXT="$1" - TEXT="\e[1m$TEXT\e[0m" # BOLD - - case "$2" in - SUCCESS) - TEXT="\e[32m${TEXT}\e[0m";; # GREEN - ERROR) - TEXT="\e[31m${TEXT}\e[0m";; # RED - *) - TEXT="\e[34m${TEXT}\e[0m";; # BLUE - esac - echo -e ${TEXT} -} - -if [ $(whoami) != "root" ]; then - echo \ - && echo "********************************************************************" \ - && echo "******************** This should be run as root ********************" \ - && echo "********************************************************************" \ - && echo \ - && exit 1 -fi - -if [[ -z $1 ]]; then - echo "================================================================================" - echo_bold "Automatic Image file resizer" - echo_bold "Description: This script shrink your image to 10MiB free space" - echo_bold "if you didn't set FREE_SPACE in MiB (see usage below)." - echo_bold "Authors: Artem Smirnov @urpylka, SirLagz" - echo - echo_bold "Usage: ./autosizer.sh IMAGE_PATH FREE_SPACE" - echo - echo_bold "Requirements: parted, losetup, e2fsck, resize2fs, bc, truncate" - echo "================================================================================" - exit 0 -fi - -echo "================================================================================" -strImgFile=$1 -echo_bold "Path to image: $strImgFile" -echo "================================================================================" - -if [[ ! -e $strImgFile ]]; then - echo_bold "Error: File doesn't exist" - echo - exit 1 -fi - -echo "================================================================================" -partinfo=`parted -m $strImgFile unit B print` -echo_bold "Partition information:\n$partinfo" -echo "================================================================================" - -partnumber=`echo "$partinfo" | grep ext4 | awk -F: '{ print $1 }'` -echo_bold "Partition number: $partnumber" -echo "================================================================================" - -partstart=`echo "$partinfo" | grep ext4 | awk -F: '{ print substr($2,0,length($2)-1) }'` -echo_bold "Partition start: $partstart (bytes)" -echo "================================================================================" - -loopback=`losetup -f --show -o $partstart $strImgFile` -echo_bold "Loopback device: $loopback" -echo "================================================================================" - -set +e -e2fsck -fvy $loopback -set -e - -echo "================================================================================" -minsize=`resize2fs -P $loopback | awk -F': ' '{ print $2 }'` -#minsize=`resize2fs -P $loopback 2> /dev/null | awk -F': ' '{ print $2 }'` -echo_bold "Minsize: $minsize (4KiB)" -echo "================================================================================" - -# Default add 10MiB free space to image, if $2 doesn't set -FREE_SPACE=${2:-10} - -FREE_SPACE=$(($FREE_SPACE*1024*1024/4096)) - -minsize=`echo "$minsize+$FREE_SPACE" | bc` -echo_bold "Minsize + $FREE_SPACE (4KiB): $minsize (4KiB)" -echo "================================================================================" - -resize2fs -p $loopback $minsize -sleep 1 -losetup -d $loopback - -echo "================================================================================" -partnewsize=`echo "$minsize * 4096" | bc` -echo_bold "New size of part: $minsize (4KiB) = $partnewsize (bytes)" -echo "================================================================================" - -newpartend=`echo "$partstart + $partnewsize" | bc` -echo_bold "New end of part (Part start + part new size):" -echo_bold "$partstart (bytes) + $partnewsize (bytes) = $newpartend (bytes)" -echo "================================================================================" - -part1=`parted $strImgFile rm 2` -echo "================================================================================" -part2=`parted $strImgFile unit B mkpart primary $partstart $newpartend` - -echo "================================================================================" -endresult=`parted -m $strImgFile unit B print free | tail -1 | awk -F: '{ print substr($2,0,length($2)-1) }'` -echo_bold "Size of result image: $endresult (bytes)" -echo "================================================================================" - -truncate -s $endresult $strImgFile - -echo "================================================================================" -partinfo=`parted -m $strImgFile unit B print` -echo_bold "Partition information:\n$partinfo" -echo "================================================================================" - -# TODO check if image needs to change PARTUUID -#sed -i 's/root=[^ ]*/root=\/dev\/mmcblk0p2/' /boot/cmdline.txt -#sed -i 's/.* \/boot vfat defaults 0 2$/\/dev\/mmcblk0p1 \/boot vfat defaults 0 2/' /etc/fstab -#sed -i 's/.* \/ ext4 defaults,noatime 0 1$/\/dev\/mmcblk0p2 \/ ext4 defaults,noatime 0 1/' /etc/fstab diff --git a/builder_docker/echo_stamp.sh b/builder_docker/echo_stamp.sh deleted file mode 100755 index 2fe68eeb..00000000 --- a/builder_docker/echo_stamp.sh +++ /dev/null @@ -1,26 +0,0 @@ -#! /usr/bin/env bash - -# -# Script for image configure -# @urpylka Artem Smirnov -# - -echo_stamp() { - # TEMPLATE: echo_stamp - # TYPE: SUCCESS, ERROR, INFO - - # More info there https://www.shellhacks.com/ru/bash-colors/ - - TEXT="$(date) | $1" - TEXT="\e[1m$TEXT\e[0m" # BOLD - - case "$2" in - SUCCESS) - TEXT="\e[32m${TEXT}\e[0m";; # GREEN - ERROR) - TEXT="\e[31m${TEXT}\e[0m";; # RED - *) - TEXT="\e[34m${TEXT}\e[0m";; # BLUE - esac - echo -e ${TEXT} -} \ No newline at end of file diff --git a/builder_docker/get_image.sh b/builder_docker/get_image.sh deleted file mode 100755 index bd83bec9..00000000 --- a/builder_docker/get_image.sh +++ /dev/null @@ -1,32 +0,0 @@ -#! /usr/bin/env bash - -# -# Script for image configure -# @urpylka Artem Smirnov -# @dvornikov-aa Andrey Dvornikov -# - -# Exit immidiately on non-zero result -set -e - -source echo_stamp.sh - -get_image() { - # TEMPLATE: get_image - local BUILD_DIR=$(dirname $1) - local RPI_ZIP_NAME=$(basename $2) - if [ ! -e "${BUILD_DIR}/${RPI_ZIP_NAME}" ]; - then - echo_stamp "1. Downloading original Linux distribution" - wget -nv -O ${BUILD_DIR}/${RPI_ZIP_NAME} $2 \ - && echo_stamp "Downloading complete" "SUCCESS" - else - echo_stamp "1. Linux distribution already donwloaded" - fi - echo_stamp "2. Unzipping Linux distribution image" - local RPI_IMAGE_NAME=$(echo ${RPI_ZIP_NAME} | sed 's/zip/img/') - unzip -p ${BUILD_DIR}/${RPI_ZIP_NAME} ${RPI_IMAGE_NAME} > $1 - echo_stamp "Unzipping complete" "SUCCESS" -} - -get_image $1 $2 diff --git a/builder_docker/build.sh b/builder_docker/image-build.sh similarity index 58% rename from builder_docker/build.sh rename to builder_docker/image-build.sh index 30ecffdd..97cd0a17 100755 --- a/builder_docker/build.sh +++ b/builder_docker/image-build.sh @@ -1,12 +1,17 @@ #! /usr/bin/env bash # -# Script for image configure -# @urpylka Artem Smirnov +# Script for build the image. Used builder script of the target repo +# Copyright (C) 2018 Copter Express Technologies +# +# Author: Artem Smirnov +# +# Distributed under MIT License (available at https://opensource.org/licenses/MIT). +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. # -# Exit immidiately on non-zero result -set -e +set -e # Exit immidiately on non-zero result export DEBIAN_FRONTEND=${DEBIAN_FRONTEND:='noninteractive'} export LANG=${LANG:='C.UTF-8'} @@ -22,6 +27,25 @@ echo "TARGET_REF: $TARGET_REF" echo "TARGET_CONFIG: $TARGET_CONFIG" echo "================================================================================" +get_image() { + # TEMPLATE: get_image + local BUILD_DIR=$(dirname $1) + local RPI_ZIP_NAME=$(basename $2) + local RPI_IMAGE_NAME=$(echo ${RPI_ZIP_NAME} | sed 's/zip/img/') + + if [ ! -e "${BUILD_DIR}/${RPI_ZIP_NAME}" ]; then + echo_stamp "Downloading original Linux distribution" \ + && wget -nv -O ${BUILD_DIR}/${RPI_ZIP_NAME} $2 \ + && echo_stamp "Downloading complete" "SUCCESS" \ + || (echo_stamp "Downloading was failed!" "ERROR"; exit 1) + else; echo_stamp "Linux distribution already donwloaded"; fi + + echo_stamp "Unzipping Linux distribution image" \ + && unzip -p ${BUILD_DIR}/${RPI_ZIP_NAME} ${RPI_IMAGE_NAME} > $1 \ + && echo_stamp "Unzipping complete" "SUCCESS" \ + || (echo_stamp "Unzipping was failed!" "ERROR"; exit 1) +} + # TODO: The repository can be already downloaded, use the TARGET_REPO also as unix path. REPO_DIR=$(mktemp -d --suffix=.builder_repo) git clone ${TARGET_REPO} --single-branch --branch ${TARGET_REF} --depth 1 ${REPO_DIR} \ @@ -36,13 +60,13 @@ cd ${CUR_DIR} && unset CUR_DIR export IMAGE_VERSION="${TARGET_REF}_${TARGET_COMMIT}" export IMAGE_PATH="$(pwd)/image/$(basename -s '.git' ${TARGET_REPO})_${IMAGE_VERSION}.img" -./get_image.sh ${IMAGE_PATH} $(jq '.source_image' -r ${TARGET_CONFIG}) +get_image ${IMAGE_PATH} $(jq '.source_image' -r ${TARGET_CONFIG}) REGISTER=':arm:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/qemu-arm-static:' if [[ $(arch) != 'armv7l' ]]; then mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc 2> /dev/null || true echo ${REGISTER} > /proc/sys/fs/binfmt_misc/register 2> /dev/null || true - ./image_config.sh copy_to_chroot ${IMAGE_PATH} './qemu-arm-resin' '/usr/bin/qemu-arm-static' + ./image-chroot.sh ${IMAGE_PATH} copy './qemu-arm-resin' '/usr/bin/qemu-arm-static' fi export IMAGE_BUILDER="$(dirname $(readlink -e "$0"))" diff --git a/builder_docker/image-chroot.sh b/builder_docker/image-chroot.sh new file mode 100755 index 00000000..c7f926fe --- /dev/null +++ b/builder_docker/image-chroot.sh @@ -0,0 +1,184 @@ +#! /usr/bin/env bash + +# +# Script for upload the image to yadisk & change the release message on GitHub +# Copyright (C) 2018 Copter Express Technologies +# +# Author: Artem Smirnov +# Author: Andrey Dvornikov +# +# Distributed under MIT License (available at https://opensource.org/licenses/MIT). +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# + +set -e # Exit immidiately on non-zero result + +echo_stamp() { + # TEMPLATE: echo_stamp + # TYPE: SUCCESS, ERROR, INFO + + # More info there https://www.shellhacks.com/ru/bash-colors/ + + TEXT="$(date '+[%Y-%m-%d %H:%M:%S]') $1" + TEXT="\e[1m$TEXT\e[0m" # BOLD + + case "$2" in + SUCCESS) + TEXT="\e[32m${TEXT}\e[0m";; # GREEN + ERROR) + TEXT="\e[31m${TEXT}\e[0m";; # RED + *) + TEXT="\e[34m${TEXT}\e[0m";; # BLUE + esac + echo -e ${TEXT} +} + +# This script doesn't work on Ubuntu because OS`s losetup does not consist --partscan (-P). + +# Idea: use `mount -o loop,offset` +# https://stefanoprenna.com/blog/2014/09/22/tutorial-how-to-mount-raw-images-img-images-on-linux/ +# REPO_DIR=$(mktemp -d --suffix=.builder_repo) +# mount -t ext4 -o loop,offset=$((94208 * 512)) image/clever_qemu_test_2_20180822_163141.img "$REPO_DIR" +# mount -t vfat -o loop,offset=$((8192 * 512)) image/clever_qemu_test_2_20180822_163141.img "$REPO_DIR/boot" + +execute() { + # execute [ [...]] + + echo_stamp "Mount loop-image: $1" + local DEV_IMAGE=$(losetup -Pf $1 --show) + sleep 0.5 + + local MOUNT_POINT=$(mktemp -d --suffix=.builder_image) + echo_stamp "Mount dirs ${MOUNT_POINT} & ${MOUNT_POINT}/boot" + mount "${DEV_IMAGE}p2" ${MOUNT_POINT} + mount "${DEV_IMAGE}p1" ${MOUNT_POINT}/boot + + echo_stamp "Bind system dirs" + + echo_stamp "Mounting /proc in chroot... " + if [ ! -d ${MOUNT_POINT}/proc ]; then + mkdir -p ${MOUNT_POINT}/proc \ + && mount -t proc -o nosuid,noexec,nodev proc ${MOUNT_POINT}/proc + && echo_stamp "OK" "SUCCESS" \ + || (echo_stamp "Failed" "ERROR"; exit 1) + else echo_stamp "/sys already exist" "SUCCESS" + fi + + echo_stamp "Mounting /sys in chroot... " + if [ ! -d ${MOUNT_POINT}/sys ]; then + mkdir -p ${MOUNT_POINT}/sys \ + && mount -t sysfs -o nosuid,noexec,nodev sysfs ${MOUNT_POINT}/sys \ + && echo_stamp "OK" "SUCCESS" \ + || (echo_stamp "Failed" "ERROR"; exit 1) + else echo_stamp "/sys already exist" "SUCCESS" + fi + + echo_stamp "Mounting /dev/ and /dev/pts in chroot... " \ + && mkdir -p -m 755 ${MOUNT_POINT}/dev/pts \ + && mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${MOUNT_POINT}/dev \ + && mount -t devpts -o gid=5,mode=620 devpts ${MOUNT_POINT}/dev/pts \ + && echo_stamp "OK" "SUCCESS" \ + || (echo_stamp "Failed" "ERROR"; exit 1) + + echo_stamp "Copy DNS records" \ + && cp -L /etc/resolv.conf ${MOUNT_POINT}/etc/resolv.conf \ + && echo_stamp "OK" "SUCCESS" \ + || (echo_stamp "Failed" "ERROR"; exit 1) + + if [[ $# > 1 ]]; then + echo_stamp "Copy script into chroot fs" + + local SCRIPT_NAME=$(basename $2) + local SCRIPT_PATH="$(mktemp -d -p ${MOUNT_POINT}/tmp --suffix=.tmp_builder_script)" + + local script_name=$(basename $2) + local script_path_root="${MOUNT_POINT}/root/${script_name}" + + # TODO: maybe copy to tmp-dir + # TODO: Find more suitable location for temporary script storage + # $(mktemp -p ${MOUNT_POINT}/tmp --suffix=.tmp_builder_script) + + cp "$2" "${script_path_root}" + # Run script in chroot with additional arguments + chroot ${MOUNT_POINT} /bin/sh -c "/root/${script_name} ${@:3}" + # Removing script from chroot fs + rm "${script_path_root}" + else + # https://wiki.archlinux.org/index.php/Change_root_(%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9) + # http://www.unix-lab.org/posts/chroot/ + # https://habrahabr.ru/post/141012/ + # https://losst.ru/vosstanovlenie-grub2 + # http://unixteam.ru/content/virtualizaciya-ili-zapuskaem-prilozhenie-v-chroot-okruzhenii-razmyshleniya + # http://help.ubuntu.ru/wiki/%D0%B2%D0%BE%D1%81%D1%81%D1%82%D0%B0%D0%BD%D0%BE%D0%B2%D0%BB%D0%B5%D0%BD%D0%B8%D0%B5_grub + echo_stamp "Entering to chroot" \ + && chroot ${MOUNT_POINT} /bin/bash + fi + + umount_system ${MOUNT_POINT} ${DEV_IMAGE} +} + +umount_system() { + # TEMPLATE: umount_system + + echo_stamp "Unmount chroot rootfs and boot partition: $1" + umount_ok=false + # Repeat 5 times + for i in {1..5}; do + umount -fR $1 \ + && (echo_stamp "OK" "SUCCESS"; umount_ok=true; break) + || (echo_stamp "Failed #$i (try 5 times)" "ERROR"; sleep 2) + done + [[ "$umount_ok" == true ]] \ + || (echo_stamp "Umount loop-image was failed" "ERROR"; exit 1) + losetup -d $2 +} + +copy_to_chroot() { + # copy_to_chroot + + echo_stamp "Mount loop-image: $1" + local DEV_IMAGE=$(losetup -Pf $1 --show) + sleep 0.5 + + local MOUNT_POINT=$(mktemp -d --suffix=.builder_image) + echo_stamp "Mount dirs ${MOUNT_POINT} & ${MOUNT_POINT}/boot" + mount "${DEV_IMAGE}p2" ${MOUNT_POINT} + mount "${DEV_IMAGE}p1" ${MOUNT_POINT}/boot + + local dir_name=$(dirname "${MOUNT_POINT}$3 /") + + [[ ! -d ${dir_name} ]] && mkdir -p ${dir_name} \ + && echo_stamp "Created ${dir_name}" "SUCCESS" + + cp -r "$2" "${MOUNT_POINT}$3" + umount_system ${MOUNT_POINT} ${DEV_IMAGE} +} + +if [ $(whoami) != "root" ]; then + echo "" + echo "********************************************************************" + echo "******************** This should be run as root ********************" + echo "********************************************************************" + echo "" + exit 1 +fi + +if [[ $# > 0 ]]; then + echo "================================================================================" + for ((i=1; i<=$#; i++)); do echo "\$$i: ${!i}"; done + echo "================================================================================" + + [[ -f $1 ]] || (echo_stamp "$1 does not exist" "ERROR"; exit 1) + + if [[ -z $2 ]] && [[ -f $3 ]]; then + case "$2" in + exec) + execute $1 $3 ${@:4};; + copy) + copy_to_chroot $1 $3 $4;; + *) + echo "Template: image-chroot.sh [ exec