image_builder: new structure of builder

This commit is contained in:
Artem Smirnov
2018-09-24 16:56:39 +03:00
parent c25a752b20
commit ccef57f311
12 changed files with 486 additions and 475 deletions

View File

@@ -1,134 +0,0 @@
#! /usr/bin/env bash
#
# Script for image configure
# @urpylka Artem Smirnov
#
# Exit immidiately on non-zero result
set -e
echo_bold() {
# TEMPLATE: echo_bold <TEXT> <TYPE>
# TYPE: SUCCESS, ERROR, INFO
# More info there https://www.shellhacks.com/ru/bash-colors/
TEXT="$1"
TEXT="\e[1m$TEXT\e[0m" # BOLD
case "$2" in
SUCCESS)
TEXT="\e[32m${TEXT}\e[0m";; # GREEN
ERROR)
TEXT="\e[31m${TEXT}\e[0m";; # RED
*)
TEXT="\e[34m${TEXT}\e[0m";; # BLUE
esac
echo -e ${TEXT}
}
if [ $(whoami) != "root" ]; then
echo \
&& echo "********************************************************************" \
&& echo "******************** This should be run as root ********************" \
&& echo "********************************************************************" \
&& echo \
&& exit 1
fi
if [[ -z $1 ]]; then
echo "================================================================================"
echo_bold "Automatic Image file resizer"
echo_bold "Description: This script shrink your image to 10MiB free space"
echo_bold "if you didn't set FREE_SPACE in MiB (see usage below)."
echo_bold "Authors: Artem Smirnov @urpylka, SirLagz"
echo
echo_bold "Usage: ./autosizer.sh IMAGE_PATH FREE_SPACE"
echo
echo_bold "Requirements: parted, losetup, e2fsck, resize2fs, bc, truncate"
echo "================================================================================"
exit 0
fi
echo "================================================================================"
strImgFile=$1
echo_bold "Path to image: $strImgFile"
echo "================================================================================"
if [[ ! -e $strImgFile ]]; then
echo_bold "Error: File doesn't exist"
echo
exit 1
fi
echo "================================================================================"
partinfo=`parted -m $strImgFile unit B print`
echo_bold "Partition information:\n$partinfo"
echo "================================================================================"
partnumber=`echo "$partinfo" | grep ext4 | awk -F: '{ print $1 }'`
echo_bold "Partition number: $partnumber"
echo "================================================================================"
partstart=`echo "$partinfo" | grep ext4 | awk -F: '{ print substr($2,0,length($2)-1) }'`
echo_bold "Partition start: $partstart (bytes)"
echo "================================================================================"
loopback=`losetup -f --show -o $partstart $strImgFile`
echo_bold "Loopback device: $loopback"
echo "================================================================================"
set +e
e2fsck -fvy $loopback
set -e
echo "================================================================================"
minsize=`resize2fs -P $loopback | awk -F': ' '{ print $2 }'`
#minsize=`resize2fs -P $loopback 2> /dev/null | awk -F': ' '{ print $2 }'`
echo_bold "Minsize: $minsize (4KiB)"
echo "================================================================================"
# Default add 10MiB free space to image, if $2 doesn't set
FREE_SPACE=${2:-10}
FREE_SPACE=$(($FREE_SPACE*1024*1024/4096))
minsize=`echo "$minsize+$FREE_SPACE" | bc`
echo_bold "Minsize + $FREE_SPACE (4KiB): $minsize (4KiB)"
echo "================================================================================"
resize2fs -p $loopback $minsize
sleep 1
losetup -d $loopback
echo "================================================================================"
partnewsize=`echo "$minsize * 4096" | bc`
echo_bold "New size of part: $minsize (4KiB) = $partnewsize (bytes)"
echo "================================================================================"
newpartend=`echo "$partstart + $partnewsize" | bc`
echo_bold "New end of part (Part start + part new size):"
echo_bold "$partstart (bytes) + $partnewsize (bytes) = $newpartend (bytes)"
echo "================================================================================"
part1=`parted $strImgFile rm 2`
echo "================================================================================"
part2=`parted $strImgFile unit B mkpart primary $partstart $newpartend`
echo "================================================================================"
endresult=`parted -m $strImgFile unit B print free | tail -1 | awk -F: '{ print substr($2,0,length($2)-1) }'`
echo_bold "Size of result image: $endresult (bytes)"
echo "================================================================================"
truncate -s $endresult $strImgFile
echo "================================================================================"
partinfo=`parted -m $strImgFile unit B print`
echo_bold "Partition information:\n$partinfo"
echo "================================================================================"
# TODO check if image needs to change PARTUUID
#sed -i 's/root=[^ ]*/root=\/dev\/mmcblk0p2/' /boot/cmdline.txt
#sed -i 's/.* \/boot vfat defaults 0 2$/\/dev\/mmcblk0p1 \/boot vfat defaults 0 2/' /etc/fstab
#sed -i 's/.* \/ ext4 defaults,noatime 0 1$/\/dev\/mmcblk0p2 \/ ext4 defaults,noatime 0 1/' /etc/fstab

View File

@@ -1,26 +0,0 @@
#! /usr/bin/env bash
#
# Script for image configure
# @urpylka Artem Smirnov
#
echo_stamp() {
# TEMPLATE: echo_stamp <TEXT> <TYPE>
# TYPE: SUCCESS, ERROR, INFO
# More info there https://www.shellhacks.com/ru/bash-colors/
TEXT="$(date) | $1"
TEXT="\e[1m$TEXT\e[0m" # BOLD
case "$2" in
SUCCESS)
TEXT="\e[32m${TEXT}\e[0m";; # GREEN
ERROR)
TEXT="\e[31m${TEXT}\e[0m";; # RED
*)
TEXT="\e[34m${TEXT}\e[0m";; # BLUE
esac
echo -e ${TEXT}
}

View File

@@ -1,32 +0,0 @@
#! /usr/bin/env bash
#
# Script for image configure
# @urpylka Artem Smirnov
# @dvornikov-aa Andrey Dvornikov
#
# Exit immidiately on non-zero result
set -e
source echo_stamp.sh
get_image() {
# TEMPLATE: get_image <IMAGE_PATH> <RPI_DONWLOAD_URL>
local BUILD_DIR=$(dirname $1)
local RPI_ZIP_NAME=$(basename $2)
if [ ! -e "${BUILD_DIR}/${RPI_ZIP_NAME}" ];
then
echo_stamp "1. Downloading original Linux distribution"
wget -nv -O ${BUILD_DIR}/${RPI_ZIP_NAME} $2 \
&& echo_stamp "Downloading complete" "SUCCESS"
else
echo_stamp "1. Linux distribution already donwloaded"
fi
echo_stamp "2. Unzipping Linux distribution image"
local RPI_IMAGE_NAME=$(echo ${RPI_ZIP_NAME} | sed 's/zip/img/')
unzip -p ${BUILD_DIR}/${RPI_ZIP_NAME} ${RPI_IMAGE_NAME} > $1
echo_stamp "Unzipping complete" "SUCCESS"
}
get_image $1 $2

View File

@@ -1,12 +1,17 @@
#! /usr/bin/env bash
#
# Script for image configure
# @urpylka Artem Smirnov
# Script for build the image. Used builder script of the target repo
# Copyright (C) 2018 Copter Express Technologies
#
# Author: Artem Smirnov <urpylka@gmail.com>
#
# Distributed under MIT License (available at https://opensource.org/licenses/MIT).
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# Exit immidiately on non-zero result
set -e
set -e # Exit immidiately on non-zero result
export DEBIAN_FRONTEND=${DEBIAN_FRONTEND:='noninteractive'}
export LANG=${LANG:='C.UTF-8'}
@@ -22,6 +27,25 @@ echo "TARGET_REF: $TARGET_REF"
echo "TARGET_CONFIG: $TARGET_CONFIG"
echo "================================================================================"
get_image() {
# TEMPLATE: get_image <IMAGE_PATH> <RPI_DONWLOAD_URL>
local BUILD_DIR=$(dirname $1)
local RPI_ZIP_NAME=$(basename $2)
local RPI_IMAGE_NAME=$(echo ${RPI_ZIP_NAME} | sed 's/zip/img/')
if [ ! -e "${BUILD_DIR}/${RPI_ZIP_NAME}" ]; then
echo_stamp "Downloading original Linux distribution" \
&& wget -nv -O ${BUILD_DIR}/${RPI_ZIP_NAME} $2 \
&& echo_stamp "Downloading complete" "SUCCESS" \
|| (echo_stamp "Downloading was failed!" "ERROR"; exit 1)
else; echo_stamp "Linux distribution already donwloaded"; fi
echo_stamp "Unzipping Linux distribution image" \
&& unzip -p ${BUILD_DIR}/${RPI_ZIP_NAME} ${RPI_IMAGE_NAME} > $1 \
&& echo_stamp "Unzipping complete" "SUCCESS" \
|| (echo_stamp "Unzipping was failed!" "ERROR"; exit 1)
}
# TODO: The repository can be already downloaded, use the TARGET_REPO also as unix path.
REPO_DIR=$(mktemp -d --suffix=.builder_repo)
git clone ${TARGET_REPO} --single-branch --branch ${TARGET_REF} --depth 1 ${REPO_DIR} \
@@ -36,13 +60,13 @@ cd ${CUR_DIR} && unset CUR_DIR
export IMAGE_VERSION="${TARGET_REF}_${TARGET_COMMIT}"
export IMAGE_PATH="$(pwd)/image/$(basename -s '.git' ${TARGET_REPO})_${IMAGE_VERSION}.img"
./get_image.sh ${IMAGE_PATH} $(jq '.source_image' -r ${TARGET_CONFIG})
get_image ${IMAGE_PATH} $(jq '.source_image' -r ${TARGET_CONFIG})
REGISTER=':arm:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/qemu-arm-static:'
if [[ $(arch) != 'armv7l' ]]; then
mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc 2> /dev/null || true
echo ${REGISTER} > /proc/sys/fs/binfmt_misc/register 2> /dev/null || true
./image_config.sh copy_to_chroot ${IMAGE_PATH} './qemu-arm-resin' '/usr/bin/qemu-arm-static'
./image-chroot.sh ${IMAGE_PATH} copy './qemu-arm-resin' '/usr/bin/qemu-arm-static'
fi
export IMAGE_BUILDER="$(dirname $(readlink -e "$0"))"

184
builder_docker/image-chroot.sh Executable file
View File

@@ -0,0 +1,184 @@
#! /usr/bin/env bash
#
# Script for upload the image to yadisk & change the release message on GitHub
# Copyright (C) 2018 Copter Express Technologies
#
# Author: Artem Smirnov <urpylka@gmail.com>
# Author: Andrey Dvornikov <dvornikov-aa@yandex.ru>
#
# Distributed under MIT License (available at https://opensource.org/licenses/MIT).
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
set -e # Exit immidiately on non-zero result
echo_stamp() {
# TEMPLATE: echo_stamp <TEXT> <TYPE>
# TYPE: SUCCESS, ERROR, INFO
# More info there https://www.shellhacks.com/ru/bash-colors/
TEXT="$(date '+[%Y-%m-%d %H:%M:%S]') $1"
TEXT="\e[1m$TEXT\e[0m" # BOLD
case "$2" in
SUCCESS)
TEXT="\e[32m${TEXT}\e[0m";; # GREEN
ERROR)
TEXT="\e[31m${TEXT}\e[0m";; # RED
*)
TEXT="\e[34m${TEXT}\e[0m";; # BLUE
esac
echo -e ${TEXT}
}
# This script doesn't work on Ubuntu because OS`s losetup does not consist --partscan (-P).
# Idea: use `mount -o loop,offset`
# https://stefanoprenna.com/blog/2014/09/22/tutorial-how-to-mount-raw-images-img-images-on-linux/
# REPO_DIR=$(mktemp -d --suffix=.builder_repo)
# mount -t ext4 -o loop,offset=$((94208 * 512)) image/clever_qemu_test_2_20180822_163141.img "$REPO_DIR"
# mount -t vfat -o loop,offset=$((8192 * 512)) image/clever_qemu_test_2_20180822_163141.img "$REPO_DIR/boot"
execute() {
# execute <IMAGE_PATH> [<EXECUTE_FILE> [...]]
echo_stamp "Mount loop-image: $1"
local DEV_IMAGE=$(losetup -Pf $1 --show)
sleep 0.5
local MOUNT_POINT=$(mktemp -d --suffix=.builder_image)
echo_stamp "Mount dirs ${MOUNT_POINT} & ${MOUNT_POINT}/boot"
mount "${DEV_IMAGE}p2" ${MOUNT_POINT}
mount "${DEV_IMAGE}p1" ${MOUNT_POINT}/boot
echo_stamp "Bind system dirs"
echo_stamp "Mounting /proc in chroot... "
if [ ! -d ${MOUNT_POINT}/proc ]; then
mkdir -p ${MOUNT_POINT}/proc \
&& mount -t proc -o nosuid,noexec,nodev proc ${MOUNT_POINT}/proc
&& echo_stamp "OK" "SUCCESS" \
|| (echo_stamp "Failed" "ERROR"; exit 1)
else echo_stamp "/sys already exist" "SUCCESS"
fi
echo_stamp "Mounting /sys in chroot... "
if [ ! -d ${MOUNT_POINT}/sys ]; then
mkdir -p ${MOUNT_POINT}/sys \
&& mount -t sysfs -o nosuid,noexec,nodev sysfs ${MOUNT_POINT}/sys \
&& echo_stamp "OK" "SUCCESS" \
|| (echo_stamp "Failed" "ERROR"; exit 1)
else echo_stamp "/sys already exist" "SUCCESS"
fi
echo_stamp "Mounting /dev/ and /dev/pts in chroot... " \
&& mkdir -p -m 755 ${MOUNT_POINT}/dev/pts \
&& mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${MOUNT_POINT}/dev \
&& mount -t devpts -o gid=5,mode=620 devpts ${MOUNT_POINT}/dev/pts \
&& echo_stamp "OK" "SUCCESS" \
|| (echo_stamp "Failed" "ERROR"; exit 1)
echo_stamp "Copy DNS records" \
&& cp -L /etc/resolv.conf ${MOUNT_POINT}/etc/resolv.conf \
&& echo_stamp "OK" "SUCCESS" \
|| (echo_stamp "Failed" "ERROR"; exit 1)
if [[ $# > 1 ]]; then
echo_stamp "Copy script into chroot fs"
local SCRIPT_NAME=$(basename $2)
local SCRIPT_PATH="$(mktemp -d -p ${MOUNT_POINT}/tmp --suffix=.tmp_builder_script)"
local script_name=$(basename $2)
local script_path_root="${MOUNT_POINT}/root/${script_name}"
# TODO: maybe copy to tmp-dir
# TODO: Find more suitable location for temporary script storage
# $(mktemp -p ${MOUNT_POINT}/tmp --suffix=.tmp_builder_script)
cp "$2" "${script_path_root}"
# Run script in chroot with additional arguments
chroot ${MOUNT_POINT} /bin/sh -c "/root/${script_name} ${@:3}"
# Removing script from chroot fs
rm "${script_path_root}"
else
# https://wiki.archlinux.org/index.php/Change_root_(%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9)
# http://www.unix-lab.org/posts/chroot/
# https://habrahabr.ru/post/141012/
# https://losst.ru/vosstanovlenie-grub2
# http://unixteam.ru/content/virtualizaciya-ili-zapuskaem-prilozhenie-v-chroot-okruzhenii-razmyshleniya
# http://help.ubuntu.ru/wiki/%D0%B2%D0%BE%D1%81%D1%81%D1%82%D0%B0%D0%BD%D0%BE%D0%B2%D0%BB%D0%B5%D0%BD%D0%B8%D0%B5_grub
echo_stamp "Entering to chroot" \
&& chroot ${MOUNT_POINT} /bin/bash
fi
umount_system ${MOUNT_POINT} ${DEV_IMAGE}
}
umount_system() {
# TEMPLATE: umount_system <MOUNT_POINT> <DEV_IMAGE>
echo_stamp "Unmount chroot rootfs and boot partition: $1"
umount_ok=false
# Repeat 5 times
for i in {1..5}; do
umount -fR $1 \
&& (echo_stamp "OK" "SUCCESS"; umount_ok=true; break)
|| (echo_stamp "Failed #$i (try 5 times)" "ERROR"; sleep 2)
done
[[ "$umount_ok" == true ]] \
|| (echo_stamp "Umount loop-image was failed" "ERROR"; exit 1)
losetup -d $2
}
copy_to_chroot() {
# copy_to_chroot <IMAGE_PATH> <MOVE_FILE> <MOVE_TO>
echo_stamp "Mount loop-image: $1"
local DEV_IMAGE=$(losetup -Pf $1 --show)
sleep 0.5
local MOUNT_POINT=$(mktemp -d --suffix=.builder_image)
echo_stamp "Mount dirs ${MOUNT_POINT} & ${MOUNT_POINT}/boot"
mount "${DEV_IMAGE}p2" ${MOUNT_POINT}
mount "${DEV_IMAGE}p1" ${MOUNT_POINT}/boot
local dir_name=$(dirname "${MOUNT_POINT}$3 /")
[[ ! -d ${dir_name} ]] && mkdir -p ${dir_name} \
&& echo_stamp "Created ${dir_name}" "SUCCESS"
cp -r "$2" "${MOUNT_POINT}$3"
umount_system ${MOUNT_POINT} ${DEV_IMAGE}
}
if [ $(whoami) != "root" ]; then
echo ""
echo "********************************************************************"
echo "******************** This should be run as root ********************"
echo "********************************************************************"
echo ""
exit 1
fi
if [[ $# > 0 ]]; then
echo "================================================================================"
for ((i=1; i<=$#; i++)); do echo "\$$i: ${!i}"; done
echo "================================================================================"
[[ -f $1 ]] || (echo_stamp "$1 does not exist" "ERROR"; exit 1)
if [[ -z $2 ]] && [[ -f $3 ]]; then
case "$2" in
exec)
execute $1 $3 ${@:4};;
copy)
copy_to_chroot $1 $3 $4;;
*)
echo "Template: image-chroot.sh <IMAGE> [ exec <SCRIPT> [...] | copy <MOVE_FILE> <MOVE_TO> ]";;
esac
else execute $1; fi
fi

View File

@@ -1,15 +1,17 @@
#! /usr/bin/env bash
#
# Script for image configure
# @urpylka Artem Smirnov
# @dvornikov-aa Andrey Dvornikov
# Script for upload the image to yadisk & change the release message on GitHub
# Copyright (C) 2018 Copter Express Technologies
#
# Author: Artem Smirnov <urpylka@gmail.com>
#
# Distributed under MIT License (available at https://opensource.org/licenses/MIT).
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# Exit immidiately on non-zero result
set -e
source echo_stamp.sh
set -e # Exit immidiately on non-zero result
publish_image() {
# TEMPLATE: publish_image_bash <IMAGE_PATH> <YA_SCRIPT> <CONFIG_FILE> <RELEASE_ID> <RELEASE_BODY>
@@ -19,34 +21,33 @@ publish_image() {
IMAGE_NAME=$(basename $1)
BUILD_DIR=$(dirname $1)
echo_stamp "Zip image"
if [ ! -e "${BUILD_DIR}/${IMAGE_NAME}.zip" ];
then
echo -e "Zip image"
if [ ! -e "${BUILD_DIR}/${IMAGE_NAME}.zip" ]; then
cd ${BUILD_DIR} && zip ${IMAGE_NAME}.zip ${IMAGE_NAME}
echo_stamp "Zipping complete!" "SUCCESS"
echo -e "Zipping complete!" "SUCCESS"
else
echo_stamp "Zip-archive already created"
echo -e "Zip-archive already created"
cd ${BUILD_DIR} && rm ${IMAGE_NAME}.zip && zip ${IMAGE_NAME}.zip ${IMAGE_NAME} \
&& echo_stamp "Old archive was deleted & create new" "SUCCESS"
&& echo -e "Old archive was deleted & create new" "SUCCESS"
fi
echo_stamp "Upload image"
echo -e "Upload image"
local IMAGE_LINK=$($2 $3 ${BUILD_DIR}/${IMAGE_NAME}.zip) \
&& echo_stamp "Upload copmlete!" "SUCCESS"
&& echo -e "Upload copmlete!" "SUCCESS"
echo_stamp "Meashure size of zip-image"
echo -e "Meashure size of zip-image"
local IMAGE_SIZE=$(du -sh ${BUILD_DIR}/${IMAGE_NAME}.zip | awk '{ print $1 }') \
&& echo_stamp "Meashuring copmlete!" "SUCCESS"
&& echo -e "Meashuring copmlete!" "SUCCESS"
echo_stamp "Meashure hash-sum of zip-image"
echo -e "Meashure hash-sum of zip-image"
local IMAGE_HASH=$(sha256sum ${BUILD_DIR}/${IMAGE_NAME}.zip | awk '{ print $1 }') \
&& echo_stamp "Meashuring copmlete!" "SUCCESS"
&& echo -e "Meashuring copmlete!" "SUCCESS"
echo ""
echo "\$5: $5"
echo ""
echo_stamp "Post message to GH"
echo -e "Post message to GH"
local NEW_RELEASE_BODY="### Download\n* [${IMAGE_NAME}.zip](${IMAGE_LINK}) (${IMAGE_SIZE})\nsha256: ${IMAGE_HASH}\n\n$5"
local DATA="{ \"body\":\"${NEW_RELEASE_BODY}\" }"
@@ -58,7 +59,7 @@ publish_image() {
local GH_PASS=$(cat $3 | jq '.github.password' -r)
local GH_URL=$(cat $3 | jq '.github.url' -r)
curl -d "$DATA" -u "${GH_LOGIN}:${GH_PASS}" --request PATCH ${GH_URL}$4 \
&& echo_stamp "Post message to GH copmlete!" "SUCCESS"
&& echo -e "Post message to GH copmlete!" "SUCCESS"
}
publish_image $1 $2 $3 $4 "$5"

203
builder_docker/image-resize.sh Executable file
View File

@@ -0,0 +1,203 @@
#! /usr/bin/env bash
#
# Script for upsize the image & for shrink free space on the image
# Copyright (C) 2018 Copter Express Technologies
#
# Author: Artem Smirnov <urpylka@gmail.com>
#
# Distributed under MIT License (available at https://opensource.org/licenses/MIT).
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
set -e # Exit immidiately on non-zero result
echo_bold() {
# TEMPLATE: echo_bold <TEXT> <TYPE>
# TYPE: SUCCESS, ERROR, INFO
# More info there https://www.shellhacks.com/ru/bash-colors/
TEXT="$1"
TEXT="\e[1m$TEXT\e[0m" # BOLD
case "$2" in
SUCCESS)
TEXT="\e[32m${TEXT}\e[0m";; # GREEN
ERROR)
TEXT="\e[31m${TEXT}\e[0m";; # RED
*)
TEXT="\e[34m${TEXT}\e[0m";; # BLUE
esac
echo -e ${TEXT}
}
resize_fs() {
# TEMPLATE: resize_fs <IMAGE_PATH> <SIZE>
set +e
# https://ru.wikipedia.org/wiki/%D0%A0%D0%B0%D0%B7%D1%80%D0%B5%D0%B6%D1%91%D0%BD%D0%BD%D1%8B%D0%B9_%D1%84%D0%B0%D0%B9%D0%BB
# https://raspberrypi.stackexchange.com/questions/13137/how-can-i-mount-a-raspberry-pi-linux-distro-image
# fdisk -l 2017-11-29-raspbian-stretch-lite.img
# https://www.stableit.ru/2011/05/losetup.html
# -f : losetup сам выбрал loop (минуя занятые)
# -P : losetup монтирует разделы в образе как отдельные подразделы,
# например /dev/loop0p1 и /dev/loop0p2
# --show : печатает имя устройства, например /dev/loop4
# http://karelzak.blogspot.ru/2015/05/resize-by-sfdisk.html
# ", +" : expand partition for volume size
# -N 2 : select second partition for work
# There is a risk that sfdisk will ask for a disk remount to update partition table
# TODO: Check sfdisk exit code
echo_bold "Truncate image" \
&& truncate -s$2 $1 \
&& echo_bold "Mount loop-image: $1" \
&& local DEV_IMAGE=$(losetup -Pf $1 --show) \
&& sleep 0.5 \
&& echo ", +" | sfdisk -N 2 ${DEV_IMAGE} \
&& sleep 0.5 \
&& losetup -d ${DEV_IMAGE} \
&& sleep 0.5 \
&& local DEV_IMAGE=$(losetup -Pf $1 --show) \
&& sleep 0.5 \
&& echo_bold "Check & repair filesystem after expand partition" \
&& e2fsck -fvy "${DEV_IMAGE}p2" \
&& echo_bold "Expand filesystem" \
&& resize2fs "${DEV_IMAGE}p2" \
&& echo_bold "Umount loop-image" \
&& losetup -d ${DEV_IMAGE}
set -e
}
shrink_free_space() {
if [[ -z $1 ]]; then
echo "================================================================================"
echo_bold "Automatic Image file resizer"
echo_bold "Description: This script shrink your image to 10MiB free space"
echo_bold "if you didn't set FREE_SPACE in MiB (see usage below)."
echo_bold "Authors: Artem Smirnov @urpylka, SirLagz"
echo
echo_bold "Usage: ./autosizer.sh <IMAGE_PATH> [<FREE_SPACE>]"
echo
echo_bold "Requirements: parted, losetup, e2fsck, resize2fs, bc, truncate"
echo "================================================================================"
exit 1
fi
echo "================================================================================"
# Default add 10MiB free space to image, if $2 doesn't set
FREE_SPACE=${2:-10}
strImgFile=$1
echo_bold "Path to image: $strImgFile"
echo "================================================================================"
if [[ ! -e $strImgFile ]]; then
echo_bold "Error: File doesn't exist"
echo
exit 1
fi
echo "================================================================================"
partinfo=`parted -m $strImgFile unit B print`
echo_bold "Partition information:\n$partinfo"
echo "================================================================================"
partnumber=`echo "$partinfo" | grep ext4 | awk -F: '{ print $1 }'`
echo_bold "Partition number: $partnumber"
echo "================================================================================"
partstart=`echo "$partinfo" | grep ext4 | awk -F: '{ print substr($2,0,length($2)-1) }'`
echo_bold "Partition start: $partstart (bytes)"
echo "================================================================================"
loopback=`losetup -f --show -o $partstart $strImgFile`
echo_bold "Loopback device: $loopback"
echo "================================================================================"
set +e
e2fsck -fvy $loopback
set -e
echo "================================================================================"
minsize=`resize2fs -P $loopback | awk -F': ' '{ print $2 }'`
#minsize=`resize2fs -P $loopback 2> /dev/null | awk -F': ' '{ print $2 }'`
echo_bold "Minsize: $minsize (4KiB)"
echo "================================================================================"
FREE_SPACE=$(($FREE_SPACE*1024*1024/4096))
minsize=`echo "$minsize+$FREE_SPACE" | bc`
echo_bold "Minsize + $FREE_SPACE (4KiB): $minsize (4KiB)"
echo "================================================================================"
resize2fs -p $loopback $minsize
sleep 1
losetup -d $loopback
echo "================================================================================"
partnewsize=`echo "$minsize * 4096" | bc`
echo_bold "New size of part: $minsize (4KiB) = $partnewsize (bytes)"
echo "================================================================================"
newpartend=`echo "$partstart + $partnewsize" | bc`
echo_bold "New end of part (Part start + part new size):"
echo_bold "$partstart (bytes) + $partnewsize (bytes) = $newpartend (bytes)"
echo "================================================================================"
part1=`parted $strImgFile rm 2`
echo "================================================================================"
part2=`parted $strImgFile unit B mkpart primary $partstart $newpartend`
echo "================================================================================"
endresult=`parted -m $strImgFile unit B print free | tail -1 | awk -F: '{ print substr($2,0,length($2)-1) }'`
echo_bold "Size of result image: $endresult (bytes)"
echo "================================================================================"
truncate -s $endresult $strImgFile
echo "================================================================================"
partinfo=`parted -m $strImgFile unit B print`
echo_bold "Partition information:\n$partinfo"
echo "================================================================================"
# TODO check if image needs to change PARTUUID
#sed -i 's/root=[^ ]*/root=\/dev\/mmcblk0p2/' /boot/cmdline.txt
#sed -i 's/.* \/boot vfat defaults 0 2$/\/dev\/mmcblk0p1 \/boot vfat defaults 0 2/' /etc/fstab
#sed -i 's/.* \/ ext4 defaults,noatime 0 1$/\/dev\/mmcblk0p2 \/ ext4 defaults,noatime 0 1/' /etc/fstab
}
if [ $(whoami) != "root" ]; then
echo ""
echo "********************************************************************"
echo "******************** This should be run as root ********************"
echo "********************************************************************"
echo ""
exit 1
fi
if [[ $# > 0 ]]; then
echo "================================================================================"
for ((i=1; i<=$#; i++)); do echo "\$$i: ${!i}"; done
echo "================================================================================"
[[ -f $1 ]] || (echo_bold "$1 does not exist" "ERROR"; exit 1)
if [[ -z $2 ]]; then
case "$2" in
min)
shrink_free_space $1 $3;;
max)
resize_fs $1 $3;;
*)
echo "Template: image-resize.sh <IMAGE> [ min <FREE_SPACE> | max <FREE_SPACE> ]";;
esac
else shrink_free_space $1; fi
fi

View File

@@ -1,168 +0,0 @@
#! /usr/bin/env bash
#
# Script for image configure
# @urpylka Artem Smirnov
# @dvornikov-aa Andrey Dvornikov
#
# Exit immidiately on non-zero result
set -e
source echo_stamp.sh
# This script doesn't work on Ubuntu because OS`s losetup does not consist --partscan (-P).
# Idea: use `mount -o loop,offset`
# https://stefanoprenna.com/blog/2014/09/22/tutorial-how-to-mount-raw-images-img-images-on-linux/
# REPO_DIR=$(mktemp -d --suffix=.builder_repo)
# mount -t ext4 -o loop,offset=$((94208 * 512)) image/clever_qemu_test_2_20180822_163141.img "$REPO_DIR"
# mount -t vfat -o loop,offset=$((8192 * 512)) image/clever_qemu_test_2_20180822_163141.img "$REPO_DIR/boot"
execute() {
# TEMPLATE: execute <IMAGE_PATH> <EXECUTE_FILE> <...>
echo_stamp "Mount loop-image: $1"
local DEV_IMAGE=$(losetup -Pf $1 --show)
sleep 0.5
local MOUNT_POINT=$(mktemp -d --suffix=.builder_image)
echo_stamp "Mount dirs ${MOUNT_POINT} & ${MOUNT_POINT}/boot"
mount "${DEV_IMAGE}p2" ${MOUNT_POINT}
mount "${DEV_IMAGE}p1" ${MOUNT_POINT}/boot
echo_stamp "Bind system dirs"
echo_stamp "Mounting /proc in chroot... "
if [ ! -d ${MOUNT_POINT}/proc ] ; then
mkdir -p ${MOUNT_POINT}/proc \
&& echo_stamp "Created ${MOUNT_POINT}/proc" "SUCCESS"
fi
mount -t proc -o nosuid,noexec,nodev proc ${MOUNT_POINT}/proc \
&& echo_stamp "OK" "SUCCESS"
echo_stamp "Mounting /sys in chroot... "
if [ ! -d ${MOUNT_POINT}/sys ] ; then
mkdir -p ${MOUNT_POINT}/sys \
&& echo_stamp "Created ${MOUNT_POINT}/sys" "SUCCESS"
fi
mount -t sysfs -o nosuid,noexec,nodev sysfs ${MOUNT_POINT}/sys \
&& echo_stamp "OK" "SUCCESS"
echo_stamp "Mounting /dev/ and /dev/pts in chroot... " \
&& mkdir -p -m 755 ${MOUNT_POINT}/dev/pts \
&& mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${MOUNT_POINT}/dev \
&& mount -t devpts -o gid=5,mode=620 devpts ${MOUNT_POINT}/dev/pts \
&& echo_stamp "OK" "SUCCESS"
echo_stamp "Copy DNS records" \
&& cp -L /etc/resolv.conf ${MOUNT_POINT}/etc/resolv.conf
if [[ $# > 1 ]]; then
echo_stamp "Entering to chroot"
local script_name=$(basename $2)
# TODO: maybe copy to tmp-dir
local script_path_root="${MOUNT_POINT}/root/${script_name}"
# Copy script into chroot fs
# TODO: Find more suitable location for temporary script storage
cp "$2" "${script_path_root}"
# Its important to save arguments (direct ${@:4} causes problems)
script_args="${@:3}"
# Run script in chroot with additional arguments
chroot ${MOUNT_POINT} /bin/sh -c "/root/${script_name} ${script_args}"
# Removing script from chroot fs
rm "${script_path_root}"
else
# https://wiki.archlinux.org/index.php/Change_root_(%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9)
# http://www.unix-lab.org/posts/chroot/
# https://habrahabr.ru/post/141012/
# https://losst.ru/vosstanovlenie-grub2
# http://unixteam.ru/content/virtualizaciya-ili-zapuskaem-prilozhenie-v-chroot-okruzhenii-razmyshleniya
# http://help.ubuntu.ru/wiki/%D0%B2%D0%BE%D1%81%D1%81%D1%82%D0%B0%D0%BD%D0%BE%D0%B2%D0%BB%D0%B5%D0%BD%D0%B8%D0%B5_grub
echo_stamp "Entering to chroot" \
&& chroot ${MOUNT_POINT} /bin/bash
fi
umount_system ${MOUNT_POINT} ${DEV_IMAGE}
}
copy_to_chroot() {
# TEMPLATE: copy_to_chroot <IMAGE_PATH> <MOVE_FILE> <MOVE_TO>
echo_stamp "Mount loop-image: $1"
local DEV_IMAGE=$(losetup -Pf $1 --show)
sleep 0.5
local MOUNT_POINT=$(mktemp -d --suffix=.builder_image)
echo_stamp "Mount dirs ${MOUNT_POINT} & ${MOUNT_POINT}/boot"
mount "${DEV_IMAGE}p2" ${MOUNT_POINT}
mount "${DEV_IMAGE}p1" ${MOUNT_POINT}/boot
dir_name=$(dirname "${MOUNT_POINT}$3 /")
if [ ! -d ${dir_name} ] ; then
mkdir -p ${dir_name} \
&& echo_stamp "Created ${dir_name}" "SUCCESS"
fi
cp -r "$2" "${MOUNT_POINT}$3"
umount_system ${MOUNT_POINT} ${DEV_IMAGE}
}
umount_system() {
# TEMPLATE: umount_system <MOUNT_POINT> <DEV_IMAGE>
echo_stamp "Umount recursive dirs: $1"
# There is a risk that umount will fail
set +e
# Successfull unmount flag (false at thismoment)
umount_ok=false
# Repeat 5 times
for i in {1..5}
do
# Unmount chroot rootfs and boot partition
umount -fR $1
# If no problems detected
if [[ $? == 0 ]]
then
echo_stamp "Successfull unmount" "SUCCESS"
# Set flag
umount_ok=true
# Exit loop
break
fi
# Unmount has failed
echo_stamp "Unmount failed" "ERROR"
# Wait for some time
sleep 2
done
set -e
# Jenkins job will fail if this condition is not true
[[ "$umount_ok" == true ]]
echo_stamp "Umount loop-image"
losetup -d $2
}
if [ $(whoami) != "root" ];
then echo "" \
&& echo "********************************************************************" \
&& echo "******************** This should be run as root ********************" \
&& echo "********************************************************************" \
&& echo "" \
&& exit 1
fi
echo "================================================================================"
for ((i=1; i<=$#; i++)); do echo "\$$i: ${!i}"; done
echo "================================================================================"
case "$1" in
execute)
# execute <IMAGE_PATH> [<EXECUTE_FILE>] [...]
execute $2 $3 ${@:4};;
copy_to_chroot)
# copy_to_chroot <IMAGE_PATH> <MOVE_FILE> <MOVE_TO>
copy_to_chroot $2 $3 $4;;
*)
echo "Enter one of: get_image, resize_fs, publish_image, execute";;
esac

View File

@@ -1,57 +0,0 @@
#! /usr/bin/env bash
#
# Script for image configure
# @urpylka Artem Smirnov
# @dvornikov-aa Andrey Dvornikov
#
# Exit immidiately on non-zero result
set -e
source echo_stamp.sh
resize_fs() {
# TEMPLATE: resize_fs <IMAGE_PATH> <SIZE>
set +e
# https://ru.wikipedia.org/wiki/%D0%A0%D0%B0%D0%B7%D1%80%D0%B5%D0%B6%D1%91%D0%BD%D0%BD%D1%8B%D0%B9_%D1%84%D0%B0%D0%B9%D0%BB
# https://raspberrypi.stackexchange.com/questions/13137/how-can-i-mount-a-raspberry-pi-linux-distro-image
# fdisk -l 2017-11-29-raspbian-stretch-lite.img
# https://www.stableit.ru/2011/05/losetup.html
# -f : losetup сам выбрал loop (минуя занятые)
# -P : losetup монтирует разделы в образе как отдельные подразделы,
# например /dev/loop0p1 и /dev/loop0p2
# --show : печатает имя устройства, например /dev/loop4
# http://karelzak.blogspot.ru/2015/05/resize-by-sfdisk.html
# ", +" : expand partition for volume size
# -N 2 : select second partition for work
# There is a risk that sfdisk will ask for a disk remount to update partition table
# TODO: Check sfdisk exit code
echo_stamp "Truncate image" \
&& truncate -s$2 $1 \
&& echo_stamp "Mount loop-image: $1" \
&& local DEV_IMAGE=$(losetup -Pf $1 --show) \
&& sleep 0.5 \
&& echo ", +" | sfdisk -N 2 ${DEV_IMAGE} \
&& sleep 0.5 \
&& losetup -d ${DEV_IMAGE} \
&& sleep 0.5 \
&& local DEV_IMAGE=$(losetup -Pf $1 --show) \
&& sleep 0.5 \
&& echo_stamp "Check & repair filesystem after expand partition" \
&& e2fsck -fvy "${DEV_IMAGE}p2" \
&& echo_stamp "Expand filesystem" \
&& resize2fs "${DEV_IMAGE}p2" \
&& echo_stamp "Umount loop-image" \
&& losetup -d ${DEV_IMAGE}
set -e
}
resize_fs $1 $2

View File

@@ -1,11 +1,16 @@
#!/usr/bin/env python
#
# Simple python uploader to YaDisk
# Smirnov Artem @urpylka
# The simple python uploader to YaDisk
# Use: python yadisk.py login password file server_dir
#
# Use:
# python yadisk.py login password file server_dir
# Copyright (C) 2018 Copter Express Technologies
#
# Author: Artem Smirnov <urpylka@gmail.com>
#
# Distributed under MIT License (available at https://opensource.org/licenses/MIT).
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
from YaDiskClient.YaDiskClient import YaDisk