mirror of
https://github.com/CopterExpress/clover.git
synced 2026-05-26 21:19:35 +00:00
image_builder: split builder_docker & builder_scripts skip travis
This commit is contained in:
19
builder_docker/Dockerfile
Normal file
19
builder_docker/Dockerfile
Normal file
@@ -0,0 +1,19 @@
|
||||
# for alpine use apk
|
||||
|
||||
FROM debian
|
||||
|
||||
ENV DEBIAN_FRONTEND 'noninteractive'
|
||||
ENV LANG 'C.UTF-8'
|
||||
ENV LC_ALL 'C.UTF-8'
|
||||
|
||||
RUN apt-get update -qq > /dev/null
|
||||
RUN apt-get install -y --no-install-recommends -qq bc jq unzip wget parted apt-utils git ca-certificates gawk > /dev/null
|
||||
RUN apt-get clean
|
||||
|
||||
COPY ./image_config.sh /builder/image_config.sh
|
||||
COPY ./build.sh /builder/build.sh
|
||||
COPY ./autosizer.sh /builder/autosizer.sh
|
||||
COPY ./qemu-arm-resin /builder/qemu-arm-resin
|
||||
|
||||
WORKDIR /builder
|
||||
CMD ./build.sh
|
||||
35
builder_docker/Jenkinsfile
vendored
Normal file
35
builder_docker/Jenkinsfile
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
pipeline {
|
||||
agent any
|
||||
parameters {
|
||||
string(name: 'GWBT_REF', defaultValue: "master")
|
||||
string(name: 'GWBT_URL', defaultValue: "https://github.com/CopterExpress/clever.git")
|
||||
string(name: 'GWBT_FILE', defaultValue: "")
|
||||
string(name: 'IMAGE_NAME', defaultValue: "\$(cat ${GWBT_FILE} | jq '.repository.name' -r)-${params.GWBT_REF}.img")
|
||||
string(name: 'GWBT_EVENT', defaultValue: 'release')
|
||||
booleanParam(name: 'ONLY_PUBLISH', defaultValue: false, description: 'ONLY PUBLISH')
|
||||
string(name: 'BUILD_DIR', defaultValue: '/mnt/hdd_builder/workspace', description: 'Build workspace')
|
||||
}
|
||||
environment {
|
||||
DEBIAN_FRONTEND = 'noninteractive'
|
||||
LANG = 'C.UTF-8'
|
||||
LC_ALL = 'C.UTF-8'
|
||||
}
|
||||
stages {
|
||||
stage('Build image') {
|
||||
when { not { expression { return params.ONLY_PUBLISH } } }
|
||||
steps {
|
||||
build job: 'CopterExpress-clever-build', parameters: [[$class: 'StringParameterValue', name: 'IMAGE_NAME', value: "${params.IMAGE_NAME}"], [$class: 'StringParameterValue', name: 'IMAGE_VERSION', value: "${params.GWBT_REF}"], [$class: 'StringParameterValue', name: 'GWBT_REF', value: "${params.GWBT_REF}"], [$class: 'StringParameterValue', name: 'GWBT_URL', value: "${params.GWBT_URL}"]]
|
||||
}
|
||||
}
|
||||
stage('Publish image') {
|
||||
when { environment name: 'GWBT_EVENT', value: 'release' }
|
||||
environment {
|
||||
CONFIG_FILE = "${params.BUILD_DIR}/coex-ci.json"
|
||||
YA_SCRIPT = "$WORKSPACE/image_builder/yadisk.py"
|
||||
}
|
||||
steps {
|
||||
sh "$WORKSPACE/image_builder/image_config.sh publish_image ${params.BUILD_DIR} ${params.IMAGE_NAME} ${YA_SCRIPT} ${CONFIG_FILE} \$(cat ${params.GWBT_FILE} | jq '.release.id' -r) \"\$(cat ${params.GWBT_FILE} | jq '.release.body' | sed 's/\"//' | rev | sed 's/\"//' | rev)\""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
112
builder_docker/README.md
Normal file
112
builder_docker/README.md
Normal file
@@ -0,0 +1,112 @@
|
||||
## Setup your builder
|
||||
|
||||
1. Install requirements
|
||||
```bash
|
||||
sudo apt-get install unzip zip git python-pip jq curl
|
||||
sudo pip install YaDiskClient
|
||||
```
|
||||
2. Mount HDD
|
||||
```bash
|
||||
nano /etc/fstab
|
||||
```
|
||||
```
|
||||
proc /proc proc defaults 0 0
|
||||
PARTUUID=37665771-01 /boot vfat defaults 0 2
|
||||
PARTUUID=37665771-02 / ext4 defaults,noatime 0 1
|
||||
# a swapfile is not a swap partition, no line here
|
||||
# use dphys-swapfile swap[on|off] for that
|
||||
/dev/sdb1 none swap sw 0 0
|
||||
/dev/sdb2 /mnt/hdd_system ext4 defaults,acl 0 0
|
||||
/dev/sdb3 /mnt/hdd_builder ext4 defaults,acl 0 0
|
||||
```
|
||||
|
||||
3. Enable swap on HDD
|
||||
> TODO
|
||||
4. And disable `dphys-swapfile`
|
||||
```bash
|
||||
sudo systemctl stop dphys-swapfile
|
||||
sudo systemctl disable dphys-swapfile
|
||||
```
|
||||
5. Create /mnt/hdd_builder/workspace/coex-ci.json
|
||||
```(json)
|
||||
{
|
||||
"yadisk":
|
||||
{
|
||||
"login":"LOGIN",
|
||||
"password":"PASS",
|
||||
"server_dir":"/clever_images"
|
||||
},
|
||||
"github":
|
||||
{
|
||||
"login":"LOGIN",
|
||||
"password":"PASS",
|
||||
"url":"https://api.github.com/repos/CopterExpress/clever/releases/"
|
||||
}
|
||||
}
|
||||
```
|
||||
6. Add webhook to release on your github project
|
||||
> TODO
|
||||
7. Install Jenkins
|
||||
> Manual https://www.digitalocean.com/community/tutorials/how-to-install-jenkins-on-ubuntu-16-04
|
||||
8. Change user & group invoked Jenkins
|
||||
```bash
|
||||
sudo sed -i 's/JENKINS_USER=$NAME/JENKINS_USER=root/' /etc/default/jenkins
|
||||
sudo sed -i 's/JENKINS_GROUP=$NAME/JENKINS_GROUP=root/' /etc/default/jenkins
|
||||
```
|
||||
9. Install Jenikins plugins
|
||||
> Pipeline, Git SCM, Matrix Authorization, github-webhook-build-trigger-plugin
|
||||
10. Create Jenkins pipeline job
|
||||
> TODO
|
||||
11. Configure Jenkins
|
||||
> TODO: Matrix autorization, GIT Token
|
||||
12. Add Jenkins service to autostart
|
||||
```bash
|
||||
sudo systemctl enable jenkins
|
||||
```
|
||||
13. Start service
|
||||
```bash
|
||||
sudo systemctl start jenkins
|
||||
```
|
||||
|
||||
## Requirements
|
||||
|
||||
* Jenkins (BlueOcean plugin, optional)
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
If JDK not installed:
|
||||
|
||||
```bash
|
||||
sudo apt-get install default-jdk
|
||||
```
|
||||
|
||||
## Для использования execute в качестве mount_image
|
||||
|
||||
```bash
|
||||
./image_config.sh execute $IMAGE_PATH << EOF
|
||||
uname -a
|
||||
EOF
|
||||
```
|
||||
|
||||
## Running the Docker
|
||||
|
||||
```bash
|
||||
docker run --privileged -it --rm -v /dev:/dev -v $(pwd)/image:/builder/image smirart/builder
|
||||
```
|
||||
|
||||
## TODO
|
||||
|
||||
* Change http на https в jenkins plugins
|
||||
* Add finally block for disconnect image
|
||||
* In Jenkins build call by name - change
|
||||
|
||||
## Варнинги Jenkins
|
||||
|
||||
```log
|
||||
[WARNING] The POM for org.jenkins-ci.tools:maven-hpi-plugin:jar:2.0 is missing, no dependency information available
|
||||
[WARNING] Failed to build parent project for io.codeclou.jenkins.github.webhook.build.trigger.plugin:github-webhook-build-trigger-plugin:hpi:1.2.0
|
||||
```
|
||||
|
||||
* https://yandex.ru/search/?text=The%20POM%20for%20org.jenkins-ci.tools%3Amaven-hpi-plugin%3Ajar%3A2.0%20is%20missing%2C%20no%20dependency%20information%20available&&lr=213
|
||||
* http://jenkins-ci.361315.n4.nabble.com/Plugin-org-jenkins-ci-tools-maven-hpi-plugin-td4751140.html
|
||||
* http://qaru.site/questions/1460710/maven-jenkins-plugin-poms-missing-for-dependency-information-on-jars
|
||||
134
builder_docker/autosizer.sh
Executable file
134
builder_docker/autosizer.sh
Executable file
@@ -0,0 +1,134 @@
|
||||
#! /usr/bin/env bash
|
||||
|
||||
#
|
||||
# Script for image configure
|
||||
# @urpylka Artem Smirnov
|
||||
#
|
||||
|
||||
# Exit immidiately on non-zero result
|
||||
set -e
|
||||
|
||||
echo_bold() {
|
||||
# TEMPLATE: echo_bold <TEXT> <TYPE>
|
||||
# TYPE: SUCCESS, ERROR, INFO
|
||||
|
||||
# More info there https://www.shellhacks.com/ru/bash-colors/
|
||||
|
||||
TEXT="$1"
|
||||
TEXT="\e[1m$TEXT\e[0m" # BOLD
|
||||
|
||||
case "$2" in
|
||||
SUCCESS)
|
||||
TEXT="\e[32m${TEXT}\e[0m";; # GREEN
|
||||
ERROR)
|
||||
TEXT="\e[31m${TEXT}\e[0m";; # RED
|
||||
*)
|
||||
TEXT="\e[34m${TEXT}\e[0m";; # BLUE
|
||||
esac
|
||||
echo -e ${TEXT}
|
||||
}
|
||||
|
||||
if [ $(whoami) != "root" ]; then
|
||||
echo \
|
||||
&& echo "********************************************************************" \
|
||||
&& echo "******************** This should be run as root ********************" \
|
||||
&& echo "********************************************************************" \
|
||||
&& echo \
|
||||
&& exit 1
|
||||
fi
|
||||
|
||||
if [[ -z $1 ]]; then
|
||||
echo "================================================================================"
|
||||
echo_bold "Automatic Image file resizer"
|
||||
echo_bold "Description: This script shrink your image to 10MiB free space"
|
||||
echo_bold "if you didn't set FREE_SPACE in MiB (see usage below)."
|
||||
echo_bold "Authors: Artem Smirnov @urpylka, SirLagz"
|
||||
echo
|
||||
echo_bold "Usage: ./autosizer.sh IMAGE_PATH FREE_SPACE"
|
||||
echo
|
||||
echo_bold "Requirements: parted, losetup, e2fsck, resize2fs, bc, truncate"
|
||||
echo "================================================================================"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "================================================================================"
|
||||
strImgFile=$1
|
||||
echo_bold "Path to image: $strImgFile"
|
||||
echo "================================================================================"
|
||||
|
||||
if [[ ! -e $strImgFile ]]; then
|
||||
echo_bold "Error: File doesn't exist"
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "================================================================================"
|
||||
partinfo=`parted -m $strImgFile unit B print`
|
||||
echo_bold "Partition information:\n$partinfo"
|
||||
echo "================================================================================"
|
||||
|
||||
partnumber=`echo "$partinfo" | grep ext4 | awk -F: '{ print $1 }'`
|
||||
echo_bold "Partition number: $partnumber"
|
||||
echo "================================================================================"
|
||||
|
||||
partstart=`echo "$partinfo" | grep ext4 | awk -F: '{ print substr($2,0,length($2)-1) }'`
|
||||
echo_bold "Partition start: $partstart (bytes)"
|
||||
echo "================================================================================"
|
||||
|
||||
loopback=`losetup -f --show -o $partstart $strImgFile`
|
||||
echo_bold "Loopback device: $loopback"
|
||||
echo "================================================================================"
|
||||
|
||||
set +e
|
||||
e2fsck -fvy $loopback
|
||||
set -e
|
||||
|
||||
echo "================================================================================"
|
||||
minsize=`resize2fs -P $loopback | awk -F': ' '{ print $2 }'`
|
||||
#minsize=`resize2fs -P $loopback 2> /dev/null | awk -F': ' '{ print $2 }'`
|
||||
echo_bold "Minsize: $minsize (4KiB)"
|
||||
echo "================================================================================"
|
||||
|
||||
# Default add 10MiB free space to image, if $2 doesn't set
|
||||
FREE_SPACE=${2:-10}
|
||||
|
||||
FREE_SPACE=$(($FREE_SPACE*1024*1024/4096))
|
||||
|
||||
minsize=`echo "$minsize+$FREE_SPACE" | bc`
|
||||
echo_bold "Minsize + $FREE_SPACE (4KiB): $minsize (4KiB)"
|
||||
echo "================================================================================"
|
||||
|
||||
resize2fs -p $loopback $minsize
|
||||
sleep 1
|
||||
losetup -d $loopback
|
||||
|
||||
echo "================================================================================"
|
||||
partnewsize=`echo "$minsize * 4096" | bc`
|
||||
echo_bold "New size of part: $minsize (4KiB) = $partnewsize (bytes)"
|
||||
echo "================================================================================"
|
||||
|
||||
newpartend=`echo "$partstart + $partnewsize" | bc`
|
||||
echo_bold "New end of part (Part start + part new size):"
|
||||
echo_bold "$partstart (bytes) + $partnewsize (bytes) = $newpartend (bytes)"
|
||||
echo "================================================================================"
|
||||
|
||||
part1=`parted $strImgFile rm 2`
|
||||
echo "================================================================================"
|
||||
part2=`parted $strImgFile unit B mkpart primary $partstart $newpartend`
|
||||
|
||||
echo "================================================================================"
|
||||
endresult=`parted -m $strImgFile unit B print free | tail -1 | awk -F: '{ print substr($2,0,length($2)-1) }'`
|
||||
echo_bold "Size of result image: $endresult (bytes)"
|
||||
echo "================================================================================"
|
||||
|
||||
truncate -s $endresult $strImgFile
|
||||
|
||||
echo "================================================================================"
|
||||
partinfo=`parted -m $strImgFile unit B print`
|
||||
echo_bold "Partition information:\n$partinfo"
|
||||
echo "================================================================================"
|
||||
|
||||
# TODO check if image needs to change PARTUUID
|
||||
#sed -i 's/root=[^ ]*/root=\/dev\/mmcblk0p2/' /boot/cmdline.txt
|
||||
#sed -i 's/.* \/boot vfat defaults 0 2$/\/dev\/mmcblk0p1 \/boot vfat defaults 0 2/' /etc/fstab
|
||||
#sed -i 's/.* \/ ext4 defaults,noatime 0 1$/\/dev\/mmcblk0p2 \/ ext4 defaults,noatime 0 1/' /etc/fstab
|
||||
94
builder_docker/build.Jenkinsfile
Normal file
94
builder_docker/build.Jenkinsfile
Normal file
@@ -0,0 +1,94 @@
|
||||
pipeline {
|
||||
agent any
|
||||
parameters {
|
||||
string(name: 'IMAGE_NAME', defaultValue: 'clever_noname.img', description: 'Output image file name')
|
||||
string(name: 'GWBT_REF', defaultValue: 'master', description: 'Checkout ref-param')
|
||||
string(name: 'IMAGE_VERSION', defaultValue: 'no_version', description: 'Image version')
|
||||
|
||||
string(name: 'BUILD_DIR', defaultValue: '/mnt/hdd_builder/workspace', description: 'Build workspace')
|
||||
|
||||
string(name: 'RPI_DONWLOAD_URL', defaultValue: 'https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2017-12-01/2017-11-29-raspbian-stretch-lite.zip')
|
||||
// TODO: Add mirrorparameters
|
||||
|
||||
string(name: 'GWBT_URL', defaultValue: 'https://github.com/CopterExpress/clever.git')
|
||||
|
||||
// Experimental function
|
||||
booleanParam(name: 'SHRINK', defaultValue: true, description: 'SHRINK IMAGE')
|
||||
booleanParam(name: 'DISCOVER_ROS_PACKAGES', defaultValue: false, description: 'DISCOVER ROS PACKAGES')
|
||||
}
|
||||
environment {
|
||||
DEBIAN_FRONTEND = 'noninteractive'
|
||||
LANG = 'C.UTF-8'
|
||||
LC_ALL = 'C.UTF-8'
|
||||
}
|
||||
stages {
|
||||
stage('Get image') {
|
||||
steps {
|
||||
sh "$WORKSPACE/image_builder/image_config.sh get_image ${params.BUILD_DIR} ${params.RPI_DONWLOAD_URL} ${params.IMAGE_NAME}"
|
||||
}
|
||||
}
|
||||
stage('Resize FS') {
|
||||
environment {
|
||||
SIZE = '7G'
|
||||
}
|
||||
steps {
|
||||
sh "$WORKSPACE/image_builder/image_config.sh resize_fs ${params.BUILD_DIR}/${params.IMAGE_NAME} $SIZE"
|
||||
}
|
||||
}
|
||||
stage('Initialize image') {
|
||||
environment {
|
||||
EXECUTE_FILE = 'image_builder/scripts/init_image.sh'
|
||||
}
|
||||
// TODO: Transfer apps.sh initialisation code here
|
||||
steps {
|
||||
sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} $WORKSPACE/$EXECUTE_FILE ${params.IMAGE_VERSION} \$(basename ${params.RPI_DONWLOAD_URL})"
|
||||
}
|
||||
}
|
||||
stage('Hardware setup') {
|
||||
environment {
|
||||
EXECUTE_FILE = 'image_builder/scripts/hardware_setup.sh'
|
||||
}
|
||||
steps {
|
||||
sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} $WORKSPACE/$EXECUTE_FILE"
|
||||
}
|
||||
}
|
||||
stage('Software install') {
|
||||
environment {
|
||||
EXECUTE_FILE = 'image_builder/scripts/software_install.sh'
|
||||
}
|
||||
steps {
|
||||
sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} $WORKSPACE/$EXECUTE_FILE"
|
||||
}
|
||||
}
|
||||
stage('Network setup') {
|
||||
environment {
|
||||
EXECUTE_FILE = 'image_builder/scripts/network_setup.sh'
|
||||
}
|
||||
steps {
|
||||
sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} $WORKSPACE/$EXECUTE_FILE"
|
||||
}
|
||||
}
|
||||
stage('Install ROS') {
|
||||
environment {
|
||||
EXECUTE_FILE = 'image_builder/scripts/ros_install.sh'
|
||||
MOVE_FILE = 'image_builder/kinetic-ros-coex.rosinstall'
|
||||
MOVE_TO = '/home/pi/ros_catkin_ws'
|
||||
}
|
||||
steps {
|
||||
sh "if ! ${params.DISCOVER_ROS_PACKAGES}; then $WORKSPACE/image_builder/image_config.sh copy_to_chroot ${params.BUILD_DIR}/${params.IMAGE_NAME} $WORKSPACE/$MOVE_FILE $MOVE_TO; fi"
|
||||
sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} $WORKSPACE/$EXECUTE_FILE ${params.GWBT_URL} ${params.GWBT_REF} ${params.DISCOVER_ROS_PACKAGES}"
|
||||
}
|
||||
}
|
||||
// TODO: Add finalising step, transfer mirror removal from ros.sh
|
||||
stage('Shrink image') {
|
||||
environment {
|
||||
EXECUTE_FILE = 'image_builder/scripts/change_boot_part.sh'
|
||||
}
|
||||
when { expression { return params.SHRINK } }
|
||||
steps {
|
||||
sh "$WORKSPACE/image_builder/autosizer.sh ${params.BUILD_DIR}/${params.IMAGE_NAME}"
|
||||
sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} $WORKSPACE/$EXECUTE_FILE"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
47
builder_docker/build.sh
Executable file
47
builder_docker/build.sh
Executable file
@@ -0,0 +1,47 @@
|
||||
#! /usr/bin/env bash
|
||||
|
||||
#
|
||||
# Script for image configure
|
||||
# @urpylka Artem Smirnov
|
||||
#
|
||||
|
||||
# Exit immidiately on non-zero result
|
||||
set -e
|
||||
|
||||
export DEBIAN_FRONTEND=${DEBIAN_FRONTEND:='noninteractive'}
|
||||
export LANG=${LANG:='C.UTF-8'}
|
||||
export LC_ALL=${LC_ALL:='C.UTF-8'}
|
||||
|
||||
export TARGET_REPO=${TARGET_REPO:='https://github.com/urpylka/clever.git'}
|
||||
export TARGET_REF=${TARGET_REF:='master'}
|
||||
export TARGET_CONFIG=${TARGET_CONFIG:='/.builder.json'}
|
||||
|
||||
echo "================================================================================"
|
||||
echo "TARGET_REPO: $TARGET_REPO"
|
||||
echo "TARGET_REF: $TARGET_REF"
|
||||
echo "TARGET_CONFIG: $TARGET_CONFIG"
|
||||
echo "================================================================================"
|
||||
|
||||
# TODO: The repository can be already downloaded, use the TARGET_REPO also as unix path.
|
||||
REPO_DIR=$(mktemp -d --suffix=.builder_repo)
|
||||
git clone ${TARGET_REPO} --single-branch --branch ${TARGET_REF} --depth 1 ${REPO_DIR} \
|
||||
|| (echo 'Error: Could not clone repo!'; return 1)
|
||||
[[ -f ${REPO_DIR}${TARGET_CONFIG} ]] && export TARGET_CONFIG=${REPO_DIR}${TARGET_CONFIG} \
|
||||
|| (echo "Error: TARGET_CONFIG doesn't exist!"; return 1)
|
||||
|
||||
export IMAGE_VERSION="${TARGET_REF}_$(date '+%Y%m%d_%H%M%S')"
|
||||
export IMAGE_PATH="$(pwd)/image/$(basename -s ".git" ${TARGET_REPO})_${IMAGE_VERSION}.img"
|
||||
|
||||
./get_image.sh ${IMAGE_PATH} $(jq '.source_image' -r ${TARGET_CONFIG})
|
||||
|
||||
REGISTER=':arm:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/qemu-arm-static:'
|
||||
if [[ $(arch) != 'armv7l' ]]; then
|
||||
mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc 2> /dev/null || true
|
||||
echo ${REGISTER} > /proc/sys/fs/binfmt_misc/register 2> /dev/null || true
|
||||
./image_config.sh copy_to_chroot ${IMAGE_PATH} './qemu-arm-resin' '/usr/bin/qemu-arm-static'
|
||||
fi
|
||||
|
||||
export IMAGE_BUILDER="$(dirname $(readlink -e "$0"))"
|
||||
export SCRIPTS_DIR="${REPO_DIR}$(jq '.scripts_dir' -r ${TARGET_CONFIG})"
|
||||
|
||||
${REPO_DIR}$(jq '.main_script' -r ${TARGET_CONFIG})
|
||||
26
builder_docker/echo_stamp.sh
Executable file
26
builder_docker/echo_stamp.sh
Executable file
@@ -0,0 +1,26 @@
|
||||
#! /usr/bin/env bash
|
||||
|
||||
#
|
||||
# Script for image configure
|
||||
# @urpylka Artem Smirnov
|
||||
#
|
||||
|
||||
echo_stamp() {
|
||||
# TEMPLATE: echo_stamp <TEXT> <TYPE>
|
||||
# TYPE: SUCCESS, ERROR, INFO
|
||||
|
||||
# More info there https://www.shellhacks.com/ru/bash-colors/
|
||||
|
||||
TEXT="$(date) | $1"
|
||||
TEXT="\e[1m$TEXT\e[0m" # BOLD
|
||||
|
||||
case "$2" in
|
||||
SUCCESS)
|
||||
TEXT="\e[32m${TEXT}\e[0m";; # GREEN
|
||||
ERROR)
|
||||
TEXT="\e[31m${TEXT}\e[0m";; # RED
|
||||
*)
|
||||
TEXT="\e[34m${TEXT}\e[0m";; # BLUE
|
||||
esac
|
||||
echo -e ${TEXT}
|
||||
}
|
||||
32
builder_docker/get_image.sh
Executable file
32
builder_docker/get_image.sh
Executable file
@@ -0,0 +1,32 @@
|
||||
#! /usr/bin/env bash
|
||||
|
||||
#
|
||||
# Script for image configure
|
||||
# @urpylka Artem Smirnov
|
||||
# @dvornikov-aa Andrey Dvornikov
|
||||
#
|
||||
|
||||
# Exit immidiately on non-zero result
|
||||
set -e
|
||||
|
||||
source echo_stamp.sh
|
||||
|
||||
get_image() {
|
||||
# TEMPLATE: get_image <IMAGE_PATH> <RPI_DONWLOAD_URL>
|
||||
local BUILD_DIR=$(dirname $1)
|
||||
local RPI_ZIP_NAME=$(basename $2)
|
||||
if [ ! -e "${BUILD_DIR}/${RPI_ZIP_NAME}" ];
|
||||
then
|
||||
echo_stamp "1. Downloading original Linux distribution"
|
||||
wget -nv -O ${BUILD_DIR}/${RPI_ZIP_NAME} $2 > /dev/null \
|
||||
&& echo_stamp "Downloading complete" "SUCCESS"
|
||||
else
|
||||
echo_stamp "1. Linux distribution already donwloaded"
|
||||
fi
|
||||
echo_stamp "2. Unzipping Linux distribution image"
|
||||
local RPI_IMAGE_NAME=$(echo ${RPI_ZIP_NAME} | sed 's/zip/img/')
|
||||
unzip -p ${BUILD_DIR}/${RPI_ZIP_NAME} ${RPI_IMAGE_NAME} > $1
|
||||
echo_stamp "Unzipping complete" "SUCCESS"
|
||||
}
|
||||
|
||||
get_image $1 $2
|
||||
168
builder_docker/image_config.sh
Executable file
168
builder_docker/image_config.sh
Executable file
@@ -0,0 +1,168 @@
|
||||
#! /usr/bin/env bash
|
||||
|
||||
#
|
||||
# Script for image configure
|
||||
# @urpylka Artem Smirnov
|
||||
# @dvornikov-aa Andrey Dvornikov
|
||||
#
|
||||
|
||||
# Exit immidiately on non-zero result
|
||||
set -e
|
||||
|
||||
source echo_stamp.sh
|
||||
|
||||
# This script doesn't work on Ubuntu because OS`s losetup does not consist --partscan (-P).
|
||||
|
||||
# Idea: use `mount -o loop,offset`
|
||||
# https://stefanoprenna.com/blog/2014/09/22/tutorial-how-to-mount-raw-images-img-images-on-linux/
|
||||
# REPO_DIR=$(mktemp -d --suffix=.builder_repo)
|
||||
# mount -t ext4 -o loop,offset=$((94208 * 512)) image/clever_qemu_test_2_20180822_163141.img "$REPO_DIR"
|
||||
# mount -t vfat -o loop,offset=$((8192 * 512)) image/clever_qemu_test_2_20180822_163141.img "$REPO_DIR/boot"
|
||||
|
||||
execute() {
|
||||
# TEMPLATE: execute <IMAGE_PATH> <EXECUTE_FILE> <...>
|
||||
|
||||
echo_stamp "Mount loop-image: $1"
|
||||
local DEV_IMAGE=$(losetup -Pf $1 --show)
|
||||
sleep 0.5
|
||||
|
||||
local MOUNT_POINT=$(mktemp -d --suffix=.builder_image)
|
||||
echo_stamp "Mount dirs ${MOUNT_POINT} & ${MOUNT_POINT}/boot"
|
||||
mount "${DEV_IMAGE}p2" ${MOUNT_POINT}
|
||||
mount "${DEV_IMAGE}p1" ${MOUNT_POINT}/boot
|
||||
|
||||
echo_stamp "Bind system dirs"
|
||||
echo_stamp "Mounting /proc in chroot... "
|
||||
if [ ! -d ${MOUNT_POINT}/proc ] ; then
|
||||
mkdir -p ${MOUNT_POINT}/proc \
|
||||
&& echo_stamp "Created ${MOUNT_POINT}/proc" "SUCCESS"
|
||||
fi
|
||||
mount -t proc -o nosuid,noexec,nodev proc ${MOUNT_POINT}/proc \
|
||||
&& echo_stamp "OK" "SUCCESS"
|
||||
|
||||
echo_stamp "Mounting /sys in chroot... "
|
||||
if [ ! -d ${MOUNT_POINT}/sys ] ; then
|
||||
mkdir -p ${MOUNT_POINT}/sys \
|
||||
&& echo_stamp "Created ${MOUNT_POINT}/sys" "SUCCESS"
|
||||
fi
|
||||
mount -t sysfs -o nosuid,noexec,nodev sysfs ${MOUNT_POINT}/sys \
|
||||
&& echo_stamp "OK" "SUCCESS"
|
||||
|
||||
echo_stamp "Mounting /dev/ and /dev/pts in chroot... " \
|
||||
&& mkdir -p -m 755 ${MOUNT_POINT}/dev/pts \
|
||||
&& mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${MOUNT_POINT}/dev \
|
||||
&& mount -t devpts -o gid=5,mode=620 devpts ${MOUNT_POINT}/dev/pts \
|
||||
&& echo_stamp "OK" "SUCCESS"
|
||||
|
||||
echo_stamp "Copy DNS records" \
|
||||
&& cp -L /etc/resolv.conf ${MOUNT_POINT}/etc/resolv.conf
|
||||
|
||||
if [[ $# > 1 ]]; then
|
||||
echo_stamp "Entering to chroot"
|
||||
local script_name=$(basename $2)
|
||||
# TODO: maybe copy to tmp-dir
|
||||
local script_path_root="${MOUNT_POINT}/root/${script_name}"
|
||||
# Copy script into chroot fs
|
||||
# TODO: Find more suitable location for temporary script storage
|
||||
cp "$2" "${script_path_root}"
|
||||
# Its important to save arguments (direct ${@:4} causes problems)
|
||||
script_args="${@:3}"
|
||||
# Run script in chroot with additional arguments
|
||||
chroot ${MOUNT_POINT} /bin/sh -c "/root/${script_name} ${script_args}"
|
||||
# Removing script from chroot fs
|
||||
rm "${script_path_root}"
|
||||
else
|
||||
# https://wiki.archlinux.org/index.php/Change_root_(%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9)
|
||||
# http://www.unix-lab.org/posts/chroot/
|
||||
# https://habrahabr.ru/post/141012/
|
||||
# https://losst.ru/vosstanovlenie-grub2
|
||||
# http://unixteam.ru/content/virtualizaciya-ili-zapuskaem-prilozhenie-v-chroot-okruzhenii-razmyshleniya
|
||||
# http://help.ubuntu.ru/wiki/%D0%B2%D0%BE%D1%81%D1%81%D1%82%D0%B0%D0%BD%D0%BE%D0%B2%D0%BB%D0%B5%D0%BD%D0%B8%D0%B5_grub
|
||||
echo_stamp "Entering to chroot" \
|
||||
&& chroot ${MOUNT_POINT} /bin/bash
|
||||
fi
|
||||
|
||||
umount_system ${MOUNT_POINT} ${DEV_IMAGE}
|
||||
}
|
||||
|
||||
copy_to_chroot() {
|
||||
# TEMPLATE: copy_to_chroot <IMAGE_PATH> <MOVE_FILE> <MOVE_TO>
|
||||
|
||||
echo_stamp "Mount loop-image: $1"
|
||||
local DEV_IMAGE=$(losetup -Pf $1 --show)
|
||||
sleep 0.5
|
||||
|
||||
local MOUNT_POINT=$(mktemp -d --suffix=.builder_image)
|
||||
echo_stamp "Mount dirs ${MOUNT_POINT} & ${MOUNT_POINT}/boot"
|
||||
mount "${DEV_IMAGE}p2" ${MOUNT_POINT}
|
||||
mount "${DEV_IMAGE}p1" ${MOUNT_POINT}/boot
|
||||
|
||||
dir_name=$(dirname "${MOUNT_POINT}$3 /")
|
||||
if [ ! -d ${dir_name} ] ; then
|
||||
mkdir -p ${dir_name} \
|
||||
&& echo_stamp "Created ${dir_name}" "SUCCESS"
|
||||
fi
|
||||
|
||||
cp -r "$2" "${MOUNT_POINT}$3"
|
||||
umount_system ${MOUNT_POINT} ${DEV_IMAGE}
|
||||
}
|
||||
|
||||
umount_system() {
|
||||
# TEMPLATE: umount_system <MOUNT_POINT> <DEV_IMAGE>
|
||||
|
||||
echo_stamp "Umount recursive dirs: $1"
|
||||
# There is a risk that umount will fail
|
||||
set +e
|
||||
# Successfull unmount flag (false at thismoment)
|
||||
umount_ok=false
|
||||
# Repeat 5 times
|
||||
for i in {1..5}
|
||||
do
|
||||
# Unmount chroot rootfs and boot partition
|
||||
umount -fR $1
|
||||
# If no problems detected
|
||||
if [[ $? == 0 ]]
|
||||
then
|
||||
echo_stamp "Successfull unmount" "SUCCESS"
|
||||
# Set flag
|
||||
umount_ok=true
|
||||
# Exit loop
|
||||
break
|
||||
fi
|
||||
# Unmount has failed
|
||||
echo_stamp "Unmount failed" "ERROR"
|
||||
# Wait for some time
|
||||
sleep 2
|
||||
done
|
||||
set -e
|
||||
# Jenkins job will fail if this condition is not true
|
||||
[[ "$umount_ok" == true ]]
|
||||
echo_stamp "Umount loop-image"
|
||||
losetup -d $2
|
||||
}
|
||||
|
||||
if [ $(whoami) != "root" ];
|
||||
then echo "" \
|
||||
&& echo "********************************************************************" \
|
||||
&& echo "******************** This should be run as root ********************" \
|
||||
&& echo "********************************************************************" \
|
||||
&& echo "" \
|
||||
&& exit 1
|
||||
fi
|
||||
|
||||
echo "================================================================================"
|
||||
for ((i=1; i<=$#; i++)); do echo "\$$i: ${!i}"; done
|
||||
echo "================================================================================"
|
||||
|
||||
case "$1" in
|
||||
execute)
|
||||
# execute <IMAGE_PATH> [<EXECUTE_FILE>] [...]
|
||||
execute $2 $3 ${@:4};;
|
||||
|
||||
copy_to_chroot)
|
||||
# copy_to_chroot <IMAGE_PATH> <MOVE_FILE> <MOVE_TO>
|
||||
copy_to_chroot $2 $3 $4;;
|
||||
|
||||
*)
|
||||
echo "Enter one of: get_image, resize_fs, publish_image, execute";;
|
||||
esac
|
||||
64
builder_docker/publish_image.sh
Executable file
64
builder_docker/publish_image.sh
Executable file
@@ -0,0 +1,64 @@
|
||||
#! /usr/bin/env bash
|
||||
|
||||
#
|
||||
# Script for image configure
|
||||
# @urpylka Artem Smirnov
|
||||
# @dvornikov-aa Andrey Dvornikov
|
||||
#
|
||||
|
||||
# Exit immidiately on non-zero result
|
||||
set -e
|
||||
|
||||
source echo_stamp.sh
|
||||
|
||||
publish_image() {
|
||||
# TEMPLATE: publish_image_bash <IMAGE_PATH> <YA_SCRIPT> <CONFIG_FILE> <RELEASE_ID> <RELEASE_BODY>
|
||||
|
||||
# https://developer.github.com/v3/repos/releases/
|
||||
|
||||
IMAGE_NAME=$(basename $1)
|
||||
BUILD_DIR=$(dirname $1)
|
||||
|
||||
echo_stamp "Zip image"
|
||||
if [ ! -e "${BUILD_DIR}/${IMAGE_NAME}.zip" ];
|
||||
then
|
||||
cd ${BUILD_DIR} && zip ${IMAGE_NAME}.zip ${IMAGE_NAME}
|
||||
echo_stamp "Zipping complete!" "SUCCESS"
|
||||
else
|
||||
echo_stamp "Zip-archive already created"
|
||||
cd ${BUILD_DIR} && rm ${IMAGE_NAME}.zip && zip ${IMAGE_NAME}.zip ${IMAGE_NAME} \
|
||||
&& echo_stamp "Old archive was deleted & create new" "SUCCESS"
|
||||
fi
|
||||
|
||||
echo_stamp "Upload image"
|
||||
local IMAGE_LINK=$($2 $3 ${BUILD_DIR}/${IMAGE_NAME}.zip) \
|
||||
&& echo_stamp "Upload copmlete!" "SUCCESS"
|
||||
|
||||
echo_stamp "Meashure size of zip-image"
|
||||
local IMAGE_SIZE=$(du -sh ${BUILD_DIR}/${IMAGE_NAME}.zip | awk '{ print $1 }') \
|
||||
&& echo_stamp "Meashuring copmlete!" "SUCCESS"
|
||||
|
||||
echo_stamp "Meashure hash-sum of zip-image"
|
||||
local IMAGE_HASH=$(sha256sum ${BUILD_DIR}/${IMAGE_NAME}.zip | awk '{ print $1 }') \
|
||||
&& echo_stamp "Meashuring copmlete!" "SUCCESS"
|
||||
|
||||
echo ""
|
||||
echo "\$5: $5"
|
||||
echo ""
|
||||
|
||||
echo_stamp "Post message to GH"
|
||||
local NEW_RELEASE_BODY="### Download\n* [${IMAGE_NAME}.zip](${IMAGE_LINK}) (${IMAGE_SIZE})\nsha256: ${IMAGE_HASH}\n\n$5"
|
||||
local DATA="{ \"body\":\"${NEW_RELEASE_BODY}\" }"
|
||||
|
||||
echo ""
|
||||
echo "\$DATA: $DATA"
|
||||
echo ""
|
||||
|
||||
local GH_LOGIN=$(cat $3 | jq '.github.login' -r)
|
||||
local GH_PASS=$(cat $3 | jq '.github.password' -r)
|
||||
local GH_URL=$(cat $3 | jq '.github.url' -r)
|
||||
curl -d "$DATA" -u "${GH_LOGIN}:${GH_PASS}" --request PATCH ${GH_URL}$4 \
|
||||
&& echo_stamp "Post message to GH copmlete!" "SUCCESS"
|
||||
}
|
||||
|
||||
publish_image $1 $2 $3 $4 "$5"
|
||||
BIN
builder_docker/qemu-arm-resin
Executable file
BIN
builder_docker/qemu-arm-resin
Executable file
Binary file not shown.
57
builder_docker/resize_fs.sh
Executable file
57
builder_docker/resize_fs.sh
Executable file
@@ -0,0 +1,57 @@
|
||||
#! /usr/bin/env bash
|
||||
|
||||
#
|
||||
# Script for image configure
|
||||
# @urpylka Artem Smirnov
|
||||
# @dvornikov-aa Andrey Dvornikov
|
||||
#
|
||||
|
||||
# Exit immidiately on non-zero result
|
||||
set -e
|
||||
|
||||
source echo_stamp.sh
|
||||
|
||||
resize_fs() {
|
||||
# TEMPLATE: resize_fs <IMAGE_PATH> <SIZE>
|
||||
|
||||
set +e
|
||||
|
||||
# https://ru.wikipedia.org/wiki/%D0%A0%D0%B0%D0%B7%D1%80%D0%B5%D0%B6%D1%91%D0%BD%D0%BD%D1%8B%D0%B9_%D1%84%D0%B0%D0%B9%D0%BB
|
||||
|
||||
# https://raspberrypi.stackexchange.com/questions/13137/how-can-i-mount-a-raspberry-pi-linux-distro-image
|
||||
# fdisk -l 2017-11-29-raspbian-stretch-lite.img
|
||||
# https://www.stableit.ru/2011/05/losetup.html
|
||||
# -f : losetup сам выбрал loop (минуя занятые)
|
||||
# -P : losetup монтирует разделы в образе как отдельные подразделы,
|
||||
# например /dev/loop0p1 и /dev/loop0p2
|
||||
# --show : печатает имя устройства, например /dev/loop4
|
||||
|
||||
# http://karelzak.blogspot.ru/2015/05/resize-by-sfdisk.html
|
||||
# ", +" : expand partition for volume size
|
||||
# -N 2 : select second partition for work
|
||||
|
||||
# There is a risk that sfdisk will ask for a disk remount to update partition table
|
||||
# TODO: Check sfdisk exit code
|
||||
|
||||
echo_stamp "Truncate image" \
|
||||
&& truncate -s$2 $1 \
|
||||
&& echo_stamp "Mount loop-image: $1" \
|
||||
&& local DEV_IMAGE=$(losetup -Pf $1 --show) \
|
||||
&& sleep 0.5 \
|
||||
&& echo ", +" | sfdisk -N 2 ${DEV_IMAGE} \
|
||||
&& sleep 0.5 \
|
||||
&& losetup -d ${DEV_IMAGE} \
|
||||
&& sleep 0.5 \
|
||||
&& local DEV_IMAGE=$(losetup -Pf $1 --show) \
|
||||
&& sleep 0.5 \
|
||||
&& echo_stamp "Check & repair filesystem after expand partition" \
|
||||
&& e2fsck -fvy "${DEV_IMAGE}p2" \
|
||||
&& echo_stamp "Expand filesystem" \
|
||||
&& resize2fs "${DEV_IMAGE}p2" \
|
||||
&& echo_stamp "Umount loop-image" \
|
||||
&& losetup -d ${DEV_IMAGE}
|
||||
|
||||
set -e
|
||||
}
|
||||
|
||||
resize_fs $1 $2
|
||||
51
builder_docker/yadisk.py
Executable file
51
builder_docker/yadisk.py
Executable file
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
#
|
||||
# Simple python uploader to YaDisk
|
||||
# Smirnov Artem @urpylka
|
||||
#
|
||||
# Use:
|
||||
# python yadisk.py login password file server_dir
|
||||
#
|
||||
|
||||
from YaDiskClient.YaDiskClient import YaDisk
|
||||
import os.path, sys, json
|
||||
|
||||
def upload(_login, _password, _server_dir, _file):
|
||||
if os.path.isfile(_file):
|
||||
disk = YaDisk(_login, _password)
|
||||
disk.upload(_file, _server_dir + '/' + os.path.basename(_file))
|
||||
link = disk.publish_doc(_server_dir + '/' + os.path.basename(_file))
|
||||
print link
|
||||
else:
|
||||
print "Error: file-path is bad"
|
||||
return 1
|
||||
|
||||
def main():
|
||||
if (len(sys.argv) == 5):
|
||||
print "login: " + sys.argv[1]
|
||||
print "password: " + sys.argv[2]
|
||||
print "server_dir: " + sys.argv[3]
|
||||
print "file: " + sys.argv[4]
|
||||
|
||||
upload(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])
|
||||
|
||||
elif (len(sys.argv) == 3):
|
||||
# print "config: " + sys.argv[1]
|
||||
# print "file: " + sys.argv[2]
|
||||
|
||||
if os.path.isfile(sys.argv[1]) and os.path.isfile(sys.argv[2]):
|
||||
|
||||
with open(sys.argv[1]) as json_data:
|
||||
d = json.load(json_data)
|
||||
upload(d['yadisk']['login'], d['yadisk']['password'], d['yadisk']['server_dir'], sys.argv[2])
|
||||
|
||||
else:
|
||||
print "Error: file-path or config-path is bad"
|
||||
return 1
|
||||
else:
|
||||
print "Error: amount of args is incorrect"
|
||||
return 1
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user