From 0dcf7d152054a30c9a6722e0e97f73d6c9dccf55 Mon Sep 17 00:00:00 2001 From: urpylka Date: Mon, 23 Apr 2018 21:28:49 +0300 Subject: [PATCH 01/23] image_builder: remove not used code Signed-off-by: Artem Smirnov --- image_builder/image_config.sh | 117 ---------------------------------- 1 file changed, 117 deletions(-) diff --git a/image_builder/image_config.sh b/image_builder/image_config.sh index 641bb331..4162a485 100755 --- a/image_builder/image_config.sh +++ b/image_builder/image_config.sh @@ -308,121 +308,6 @@ umount_system() { losetup -d $2 } -set_config_var() { - lua - "$1" "$2" "$3" < "$3.bak" -local key=assert(arg[1]) -local value=assert(arg[2]) -local fn=assert(arg[3]) -local file=assert(io.open(fn)) -local made_change=false -for line in file:lines() do - if line:match("^#?%s*"..key.."=.*$") then - line=key.."="..value - made_change=true - end - print(line) -end - -if not made_change then - print(key.."="..value) -end -EOF - mv "$3.bak" "$3" -} - -configure_system() { - - # TEMPLATE: configure_system $IMAGE $MOUNT_POINT $ROOT_PARTITON $BOOT_PARTITION - - local BLACKLIST=/etc/modprobe.d/raspi-blacklist.conf - local CONFIG=/boot/config.txt - - # Partitions numbers - local BOOT_PARTITION=1 - local ROOT_PARTITION=2 - - BLACKLIST=$2$BLACKLIST - CONFIG=$2$CONFIG - - # 1. Примонитровать образ - - # https://raspberrypi.stackexchange.com/questions/13137/how-can-i-mount-a-raspberry-pi-linux-distro-image - # mount -v -o offset=48234496 -t ext4 2017-11-29-raspbian-stretch-lite.img $MOUNT_POINT - # mount -v -o offset=4194304,sizelimit=29360128 -t vfat 2017-11-29-raspbian-stretch-lite.img $MOUNT_POINT/boot - # - # fdisk -l 2017-11-29-raspbian-stretch-lite.img - # https://www.stableit.ru/2011/05/losetup.html - # -f : losetup сам выбрал loop (минуя занятые) - # -P : losetup монтирует разделы в образе как отдельные подразделы, - # например /dev/loop0p1 и /dev/loop0p2 - # --show : печатает имя устройства, например /dev/loop4 - echo -e "\033[0;31m\033[1mMount loop-image: $1\033[0m\033[0m" - DEV_IMAGE=$(losetup -Pf $1 --show) - sleep 0.5 - - echo -e "\033[0;31m\033[1mMount dirs $2 & $2/boot\033[0m\033[0m" - mount ${DEV_IMAGE}p${ROOT_PARTITION} $2 - mount ${DEV_IMAGE}p${BOOT_PARTITION} $2/boot - - # 2. Изменить необходимые настройки - - # 2.1. Включить sshd - echo -e "\033[0;31m\033[1mTurn on sshd\033[0m\033[0m" - touch $2/boot/ssh - - # 2.2. Включить GPIO - # Включено по умолчанию - - # 2.3. Включить I2C - echo -e "\033[0;31m\033[1mTurn on I2C\033[0m\033[0m" - - set_config_var dtparam=i2c_arm on $CONFIG && - if ! [ -e $BLACKLIST ]; then - touch $BLACKLIST - fi - sed $BLACKLIST -i -e "s/^\(blacklist[[:space:]]*i2c[-_]bcm2708\)/#\1/" - sed $2/etc/modules -i -e "s/^#[[:space:]]*\(i2c[-_]dev\)/\1/" - if ! grep -q "^i2c[-_]dev" $2/etc/modules; then - printf "i2c-dev\n" >> $2/etc/modules - fi - - # 2.4. Включить SPI - echo -e "\033[0;31m\033[1mTurn on SPI\033[0m\033[0m" - - set_config_var dtparam=spi on $CONFIG && - if ! [ -e $BLACKLIST ]; then - touch $BLACKLIST - fi - sed $BLACKLIST -i -e "s/^\(blacklist[[:space:]]*spi[-_]bcm2708\)/#\1/" - - # 2.5. Включить raspicam - # Включена по умолчанию вроде как - - # 2.6. Настроить AP wifi - # 2.7. Настроить сеть на wlan - # 2.8. Настроить DHCPd на wlan - - # Отмонтировать образ - umount_system $2 $DEV_IMAGE -} - - -prepare_fs() { - - # STATIC FUNCTION - # TEMPLATE: prepare_fs $IMAGE $SIZE - - date - # Удаляем старый образ - # -f : не выводить ошибки, если файла нет - rm -f $1 - # Копируем origin образ - # --progress : Вывод прогресс-бара - rsync --progress -av $1.orig $1 - expand_image $1 $2G - date -} - install_docker() { # STATIC FUNCTION @@ -490,8 +375,6 @@ echo "\$7: $7" # test_docker # install_docker -# prepare_fs -# configure_system case "$1" in mount_system) # mount_system $IMAGE $MOUNT_POINT From 0b9df02faafe7b07c775ce80a78c5087eecd65f2 Mon Sep 17 00:00:00 2001 From: urpylka Date: Mon, 23 Apr 2018 21:35:57 +0300 Subject: [PATCH 02/23] image_builder: translate couple comments Signed-off-by: Artem Smirnov --- image_builder/image_config.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/image_builder/image_config.sh b/image_builder/image_config.sh index 4162a485..9a5ece88 100755 --- a/image_builder/image_config.sh +++ b/image_builder/image_config.sh @@ -51,8 +51,8 @@ resize_fs() { # --show : печатает имя устройства, например /dev/loop4 # http://karelzak.blogspot.ru/2015/05/resize-by-sfdisk.html - # ", +" : расширяет раздел до размеров образа - # -N 2 : выбирает раздел 2 для работы + # ", +" : expand partition for volume size + # -N 2 : select second partition for work # There is a risk that sfdisk will ask for a disk remount to update partition table # TODO: Check sfdisk exit code From a7e003222599da2419b9dc702bfa7fbf980e9fa1 Mon Sep 17 00:00:00 2001 From: urpylka Date: Mon, 23 Apr 2018 21:50:34 +0300 Subject: [PATCH 03/23] image_builder: fix syntax Signed-off-by: Artem Smirnov --- image_builder/image_config.sh | 42 ++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/image_builder/image_config.sh b/image_builder/image_config.sh index 9a5ece88..7b357aab 100755 --- a/image_builder/image_config.sh +++ b/image_builder/image_config.sh @@ -377,24 +377,30 @@ echo "\$7: $7" # install_docker case "$1" in - mount_system) # mount_system $IMAGE $MOUNT_POINT - mount_system $2 $3;; - - get_image) # get_image $BUILD_DIR $RPI_DONWLOAD_URL $IMAGE_NAME - get_image $2 $3 $4;; - - resize_fs) # resize_fs $SIZE $BUILD_DIR $IMAGE_NAME - resize_fs $2 $3 $4 $5;; - - publish_image) # publish_image_python $BUILD_DIR $IMAGE_NAME $WORKSPACE $CONFIG_FILE $RELEASE_ID $RELEASE_BODY - publish_image_python $2 $3 $4 $5 $6 $7;; - - publish_image_bash) # publish_image_bash $BUILD_DIR $IMAGE_NAME $WORKSPACE $CONFIG_FILE $RELEASE_ID $RELEASE_BODY - publish_image_bash $2 $3 $4 $5 $6 $7;; - - execute) # execute $IMAGE $MOUNT_POINT $EXECUTE_FILE ... - execute $2 $3 $4 ${@:5};; - *) echo "Enter one of: mount_system, get_image, resize_fs, publish_image, execute";; + + mount_system) + # mount_system $IMAGE $MOUNT_POINT + mount_system $2 $3;; + + get_image) + # get_image $BUILD_DIR $RPI_DONWLOAD_URL $IMAGE_NAME + get_image $2 $3 $4;; + + resize_fs) + # resize_fs $SIZE $BUILD_DIR $IMAGE_NAME + resize_fs $2 $3 $4 $5;; + + publish_image) + # publish_image_python $BUILD_DIR $IMAGE_NAME $WORKSPACE $CONFIG_FILE $RELEASE_ID $RELEASE_BODY + publish_image_python $2 $3 $4 $5 $6 $7;; + + publish_image_bash) + # publish_image_bash $BUILD_DIR $IMAGE_NAME $WORKSPACE $CONFIG_FILE $RELEASE_ID $RELEASE_BODY + publish_image_bash $2 $3 $4 $5 $6 $7;; + + execute) + # execute $IMAGE $MOUNT_POINT $EXECUTE_FILE ... + execute $2 $3 $4 ${@:5};; esac From 1a0d762c4e3435c9276838781dc3115e0ffe3fba Mon Sep 17 00:00:00 2001 From: urpylka Date: Mon, 23 Apr 2018 22:09:58 +0300 Subject: [PATCH 04/23] image_builder: hardware configure by raspi-config Signed-off-by: Artem Smirnov --- image_builder/scripts/hardware_setup.sh | 141 ++++-------------------- 1 file changed, 24 insertions(+), 117 deletions(-) diff --git a/image_builder/scripts/hardware_setup.sh b/image_builder/scripts/hardware_setup.sh index 543c06d9..bc70aab1 100755 --- a/image_builder/scripts/hardware_setup.sh +++ b/image_builder/scripts/hardware_setup.sh @@ -1,135 +1,42 @@ #!/bin/bash +# Exit immidiately on non-zero result set -e -################################################################################################################################## -# Настройка интерфейсов -################################################################################################################################## +# +# Script for image configure +# @urpylka Artem Smirnov +# -# вот так все в принципе должно включиться -# /usr/bin/raspi-config nonint do_i2c 0 -# /usr/bin/raspi-config nonint do_spi 0 -# /usr/bin/raspi-config nonint do_camera 0 -# /usr/bin/raspi-config nonint do_rgpio 0 -# /usr/bin/raspi-config nonint do_ssh 0 +################################################## +# Configure hardware interfaces +################################################## -# по идеи эти настройки должны проводиться до по другому как сделано в prepare_image.sh - -set_config_var() { - lua - "$1" "$2" "$3" < "$3.bak" -local key=assert(arg[1]) -local value=assert(arg[2]) -local fn=assert(arg[3]) -local file=assert(io.open(fn)) -local made_change=false -for line in file:lines() do - if line:match("^#?%s*"..key.."=.*$") then - line=key.."="..value - made_change=true - end - print(line) -end - -if not made_change then - print(key.."="..value) -end -EOF - mv "$3.bak" "$3" -} - -BLACKLIST=/etc/modprobe.d/raspi-blacklist.conf -CONFIG=/boot/config.txt - -# 2. Изменить необходимые настройки - -# 2.1. Включить sshd +# 1. Enable sshd echo -e "\033[0;31m\033[1m$(date) | #1 Turn on sshd\033[0m\033[0m" touch /boot/ssh +# /usr/bin/raspi-config nonint do_ssh 0 -# 2.2. Включить GPIO -# Включено по умолчанию +# 2. Enable GPIO +echo -e "\033[0;31m\033[1m$(date) | #2 SSHd enabled by default\033[0m\033[0m" -# 2.3. Включить I2C -echo -e "\033[0;31m\033[1m$(date) | #2 Turn on I2C\033[0m\033[0m" +# 3. Enable I2C +echo -e "\033[0;31m\033[1m$(date) | #3 Turn on I2C\033[0m\033[0m" +/usr/bin/raspi-config nonint do_i2c 0 -set_config_var dtparam=i2c_arm on $CONFIG && - if ! [ -e $BLACKLIST ]; then - touch $BLACKLIST - fi - sed $BLACKLIST -i -e "s/^\(blacklist[[:space:]]*i2c[-_]bcm2708\)/#\1/" - sed /etc/modules -i -e "s/^#[[:space:]]*\(i2c[-_]dev\)/\1/" - if ! grep -q "^i2c[-_]dev" /etc/modules; then - printf "i2c-dev\n" >> /etc/modules - fi +# 4. Enable SPI +echo -e "\033[0;31m\033[1m$(date) | #4 Turn on SPI\033[0m\033[0m" +/usr/bin/raspi-config nonint do_spi 0 -# 2.4. Включить SPI -echo -e "\033[0;31m\033[1m$(date) | #3 Turn on SPI\033[0m\033[0m" +# 5. Enable raspicam +echo -e "\033[0;31m\033[1m$(date) | #5 Turn on raspicam\033[0m\033[0m" +/usr/bin/raspi-config nonint do_camera 0 -set_config_var dtparam=spi on $CONFIG && - if ! [ -e $BLACKLIST ]; then - touch $BLACKLIST - fi - sed $BLACKLIST -i -e "s/^\(blacklist[[:space:]]*spi[-_]bcm2708\)/#\1/" - -# 2.5. Включить raspicam -echo -e "\033[0;31m\033[1m$(date) | #4 Turn on raspicam\033[0m\033[0m" - -get_config_var() { - lua - "$1" "$2" <> /etc/modules +echo -e "\033[0;31m\033[1m$(date) | #6 Turn on v4l2 driver\033[0m\033[0m" if ! grep -q "^bcm2835-v4l2" /etc/modules; then printf "bcm2835-v4l2\n" >> /etc/modules fi -echo -e "\033[0;31m\033[1m$(date) | #5 End of configuring interfaces\033[0m\033[0m" +echo -e "\033[0;31m\033[1m$(date) | End of configure hardware interfaces\033[0m\033[0m" From 40be77f9c39cf3a498553ad550b08f439c8ae9ac Mon Sep 17 00:00:00 2001 From: urpylka Date: Mon, 23 Apr 2018 22:47:14 +0300 Subject: [PATCH 05/23] image_builder: refactor publish_image functions Signed-off-by: Artem Smirnov --- image_builder/image_config.sh | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/image_builder/image_config.sh b/image_builder/image_config.sh index 7b357aab..ac30c9bc 100755 --- a/image_builder/image_config.sh +++ b/image_builder/image_config.sh @@ -87,14 +87,18 @@ publish_image_python() { # https://developer.github.com/v3/repos/releases/ #RELEASE_BODY="### Changelog\n* Add /boot/cmdline.txt net.ifnames=0 https://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/\n* Updated cophelper\n* Installed copstat" - echo 'Zip image' \ - && if [ ! -e "$1/$2.zip" ]; - then zip $1/$2.zip $1/$2 - fi + echo 'Zip image' + if [ ! -e "$1/$2.zip" ]; + then cd $1 && zip $2.zip $2 + fi + echo 'Upload image' \ - && local IMAGE_LINK=$($3/image_builder/yadisk.py $1/$4 $1/$2.zip) \ - && local IMAGE_SIZE=$(du -sh $1/$2.zip | awk '{ print $1 }') \ - && echo "Make downloads in GH-release" \ + && local IMAGE_LINK=$($3/image_builder/yadisk.py $1/$4 $1/$2.zip) + + echo 'Meashuring size of zip-image' \ + && local IMAGE_SIZE=$(du -sh $1/$2.zip | awk '{ print $1 }') + + echo 'Post message to GH' \ && $3/image_builder/git_release.py $1/$4 $5 $6 $2 $IMAGE_LINK $IMAGE_SIZE # echo "Fake publish" } @@ -107,13 +111,18 @@ publish_image_bash() { # https://developer.github.com/v3/repos/releases/ #RELEASE_BODY="### Changelog\n* Add /boot/cmdline.txt net.ifnames=0 https://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/\n* Updated cophelper\n* Installed copstat" - echo 'Zip image' \ - && if [ ! -e "$1/$2.zip" ]; - then zip $1/$2.zip $1/$2 - fi + echo 'Zip image' + if [ ! -e "$1/$2.zip" ]; + then cd $1 && zip $2.zip $2 + fi + echo 'Upload image' \ - && local IMAGE_LINK=$($3/image_builder/yadisk.py $1/$4 $1/$2.zip) \ - && local IMAGE_SIZE=$(du -sh $1/$2.zip | awk '{ print $1 }') \ + && local IMAGE_LINK=$($3/image_builder/yadisk.py $1/$4 $1/$2.zip) + + echo 'Meashuring size of zip-image' \ + && local IMAGE_SIZE=$(du -sh $1/$2.zip | awk '{ print $1 }') + + echo 'Post message to GH' \ && local NEW_RELEASE_BODY="### Download\n* [$2.zip]($IMAGE_LINK) ($IMAGE_SIZE)\n\n$6" \ && local DATA="{ \"body\":\"$NEW_RELEASE_BODY\" }" \ && curl -d "$(echo $DATA)" -u "LOGIN:PASS" --request PATCH https://api.github.com/repos/ONWER/REPO/releases/$5 From 679cd37ec6b86255ec7e836ff033d1cea0815918 Mon Sep 17 00:00:00 2001 From: urpylka Date: Tue, 24 Apr 2018 12:49:21 +0300 Subject: [PATCH 06/23] image_builder: fix bag with IMAGE_VERSION Signed-off-by: Artem Smirnov --- image_builder/jenkinsfile-build | 4 ++-- image_builder/jenkinsfile-release | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/image_builder/jenkinsfile-build b/image_builder/jenkinsfile-build index 763cc88a..bcaf72c8 100644 --- a/image_builder/jenkinsfile-build +++ b/image_builder/jenkinsfile-build @@ -2,8 +2,8 @@ pipeline { agent any parameters { string(name: 'IMAGE_NAME', defaultValue: 'clever_noname.img', description: 'Output image file name') - string(name: 'GWBT_RELEASE_TAG_NAME', defaultValue: "master", description: 'Release tag name') - string(name: 'IMAGE_VERSION', defaultValue: "${params.GWBT_RELEASE_TAG_NAME}", description: 'Image version') + string(name: 'GWBT_RELEASE_TAG_NAME', defaultValue: 'master', description: 'Release tag name') + string(name: 'IMAGE_VERSION', defaultValue: 'no_version', description: 'Image version') string(name: 'BUILD_DIR', defaultValue: '/mnt/hdd_builder/workspace', description: 'Build workspace') string(name: 'MOUNT_POINT', defaultValue: '/mnt/hdd_builder/image', description: 'Mount point') diff --git a/image_builder/jenkinsfile-release b/image_builder/jenkinsfile-release index 3c402678..91a36f95 100644 --- a/image_builder/jenkinsfile-release +++ b/image_builder/jenkinsfile-release @@ -21,7 +21,7 @@ pipeline { stage('Build image') { when { not { expression { return params.ONLY_PUBLISH } } } steps { - build job: 'CopterExpress-clever-build', parameters: [[$class: 'StringParameterValue', name: 'IMAGE_NAME', value: "${params.IMAGE_NAME}"], [$class: 'StringParameterValue', name: 'GWBT_RELEASE_TAG_NAME', value: "${params.GWBT_RELEASE_TAG_NAME}"]] + build job: 'CopterExpress-clever-build', parameters: [[$class: 'StringParameterValue', name: 'IMAGE_NAME', value: "${params.IMAGE_NAME}"], [$class: 'StringParameterValue', name: 'IMAGE_VERSION', value: "${params.GWBT_RELEASE_TAG_NAME}"], [$class: 'StringParameterValue', name: 'GWBT_RELEASE_TAG_NAME', value: "${params.GWBT_RELEASE_TAG_NAME}"]] sh "echo ${params.IMAGE_NAME}" } } From d8fc2c088fcd3163952aa8fb4d29753e6994aa64 Mon Sep 17 00:00:00 2001 From: urpylka Date: Tue, 24 Apr 2018 13:36:26 +0300 Subject: [PATCH 07/23] image_builder: fix bag with switch-case order in bash Signed-off-by: Artem Smirnov --- image_builder/image_config.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/image_builder/image_config.sh b/image_builder/image_config.sh index ac30c9bc..6f7367ca 100755 --- a/image_builder/image_config.sh +++ b/image_builder/image_config.sh @@ -386,9 +386,6 @@ echo "\$7: $7" # install_docker case "$1" in - *) - echo "Enter one of: mount_system, get_image, resize_fs, publish_image, execute";; - mount_system) # mount_system $IMAGE $MOUNT_POINT mount_system $2 $3;; @@ -412,4 +409,7 @@ case "$1" in execute) # execute $IMAGE $MOUNT_POINT $EXECUTE_FILE ... execute $2 $3 $4 ${@:5};; + + *) + echo "Enter one of: mount_system, get_image, resize_fs, publish_image, execute";; esac From 8519a6c6a4c3eed1fffae347eafb5fba593c6f7d Mon Sep 17 00:00:00 2001 From: urpylka Date: Tue, 24 Apr 2018 13:46:35 +0300 Subject: [PATCH 08/23] image_builder: refactor path to execute-files Signed-off-by: Artem Smirnov --- image_builder/jenkinsfile-build | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/image_builder/jenkinsfile-build b/image_builder/jenkinsfile-build index bcaf72c8..e11de1d3 100644 --- a/image_builder/jenkinsfile-build +++ b/image_builder/jenkinsfile-build @@ -32,43 +32,43 @@ pipeline { } stage('Initialize image') { environment { - EXECUTE_FILE = 'scripts/init_image.sh' + EXECUTE_FILE = 'image_builder/scripts/init_image.sh' } // TODO: Transfer apps.sh initialisation code here steps { - sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} ${params.MOUNT_POINT} $WORKSPACE/image_builder/$EXECUTE_FILE ${params.IMAGE_VERSION} \$(basename ${params.RPI_DONWLOAD_URL})" + sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} ${params.MOUNT_POINT} $WORKSPACE/$EXECUTE_FILE ${params.IMAGE_VERSION} \$(basename ${params.RPI_DONWLOAD_URL})" } } stage('Hardware setup') { environment { - EXECUTE_FILE = 'scripts/hardware_setup.sh' + EXECUTE_FILE = 'image_builder/scripts/hardware_setup.sh' } steps { - sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} ${params.MOUNT_POINT} $WORKSPACE/image_builder/$EXECUTE_FILE" + sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} ${params.MOUNT_POINT} $WORKSPACE/$EXECUTE_FILE" } } stage('Software install') { environment { - EXECUTE_FILE = 'scripts/software_install.sh' + EXECUTE_FILE = 'image_builder/scripts/software_install.sh' } steps { - sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} ${params.MOUNT_POINT} $WORKSPACE/image_builder/$EXECUTE_FILE" + sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} ${params.MOUNT_POINT} $WORKSPACE/$EXECUTE_FILE" } } stage('Network setup') { environment { - EXECUTE_FILE = 'scripts/network_setup.sh' + EXECUTE_FILE = 'image_builder/scripts/network_setup.sh' } steps { - sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} ${params.MOUNT_POINT} $WORKSPACE/image_builder/$EXECUTE_FILE" + sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} ${params.MOUNT_POINT} $WORKSPACE/$EXECUTE_FILE" } } stage('Install ROS') { environment { - EXECUTE_FILE = 'scripts/ros_install.sh' + EXECUTE_FILE = 'image_builder/scripts/ros_install.sh' } steps { - sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} ${params.MOUNT_POINT} $WORKSPACE/image_builder/$EXECUTE_FILE" + sh "$WORKSPACE/image_builder/image_config.sh execute ${params.BUILD_DIR}/${params.IMAGE_NAME} ${params.MOUNT_POINT} $WORKSPACE/$EXECUTE_FILE" } } // TODO: Add finalising step, transfer mirror removal from ros.sh From f901167009850907d731015a969cfbda15b63446 Mon Sep 17 00:00:00 2001 From: urpylka Date: Tue, 24 Apr 2018 13:48:54 +0300 Subject: [PATCH 09/23] image_builder: replace ini to json, requests to curl Signed-off-by: Artem Smirnov --- image_builder/git_release.py | 39 --------------------------- image_builder/image_config.sh | 45 +++++++------------------------ image_builder/jenkinsfile-release | 2 +- 3 files changed, 11 insertions(+), 75 deletions(-) delete mode 100755 image_builder/git_release.py diff --git a/image_builder/git_release.py b/image_builder/git_release.py deleted file mode 100755 index 096eabaa..00000000 --- a/image_builder/git_release.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python - -# -# Simple github release body-editor -# @urpylka Artem Smirnov -# -# Use: -# python git_release.py CONFIG_FILE RELEASE_ID RELEASE_BODY -# - - -from ConfigParser import SafeConfigParser -import requests, sys, urllib - -def json_wrapper(image_name, image_link, image_size, old_text): - # Don't need for Jenkins plugin - #old_text = urllib.unquote_plus(old_text) - buffer = "### Download\n* [" + image_name + ".zip](" + image_link + ") (" + image_size + ")\n\n" + old_text - js = {} - js["body"] = buffer - return js - - -def main(): - - cfgParser = SafeConfigParser() - cfgParser.read(sys.argv[1]) - - js = json_wrapper(sys.argv[4], sys.argv[5], sys.argv[6], sys.argv[3]) - - r = requests.patch(cfgParser.get('github','url') + sys.argv[2], json=js, auth=(cfgParser.get('github','login'), cfgParser.get('github','password'))) - - if r.status_code == 200: - print("Message has been successfully added!") - else: - return 1 - -if __name__ == '__main__': - main() diff --git a/image_builder/image_config.sh b/image_builder/image_config.sh index 6f7367ca..a9a17161 100755 --- a/image_builder/image_config.sh +++ b/image_builder/image_config.sh @@ -79,31 +79,7 @@ resize_fs() { set -e } -publish_image_python() { - -# STATIC FUNCTION -# TEMPLATE: publish_image_python $BUILD_DIR $IMAGE_NAME $WORKSPACE $CONFIG_FILE $RELEASE_ID $RELEASE_BODY - -# https://developer.github.com/v3/repos/releases/ -#RELEASE_BODY="### Changelog\n* Add /boot/cmdline.txt net.ifnames=0 https://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/\n* Updated cophelper\n* Installed copstat" - - echo 'Zip image' - if [ ! -e "$1/$2.zip" ]; - then cd $1 && zip $2.zip $2 - fi - - echo 'Upload image' \ - && local IMAGE_LINK=$($3/image_builder/yadisk.py $1/$4 $1/$2.zip) - - echo 'Meashuring size of zip-image' \ - && local IMAGE_SIZE=$(du -sh $1/$2.zip | awk '{ print $1 }') - - echo 'Post message to GH' \ - && $3/image_builder/git_release.py $1/$4 $5 $6 $2 $IMAGE_LINK $IMAGE_SIZE -# echo "Fake publish" -} - -publish_image_bash() { +publish_image() { # STATIC FUNCTION # TEMPLATE: publish_image_bash $BUILD_DIR $IMAGE_NAME $WORKSPACE $CONFIG_FILE $RELEASE_ID $RELEASE_BODY @@ -117,15 +93,18 @@ publish_image_bash() { fi echo 'Upload image' \ - && local IMAGE_LINK=$($3/image_builder/yadisk.py $1/$4 $1/$2.zip) + && local IMAGE_LINK=$($3/image_builder/yadisk.py $4 $1/$2.zip) echo 'Meashuring size of zip-image' \ && local IMAGE_SIZE=$(du -sh $1/$2.zip | awk '{ print $1 }') echo 'Post message to GH' \ - && local NEW_RELEASE_BODY="### Download\n* [$2.zip]($IMAGE_LINK) ($IMAGE_SIZE)\n\n$6" \ - && local DATA="{ \"body\":\"$NEW_RELEASE_BODY\" }" \ - && curl -d "$(echo $DATA)" -u "LOGIN:PASS" --request PATCH https://api.github.com/repos/ONWER/REPO/releases/$5 + local NEW_RELEASE_BODY="### Download\n* [$2.zip]($IMAGE_LINK) ($IMAGE_SIZE)\n\n$6" + local DATA="{ \"body\":\"$NEW_RELEASE_BODY\" }" + local GH_LOGIN=$(cat $4 | jq 'github.login') + local GH_PASS=$(cat $4 | jq 'github.password') + local GH_URL=$(cat $4 | jq 'github.url') + curl -d "$(echo $DATA)" -u "$GH_LOGIN:$GH_PASS" --request PATCH $GH_URL$5 } burn_image() { @@ -399,12 +378,8 @@ case "$1" in resize_fs $2 $3 $4 $5;; publish_image) - # publish_image_python $BUILD_DIR $IMAGE_NAME $WORKSPACE $CONFIG_FILE $RELEASE_ID $RELEASE_BODY - publish_image_python $2 $3 $4 $5 $6 $7;; - - publish_image_bash) - # publish_image_bash $BUILD_DIR $IMAGE_NAME $WORKSPACE $CONFIG_FILE $RELEASE_ID $RELEASE_BODY - publish_image_bash $2 $3 $4 $5 $6 $7;; + # publish_image $BUILD_DIR $IMAGE_NAME $WORKSPACE $CONFIG_FILE $RELEASE_ID $RELEASE_BODY + publish_image $2 $3 $4 $5 $6 $7;; execute) # execute $IMAGE $MOUNT_POINT $EXECUTE_FILE ... diff --git a/image_builder/jenkinsfile-release b/image_builder/jenkinsfile-release index 91a36f95..14b9fbb0 100644 --- a/image_builder/jenkinsfile-release +++ b/image_builder/jenkinsfile-release @@ -31,7 +31,7 @@ pipeline { CONFIG_FILE = 'coex-ci.conf' } steps { - sh "$WORKSPACE/image_builder/image_config.sh publish_image ${params.BUILD_DIR} ${params.IMAGE_NAME} $WORKSPACE $CONFIG_FILE ${params.GWBT_RELEASE_ID} \'${params.GWBT_RELEASE_BODY}\'" + sh "$WORKSPACE/image_builder/image_config.sh publish_image ${params.BUILD_DIR} ${params.IMAGE_NAME} $WORKSPACE ${params.BUILD_DIR}/$CONFIG_FILE ${params.GWBT_RELEASE_ID} \'${params.GWBT_RELEASE_BODY}\'" } } } From 3dc9575c9e4c2c27d42def55e0aaf68554e8f281 Mon Sep 17 00:00:00 2001 From: urpylka Date: Tue, 24 Apr 2018 14:00:38 +0300 Subject: [PATCH 10/23] image_builder: refactor comment Signed-off-by: Artem Smirnov --- image_builder/image_config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image_builder/image_config.sh b/image_builder/image_config.sh index a9a17161..8350ef80 100755 --- a/image_builder/image_config.sh +++ b/image_builder/image_config.sh @@ -299,7 +299,7 @@ umount_system() { install_docker() { # STATIC FUNCTION - # TEMPLATE: install_docker $IMAGE $MOUNT_POINT $DEV_ROOTFS $DEV_BOOT + # TEMPLATE: install_docker $IMAGE $MOUNT_POINT # https://askubuntu.com/questions/485567/unexpected-end-of-file mount_system $1 $2 << EOF From c945bb9d03bd60ba7c674252e7914a398f74ff28 Mon Sep 17 00:00:00 2001 From: urpylka Date: Tue, 24 Apr 2018 14:09:43 +0300 Subject: [PATCH 11/23] image_builder: made var YA_SCRIPT Signed-off-by: Artem Smirnov --- image_builder/image_config.sh | 6 +++--- image_builder/jenkinsfile-release | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/image_builder/image_config.sh b/image_builder/image_config.sh index 8350ef80..8bdbfc9e 100755 --- a/image_builder/image_config.sh +++ b/image_builder/image_config.sh @@ -82,7 +82,7 @@ resize_fs() { publish_image() { # STATIC FUNCTION -# TEMPLATE: publish_image_bash $BUILD_DIR $IMAGE_NAME $WORKSPACE $CONFIG_FILE $RELEASE_ID $RELEASE_BODY +# TEMPLATE: publish_image_bash $BUILD_DIR $IMAGE_NAME $YA_SCRIPT $CONFIG_FILE $RELEASE_ID $RELEASE_BODY # https://developer.github.com/v3/repos/releases/ #RELEASE_BODY="### Changelog\n* Add /boot/cmdline.txt net.ifnames=0 https://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/\n* Updated cophelper\n* Installed copstat" @@ -93,7 +93,7 @@ publish_image() { fi echo 'Upload image' \ - && local IMAGE_LINK=$($3/image_builder/yadisk.py $4 $1/$2.zip) + && local IMAGE_LINK=$($3 $4 $1/$2.zip) echo 'Meashuring size of zip-image' \ && local IMAGE_SIZE=$(du -sh $1/$2.zip | awk '{ print $1 }') @@ -378,7 +378,7 @@ case "$1" in resize_fs $2 $3 $4 $5;; publish_image) - # publish_image $BUILD_DIR $IMAGE_NAME $WORKSPACE $CONFIG_FILE $RELEASE_ID $RELEASE_BODY + # publish_image $BUILD_DIR $IMAGE_NAME $YA_SCRIPT $CONFIG_FILE $RELEASE_ID $RELEASE_BODY publish_image $2 $3 $4 $5 $6 $7;; execute) diff --git a/image_builder/jenkinsfile-release b/image_builder/jenkinsfile-release index 14b9fbb0..a1eabbe5 100644 --- a/image_builder/jenkinsfile-release +++ b/image_builder/jenkinsfile-release @@ -29,9 +29,10 @@ pipeline { when { environment name: 'GWBT_TRIGGER', value: 'release' } environment { CONFIG_FILE = 'coex-ci.conf' + YA_SCRIPT = 'image_builder/yadisk.py' } steps { - sh "$WORKSPACE/image_builder/image_config.sh publish_image ${params.BUILD_DIR} ${params.IMAGE_NAME} $WORKSPACE ${params.BUILD_DIR}/$CONFIG_FILE ${params.GWBT_RELEASE_ID} \'${params.GWBT_RELEASE_BODY}\'" + sh "$WORKSPACE/image_builder/image_config.sh publish_image ${params.BUILD_DIR} ${params.IMAGE_NAME} $WORKSPACE/$YA_SCRIPT ${params.BUILD_DIR}/$CONFIG_FILE ${params.GWBT_RELEASE_ID} \'${params.GWBT_RELEASE_BODY}\'" } } } From a65a50ffd3d39b91593776cb4e2b0aa4785a339e Mon Sep 17 00:00:00 2001 From: urpylka Date: Tue, 24 Apr 2018 15:00:29 +0300 Subject: [PATCH 12/23] image_builder: change ini to json in yadisk.py Signed-off-by: Artem Smirnov --- image_builder/yadisk.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/image_builder/yadisk.py b/image_builder/yadisk.py index b42c4c8c..f92fbb4e 100755 --- a/image_builder/yadisk.py +++ b/image_builder/yadisk.py @@ -9,7 +9,7 @@ # from YaDiskClient.YaDiskClient import YaDisk -import os.path, sys +import os.path, sys, json def upload(_login, _password, _server_dir, _file): if os.path.isfile(_file): @@ -36,14 +36,10 @@ def main(): if os.path.isfile(sys.argv[1]) and os.path.isfile(sys.argv[2]): - from ConfigParser import SafeConfigParser - cfgParser = SafeConfigParser() - cfgParser.read(sys.argv[1]) - # print "login: " + cfgParser.get('yadisk','login') - # print "password: " + cfgParser.get('yadisk','password') - # print "server_dir: " + cfgParser.get('yadisk','server_dir') + with open('coex-ci.json') as json_data: + d = json.load(json_data) + upload(d['yadisk']['login'], d['yadisk']['password'], d['yadisk']['server_dir'], sys.argv[2]) - upload(cfgParser.get('yadisk','login'), cfgParser.get('yadisk','password'), cfgParser.get('yadisk','server_dir'), sys.argv[2]) else: print "Error: file-path or config-path is bad" return 1 From 846ec3a4301c78ddf3f0b0fb05f562e709586aed Mon Sep 17 00:00:00 2001 From: urpylka Date: Tue, 24 Apr 2018 15:07:02 +0300 Subject: [PATCH 13/23] image_builder: made absolute path in jenkinsfile Signed-off-by: Artem Smirnov --- image_builder/jenkinsfile-release | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/image_builder/jenkinsfile-release b/image_builder/jenkinsfile-release index a1eabbe5..5f09a9d0 100644 --- a/image_builder/jenkinsfile-release +++ b/image_builder/jenkinsfile-release @@ -28,11 +28,11 @@ pipeline { stage('Publish image') { when { environment name: 'GWBT_TRIGGER', value: 'release' } environment { - CONFIG_FILE = 'coex-ci.conf' - YA_SCRIPT = 'image_builder/yadisk.py' + CONFIG_FILE = "${params.BUILD_DIR}/coex-ci.json" + YA_SCRIPT = "$WORKSPACE/image_builder/yadisk.py" } steps { - sh "$WORKSPACE/image_builder/image_config.sh publish_image ${params.BUILD_DIR} ${params.IMAGE_NAME} $WORKSPACE/$YA_SCRIPT ${params.BUILD_DIR}/$CONFIG_FILE ${params.GWBT_RELEASE_ID} \'${params.GWBT_RELEASE_BODY}\'" + sh "$WORKSPACE/image_builder/image_config.sh publish_image ${params.BUILD_DIR} ${params.IMAGE_NAME} $YA_SCRIPT $CONFIG_FILE ${params.GWBT_RELEASE_ID} \'${params.GWBT_RELEASE_BODY}\'" } } } From 29d66ee264be12cb04fd48e1033e1a521c131ca7 Mon Sep 17 00:00:00 2001 From: urpylka Date: Tue, 24 Apr 2018 15:27:09 +0300 Subject: [PATCH 14/23] image_builder: add readme-file about builder Signed-off-by: Artem Smirnov --- image_builder/README.md | 69 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 image_builder/README.md diff --git a/image_builder/README.md b/image_builder/README.md new file mode 100644 index 00000000..350d183a --- /dev/null +++ b/image_builder/README.md @@ -0,0 +1,69 @@ +## Setup your builder + +1. Install requirements +```(bash) +sudo apt-get install unzip zip git python-pip jq curl +sudo pip install YaDiskClient +``` +2. Mount HDD +> TODO + +3. Enable swap on HDD +> TODO: + +And disable `dphys-swapfile` +```(bash) +sudo systemctl stop dphys-swapfile +sudo systemctl disable dphys-swapfile +``` + +3. Create /mnt/hdd_builder/workspace/coex-ci.json +```(json) +{ + "yadisk": + { + "login":"LOGIN", + "password":"PASS", + "server_dir":"/clever_images" + }, + "github": + { + "login":"LOGIN", + "password":"PASS", + "url":"https://api.github.com/repos/CopterExpress/clever/releases/" + } + +``` +6. Add webhook to release on your github project +> TODO + +9. Install Jenkins +> Manual https://www.digitalocean.com/community/tutorials/how-to-install-jenkins-on-ubuntu-16-04 + +10. Change user & group invoked Jenkins +```(bash) +sudo sed -i 's/JENKINS_USER=$NAME/JENKINS_USER=root/' /etc/default/jenkins +sudo sed -i 's/JENKINS_GROUP=$NAME/JENKINS_GROUP=root/' /etc/default/jenkins +``` +11. Install Jenikins plugins +> Pipeline, Git SCM + +12. Create Jenkins pipeline job +> TODO + +13. Configure Jenkins +> TODO: Matrix autorization, GIT Token + +13. Add Jenkins service to autostart +```(bash) +sudo systemctl enable jenkins +``` + +14. Start service +```(bash) +sudo systemctl start jenkins +``` + +## Requirements + +* Jenkins (BlueOcean plugin, optional) From 9fefc6428e38e45ce53f3428e1827859c224bbe7 Mon Sep 17 00:00:00 2001 From: urpylka Date: Tue, 24 Apr 2018 21:06:41 +0300 Subject: [PATCH 15/23] image_builder: fix small bugs Signed-off-by: Artem Smirnov --- image_builder/image_config.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/image_builder/image_config.sh b/image_builder/image_config.sh index 8bdbfc9e..7d7e10ef 100755 --- a/image_builder/image_config.sh +++ b/image_builder/image_config.sh @@ -98,13 +98,13 @@ publish_image() { echo 'Meashuring size of zip-image' \ && local IMAGE_SIZE=$(du -sh $1/$2.zip | awk '{ print $1 }') - echo 'Post message to GH' \ - local NEW_RELEASE_BODY="### Download\n* [$2.zip]($IMAGE_LINK) ($IMAGE_SIZE)\n\n$6" - local DATA="{ \"body\":\"$NEW_RELEASE_BODY\" }" - local GH_LOGIN=$(cat $4 | jq 'github.login') - local GH_PASS=$(cat $4 | jq 'github.password') - local GH_URL=$(cat $4 | jq 'github.url') - curl -d "$(echo $DATA)" -u "$GH_LOGIN:$GH_PASS" --request PATCH $GH_URL$5 + echo 'Post message to GH' + local NEW_RELEASE_BODY="### Download\n* [$2.zip]($IMAGE_LINK) ($IMAGE_SIZE)\n\n$6" + local DATA="{ \"body\":\"$NEW_RELEASE_BODY\" }" + local GH_LOGIN=$(cat $4 | jq '.github.login') + local GH_PASS=$(cat $4 | jq '.github.password') + local GH_URL=$(cat $4 | jq '.github.url') + curl -d "$(echo $DATA)" -u "$GH_LOGIN:$GH_PASS" --request PATCH $GH_URL$5 } burn_image() { From 460761ef682a0755d31925a90349f84abbd82baa Mon Sep 17 00:00:00 2001 From: urpylka Date: Tue, 24 Apr 2018 21:27:12 +0300 Subject: [PATCH 16/23] image_builder: fix small bug Signed-off-by: Artem Smirnov --- image_builder/yadisk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image_builder/yadisk.py b/image_builder/yadisk.py index f92fbb4e..cacd1517 100755 --- a/image_builder/yadisk.py +++ b/image_builder/yadisk.py @@ -36,7 +36,7 @@ def main(): if os.path.isfile(sys.argv[1]) and os.path.isfile(sys.argv[2]): - with open('coex-ci.json') as json_data: + with open(sys.argv[1]) as json_data: d = json.load(json_data) upload(d['yadisk']['login'], d['yadisk']['password'], d['yadisk']['server_dir'], sys.argv[2]) From 7f7276e34a50d48abab1b72fb66129299a54955e Mon Sep 17 00:00:00 2001 From: urpylka Date: Tue, 24 Apr 2018 22:06:27 +0300 Subject: [PATCH 17/23] image_builder: fix bug with GH credentials after parsing json Signed-off-by: Artem Smirnov --- image_builder/image_config.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/image_builder/image_config.sh b/image_builder/image_config.sh index 7d7e10ef..dc0abd16 100755 --- a/image_builder/image_config.sh +++ b/image_builder/image_config.sh @@ -101,9 +101,9 @@ publish_image() { echo 'Post message to GH' local NEW_RELEASE_BODY="### Download\n* [$2.zip]($IMAGE_LINK) ($IMAGE_SIZE)\n\n$6" local DATA="{ \"body\":\"$NEW_RELEASE_BODY\" }" - local GH_LOGIN=$(cat $4 | jq '.github.login') - local GH_PASS=$(cat $4 | jq '.github.password') - local GH_URL=$(cat $4 | jq '.github.url') + local GH_LOGIN=$(cat $4 | jq '.github.login' -r) + local GH_PASS=$(cat $4 | jq '.github.password' -r) + local GH_URL=$(cat $4 | jq '.github.url' -r) curl -d "$(echo $DATA)" -u "$GH_LOGIN:$GH_PASS" --request PATCH $GH_URL$5 } From a4841de17e91e0a06e29af4a4a7ead13f0e30682 Mon Sep 17 00:00:00 2001 From: urpylka Date: Tue, 24 Apr 2018 22:34:20 +0300 Subject: [PATCH 18/23] image_builder: try fix with send big string as argument Signed-off-by: Artem Smirnov --- .../{jenkinsfile-build => build.Jenkinsfile} | 2 +- image_builder/image_config.sh | 67 ++++++++++--------- image_builder/jenkinsfile | 36 ++++++++++ image_builder/jenkinsfile-release | 39 ----------- 4 files changed, 74 insertions(+), 70 deletions(-) rename image_builder/{jenkinsfile-build => build.Jenkinsfile} (96%) create mode 100644 image_builder/jenkinsfile delete mode 100644 image_builder/jenkinsfile-release diff --git a/image_builder/jenkinsfile-build b/image_builder/build.Jenkinsfile similarity index 96% rename from image_builder/jenkinsfile-build rename to image_builder/build.Jenkinsfile index e11de1d3..4996a5ee 100644 --- a/image_builder/jenkinsfile-build +++ b/image_builder/build.Jenkinsfile @@ -2,7 +2,7 @@ pipeline { agent any parameters { string(name: 'IMAGE_NAME', defaultValue: 'clever_noname.img', description: 'Output image file name') - string(name: 'GWBT_RELEASE_TAG_NAME', defaultValue: 'master', description: 'Release tag name') + string(name: 'GWBT_REF', defaultValue: 'master', description: 'Checkout ref-param') string(name: 'IMAGE_VERSION', defaultValue: 'no_version', description: 'Image version') string(name: 'BUILD_DIR', defaultValue: '/mnt/hdd_builder/workspace', description: 'Build workspace') diff --git a/image_builder/image_config.sh b/image_builder/image_config.sh index dc0abd16..14fcc3da 100755 --- a/image_builder/image_config.sh +++ b/image_builder/image_config.sh @@ -79,34 +79,6 @@ resize_fs() { set -e } -publish_image() { - -# STATIC FUNCTION -# TEMPLATE: publish_image_bash $BUILD_DIR $IMAGE_NAME $YA_SCRIPT $CONFIG_FILE $RELEASE_ID $RELEASE_BODY - -# https://developer.github.com/v3/repos/releases/ -#RELEASE_BODY="### Changelog\n* Add /boot/cmdline.txt net.ifnames=0 https://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/\n* Updated cophelper\n* Installed copstat" - - echo 'Zip image' - if [ ! -e "$1/$2.zip" ]; - then cd $1 && zip $2.zip $2 - fi - - echo 'Upload image' \ - && local IMAGE_LINK=$($3 $4 $1/$2.zip) - - echo 'Meashuring size of zip-image' \ - && local IMAGE_SIZE=$(du -sh $1/$2.zip | awk '{ print $1 }') - - echo 'Post message to GH' - local NEW_RELEASE_BODY="### Download\n* [$2.zip]($IMAGE_LINK) ($IMAGE_SIZE)\n\n$6" - local DATA="{ \"body\":\"$NEW_RELEASE_BODY\" }" - local GH_LOGIN=$(cat $4 | jq '.github.login' -r) - local GH_PASS=$(cat $4 | jq '.github.password' -r) - local GH_URL=$(cat $4 | jq '.github.url' -r) - curl -d "$(echo $DATA)" -u "$GH_LOGIN:$GH_PASS" --request PATCH $GH_URL$5 -} - burn_image() { # STATIC FUNCTION @@ -341,6 +313,41 @@ EOF # service docker start # https://forums.docker.com/t/cannot-connect-to-the-docker-daemon-is-the-docker-daemon-running-on-this-host/8925/17 +publish_image() { + +# STATIC FUNCTION +# TEMPLATE: publish_image_bash $BUILD_DIR $IMAGE_NAME $YA_SCRIPT $CONFIG_FILE $RELEASE_ID $RELEASE_BODY + +# https://developer.github.com/v3/repos/releases/ +#RELEASE_BODY="### Changelog\n* Add /boot/cmdline.txt net.ifnames=0 https://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/\n* Updated cophelper\n* Installed copstat" + + echo -e "\033[0;31m\033[1m$(date) | Zip image\033[0m\033[0m" + if [ ! -e "$1/$2.zip" ]; + then + cd $1 && zip $2.zip $2 + echo -e "\033[0;31m\033[1m$(date) | Zipping complete!\033[0m\033[0m" + else + echo -e "\033[0;31m\033[1m$(date) | Zip-archive already created\033[0m\033[0m" + fi + + echo -e "\033[0;31m\033[1m$(date) | Upload image\033[0m\033[0m" + local IMAGE_LINK=$($3 $4 $1/$2.zip) + echo -e "\033[0;31m\033[1m$(date) | Upload copmlete!\033[0m\033[0m" + + + echo -e "\033[0;31m\033[1m$(date) | Meashure size of zip-image\033[0m\033[0m" + local IMAGE_SIZE=$(du -sh $1/$2.zip | awk '{ print $1 }') + echo -e "\033[0;31m\033[1m$(date) | Meashuring copmlete!\033[0m\033[0m" + + echo -e "\033[0;31m\033[1m$(date) | Post message to GH\033[0m\033[0m" + local NEW_RELEASE_BODY="### Download\n* [$2.zip]($IMAGE_LINK) ($IMAGE_SIZE)\n\n$(echo $6)" + local DATA="{ \"body\":\"$NEW_RELEASE_BODY\" }" + local GH_LOGIN=$(cat $4 | jq '.github.login' -r) + local GH_PASS=$(cat $4 | jq '.github.password' -r) + local GH_URL=$(cat $4 | jq '.github.url' -r) + curl -d "$DATA" -u "$GH_LOGIN:$GH_PASS" --request PATCH $GH_URL$5 + echo -e "\033[0;31m\033[1m$(date) | Post message to GH copmlete!\033[0m\033[0m" +} if [ $(whoami) != "root" ]; then echo "" \ @@ -360,9 +367,9 @@ echo "\$5: $5" echo "\$6: $6" echo "\$7: $7" - # test_docker # install_docker +# burn_image case "$1" in mount_system) @@ -379,7 +386,7 @@ case "$1" in publish_image) # publish_image $BUILD_DIR $IMAGE_NAME $YA_SCRIPT $CONFIG_FILE $RELEASE_ID $RELEASE_BODY - publish_image $2 $3 $4 $5 $6 $7;; + publish_image $2 $3 $4 $5 $6 "$7";; execute) # execute $IMAGE $MOUNT_POINT $EXECUTE_FILE ... diff --git a/image_builder/jenkinsfile b/image_builder/jenkinsfile new file mode 100644 index 00000000..6eab29e6 --- /dev/null +++ b/image_builder/jenkinsfile @@ -0,0 +1,36 @@ +pipeline { + agent any + parameters { + string(name: 'GWBT_REF', defaultValue: "master") + string(name: 'GWBT_URL', defaultValue: "https://github.com/CopterExpress/clever.git") + string(name: 'GWBT_FILE', defaultValue: "") + string(name: 'IMAGE_NAME', defaultValue: "\$(cat ${GWBT_FILE} | jq '.repository.name' -r)_${params.GWBT_REF}_\$(cat ${GWBT_FILE} | jq '.release.published_at' -r).img") + string(name: 'GWBT_EVENT', defaultValue: 'release') + booleanParam(name: 'ONLY_PUBLISH', defaultValue: false, description: 'ONLY PUBLISH') + string(name: 'BUILD_DIR', defaultValue: '/mnt/hdd_builder/workspace', description: 'Build workspace') + } + environment { + DEBIAN_FRONTEND = 'noninteractive' + LANG = 'C.UTF-8' + LC_ALL = 'C.UTF-8' + } + stages { + stage('Build image') { + when { not { expression { return params.ONLY_PUBLISH } } } + steps { + build job: 'CopterExpress-clever-build', parameters: [[$class: 'StringParameterValue', name: 'IMAGE_NAME', value: "${params.IMAGE_NAME}"], [$class: 'StringParameterValue', name: 'IMAGE_VERSION', value: "${params.GWBT_REF}"], [$class: 'StringParameterValue', name: 'GWBT_REF', value: "${params.GWBT_REF}"], [$class: 'StringParameterValue', name: 'GWBT_URL', value: "${params.GWBT_URL}"]] + } + } + stage('Publish image') { + when { environment name: 'GWBT_EVENT', value: 'release' } + environment { + CONFIG_FILE = "${params.BUILD_DIR}/coex-ci.json" + YA_SCRIPT = "$WORKSPACE/image_builder/yadisk.py" + } + steps { + //sh "$WORKSPACE/image_builder/image_config.sh publish_image_jenkins ${params.BUILD_DIR} ${params.IMAGE_NAME} ${YA_SCRIPT} ${CONFIG_FILE} ${params.GWBT_FILE}" + sh "$WORKSPACE/image_builder/image_config.sh publish_image ${params.BUILD_DIR} ${params.IMAGE_NAME} ${YA_SCRIPT} ${CONFIG_FILE} \$(cat ${params.GWBT_FILE} | jq '.release.id' -r) \"\$(cat ${params.GWBT_FILE} | jq '.release.body' | sed 's/\"//' | rev | sed 's/\"//' | rev)\"" + } + } + } +} diff --git a/image_builder/jenkinsfile-release b/image_builder/jenkinsfile-release deleted file mode 100644 index 5f09a9d0..00000000 --- a/image_builder/jenkinsfile-release +++ /dev/null @@ -1,39 +0,0 @@ -pipeline { - agent any - parameters { - //string(name: 'GWBT_REPO_NAME', defaultValue: 'master', description: 'GWBT_REPO_NAME') - //string(name: 'GWBT_RELEASE_TAG_NAME', defaultValue: 'master', description: 'REPO') - //string(name: 'GWBT_RELEASE_PUBLISHED_AT', defaultValue: 'master', description: 'GWBT_RELEASE_PUBLISHED_AT') - string(name: 'GWBT_RELEASE_ID', defaultValue: '', description: 'GWBT_RELEASE_ID') - string(name: 'GWBT_RELEASE_BODY', defaultValue: '', description: 'GWBT_RELEASE_BODY') - string(name: 'IMAGE_NAME', defaultValue: "${params.GWBT_REPO_NAME}_${params.GWBT_RELEASE_TAG_NAME}_${params.GWBT_RELEASE_PUBLISHED_AT}.img", description: 'IMAGE_NAME') - string(name: 'GWBT_TRIGGER', defaultValue: 'release', description: 'GWBT_TRIGGER') - booleanParam(name: 'ONLY_PUBLISH', defaultValue: false, description: 'ONLY PUBLISH') - - string(name: 'BUILD_DIR', defaultValue: '/mnt/hdd_builder/workspace', description: 'Build workspace') - } - environment { - DEBIAN_FRONTEND = 'noninteractive' - LANG = 'C.UTF-8' - LC_ALL = 'C.UTF-8' - } - stages { - stage('Build image') { - when { not { expression { return params.ONLY_PUBLISH } } } - steps { - build job: 'CopterExpress-clever-build', parameters: [[$class: 'StringParameterValue', name: 'IMAGE_NAME', value: "${params.IMAGE_NAME}"], [$class: 'StringParameterValue', name: 'IMAGE_VERSION', value: "${params.GWBT_RELEASE_TAG_NAME}"], [$class: 'StringParameterValue', name: 'GWBT_RELEASE_TAG_NAME', value: "${params.GWBT_RELEASE_TAG_NAME}"]] - sh "echo ${params.IMAGE_NAME}" - } - } - stage('Publish image') { - when { environment name: 'GWBT_TRIGGER', value: 'release' } - environment { - CONFIG_FILE = "${params.BUILD_DIR}/coex-ci.json" - YA_SCRIPT = "$WORKSPACE/image_builder/yadisk.py" - } - steps { - sh "$WORKSPACE/image_builder/image_config.sh publish_image ${params.BUILD_DIR} ${params.IMAGE_NAME} $YA_SCRIPT $CONFIG_FILE ${params.GWBT_RELEASE_ID} \'${params.GWBT_RELEASE_BODY}\'" - } - } - } -} From 66121e6d5da3994ab5eaac447ecd694177d3e617 Mon Sep 17 00:00:00 2001 From: Smirnov Artem Date: Thu, 26 Apr 2018 23:41:30 +0300 Subject: [PATCH 19/23] Rename jenkinsfile to Jenkinsfile --- image_builder/{jenkinsfile => Jenkinsfile} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename image_builder/{jenkinsfile => Jenkinsfile} (100%) diff --git a/image_builder/jenkinsfile b/image_builder/Jenkinsfile similarity index 100% rename from image_builder/jenkinsfile rename to image_builder/Jenkinsfile From 18cfb08054db7cf6588157d27bd57348abb81433 Mon Sep 17 00:00:00 2001 From: urpylka Date: Thu, 26 Apr 2018 23:43:55 +0300 Subject: [PATCH 20/23] image_builder: fix comment --- image_builder/Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/image_builder/Jenkinsfile b/image_builder/Jenkinsfile index 6eab29e6..f515948c 100644 --- a/image_builder/Jenkinsfile +++ b/image_builder/Jenkinsfile @@ -28,7 +28,6 @@ pipeline { YA_SCRIPT = "$WORKSPACE/image_builder/yadisk.py" } steps { - //sh "$WORKSPACE/image_builder/image_config.sh publish_image_jenkins ${params.BUILD_DIR} ${params.IMAGE_NAME} ${YA_SCRIPT} ${CONFIG_FILE} ${params.GWBT_FILE}" sh "$WORKSPACE/image_builder/image_config.sh publish_image ${params.BUILD_DIR} ${params.IMAGE_NAME} ${YA_SCRIPT} ${CONFIG_FILE} \$(cat ${params.GWBT_FILE} | jq '.release.id' -r) \"\$(cat ${params.GWBT_FILE} | jq '.release.body' | sed 's/\"//' | rev | sed 's/\"//' | rev)\"" } } From b0cbc677999bb64be2479eee72d31080a7cd418e Mon Sep 17 00:00:00 2001 From: urpylka Date: Fri, 27 Apr 2018 13:26:28 +0300 Subject: [PATCH 21/23] image_config: add debug output --- image_builder/image_config.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/image_builder/image_config.sh b/image_builder/image_config.sh index 14fcc3da..cf43d269 100755 --- a/image_builder/image_config.sh +++ b/image_builder/image_config.sh @@ -339,6 +339,9 @@ publish_image() { local IMAGE_SIZE=$(du -sh $1/$2.zip | awk '{ print $1 }') echo -e "\033[0;31m\033[1m$(date) | Meashuring copmlete!\033[0m\033[0m" + echo "\$6: $6" + echo "\$7: $7" + echo -e "\033[0;31m\033[1m$(date) | Post message to GH\033[0m\033[0m" local NEW_RELEASE_BODY="### Download\n* [$2.zip]($IMAGE_LINK) ($IMAGE_SIZE)\n\n$(echo $6)" local DATA="{ \"body\":\"$NEW_RELEASE_BODY\" }" From 361c89d0163924de156cf21789e34f79f26bb4af Mon Sep 17 00:00:00 2001 From: urpylka Date: Fri, 27 Apr 2018 13:35:32 +0300 Subject: [PATCH 22/23] image_config: little fix & add more debug output --- image_builder/image_config.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/image_builder/image_config.sh b/image_builder/image_config.sh index cf43d269..96e1f868 100755 --- a/image_builder/image_config.sh +++ b/image_builder/image_config.sh @@ -343,8 +343,9 @@ publish_image() { echo "\$7: $7" echo -e "\033[0;31m\033[1m$(date) | Post message to GH\033[0m\033[0m" - local NEW_RELEASE_BODY="### Download\n* [$2.zip]($IMAGE_LINK) ($IMAGE_SIZE)\n\n$(echo $6)" + local NEW_RELEASE_BODY="### Download\n* [$2.zip]($IMAGE_LINK) ($IMAGE_SIZE)\n\n$6" local DATA="{ \"body\":\"$NEW_RELEASE_BODY\" }" + echo "\$DATA: $DATA" local GH_LOGIN=$(cat $4 | jq '.github.login' -r) local GH_PASS=$(cat $4 | jq '.github.password' -r) local GH_URL=$(cat $4 | jq '.github.url' -r) From 212c6ca5acb1b950e6abc41f6963d56f35338baa Mon Sep 17 00:00:00 2001 From: urpylka Date: Fri, 27 Apr 2018 13:42:12 +0300 Subject: [PATCH 23/23] image_config: fix output debug --- image_builder/image_config.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/image_builder/image_config.sh b/image_builder/image_config.sh index 96e1f868..0e39ee6f 100755 --- a/image_builder/image_config.sh +++ b/image_builder/image_config.sh @@ -338,14 +338,15 @@ publish_image() { echo -e "\033[0;31m\033[1m$(date) | Meashure size of zip-image\033[0m\033[0m" local IMAGE_SIZE=$(du -sh $1/$2.zip | awk '{ print $1 }') echo -e "\033[0;31m\033[1m$(date) | Meashuring copmlete!\033[0m\033[0m" - + echo "" echo "\$6: $6" - echo "\$7: $7" - + echo "" echo -e "\033[0;31m\033[1m$(date) | Post message to GH\033[0m\033[0m" local NEW_RELEASE_BODY="### Download\n* [$2.zip]($IMAGE_LINK) ($IMAGE_SIZE)\n\n$6" local DATA="{ \"body\":\"$NEW_RELEASE_BODY\" }" + echo "" echo "\$DATA: $DATA" + echo "" local GH_LOGIN=$(cat $4 | jq '.github.login' -r) local GH_PASS=$(cat $4 | jq '.github.password' -r) local GH_URL=$(cat $4 | jq '.github.url' -r)