image_builder: refactor

2026-06-08 02:24:32 +00:00 · 2018-04-14 00:01:47 +03:00
parent 2e75d531a6
commit 2628d912cd
15 changed files with 584 additions and 684 deletions
--- a/image_builder/scripts/hardware_setup.sh
+++ b/image_builder/scripts/hardware_setup.sh
@@ -0,0 +1,135 @@
+#!/bin/bash
+
+set -e
+
+##################################################################################################################################
+# Настройка интерфейсов
+##################################################################################################################################
+
+# вот так все в принципе должно включиться
+# /usr/bin/raspi-config nonint do_i2c 0
+# /usr/bin/raspi-config nonint do_spi 0
+# /usr/bin/raspi-config nonint do_camera 0
+# /usr/bin/raspi-config nonint do_rgpio 0
+# /usr/bin/raspi-config nonint do_ssh 0
+
+# по идеи эти настройки должны проводиться до по другому как сделано в prepare_image.sh
+
+set_config_var() {
+  lua - "$1" "$2" "$3" <<EOF > "$3.bak"
+local key=assert(arg[1])
+local value=assert(arg[2])
+local fn=assert(arg[3])
+local file=assert(io.open(fn))
+local made_change=false
+for line in file:lines() do
+  if line:match("^#?%s*"..key.."=.*$") then
+    line=key.."="..value
+    made_change=true
+  end
+  print(line)
+end
+
+if not made_change then
+  print(key.."="..value)
+end
+EOF
+  mv "$3.bak" "$3"
+}
+
+BLACKLIST=/etc/modprobe.d/raspi-blacklist.conf
+CONFIG=/boot/config.txt
+
+# 2. Изменить необходимые настройки
+
+#   2.1. Включить sshd
+echo -e "\033[0;31m\033[1m$(date) | #1 Turn on sshd\033[0m\033[0m"
+touch /boot/ssh
+
+#   2.2. Включить GPIO
+# Включено по умолчанию
+
+#   2.3. Включить I2C
+echo -e "\033[0;31m\033[1m$(date) | #2 Turn on I2C\033[0m\033[0m"
+
+set_config_var dtparam=i2c_arm on $CONFIG &&
+  if ! [ -e $BLACKLIST ]; then
+    touch $BLACKLIST
+  fi
+  sed $BLACKLIST -i -e "s/^\(blacklist[[:space:]]*i2c[-_]bcm2708\)/#\1/"
+  sed /etc/modules -i -e "s/^#[[:space:]]*\(i2c[-_]dev\)/\1/"
+  if ! grep -q "^i2c[-_]dev" /etc/modules; then
+    printf "i2c-dev\n" >> /etc/modules
+  fi
+
+#   2.4. Включить SPI
+echo -e "\033[0;31m\033[1m$(date) | #3 Turn on SPI\033[0m\033[0m"
+
+set_config_var dtparam=spi on $CONFIG &&
+  if ! [ -e $BLACKLIST ]; then
+    touch $BLACKLIST
+  fi
+  sed $BLACKLIST -i -e "s/^\(blacklist[[:space:]]*spi[-_]bcm2708\)/#\1/"
+
+#   2.5. Включить raspicam
+echo -e "\033[0;31m\033[1m$(date) | #4 Turn on raspicam\033[0m\033[0m"
+
+get_config_var() {
+  lua - "$1" "$2" <<EOF
+local key=assert(arg[1])
+local fn=assert(arg[2])
+local file=assert(io.open(fn))
+local found=false
+for line in file:lines() do
+  local val = line:match("^%s*"..key.."=(.*)$")
+  if (val ~= nil) then
+    print(val)
+    found=true
+    break
+  end
+end
+if not found then
+   print(0)
+end
+EOF
+}
+
+# тут уже немного иначе, но по сути одно и тоже
+# https://github.com/RPi-Distro/raspi-config/blob/master/raspi-config#L1136
+# $1 is 0 to disable camera, 1 to enable it
+set_camera() {
+  # Stop if /boot is not a mountpoint
+  #if ! mountpoint -q /boot; then
+  #  return 1
+  #fi
+
+  [ -e $CONFIG ] || touch $CONFIG
+
+  if [ "$1" -eq 0 ]; then # disable camera
+    set_config_var start_x 0 $CONFIG
+    sed $CONFIG -i -e "s/^startx/#startx/"
+    sed $CONFIG -i -e "s/^start_file/#start_file/"
+    sed $CONFIG -i -e "s/^fixup_file/#fixup_file/"
+  else # enable camera
+    set_config_var start_x 1 $CONFIG
+    CUR_GPU_MEM=$(get_config_var gpu_mem $CONFIG)
+    if [ -z "$CUR_GPU_MEM" ] || [ "$CUR_GPU_MEM" -lt 128 ]; then
+      set_config_var gpu_mem 128 $CONFIG
+    fi
+    sed $CONFIG -i -e "s/^startx/#startx/"
+    sed $CONFIG -i -e "s/^fixup_file/#fixup_file/"
+  fi
+}
+
+if [ ! -e /boot/start_x.elf ];
+then echo "Your firmware appears to be out of date (no start_x.elf). Please update"
+else set_camera 1
+fi
+
+# Включение V4L драйвера http://robocraft.ru/blog/electronics/3158.html
+#echo "bcm2835-v4l2" >> /etc/modules
+if ! grep -q "^bcm2835-v4l2" /etc/modules; then
+  printf "bcm2835-v4l2\n" >> /etc/modules
+fi
+
+echo -e "\033[0;31m\033[1m$(date) | #5 End of configuring interfaces\033[0m\033[0m"
--- a/image_builder/scripts/init_image.sh
+++ b/image_builder/scripts/init_image.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+##################################################################################################################################
+# Image initialisation
+##################################################################################################################################
+
+# Add apt key to allow local mirror usage during image build
+#wget -O - ftp://192.168.0.10/coex-mirror.gpg | apt-key add -
+# Generate a backup of the original source.list
+#cp /etc/apt/sources.list /var/sources.list.bak
+# Add the local mirror as the first priority repository
+#wget -O - ftp://192.168.0.10/coex-mirror.list 2>/dev/null | cat - /etc/apt/sources.list > /var/sources.list && mv /var/sources.list /etc/apt/sources.list
+
+echo -e "\033[0;31m\033[1m$(date) | #1 apt cache update\033[0m\033[0m"
+
+# Clean repostory cache
+apt-get clean
+# Update repository cache
+apt-get update
+# && apt upgrade -y
+
+echo -e "\033[0;31m\033[1m$(date) | #2 Write clever information\033[0m\033[0m"
+
+# Clever image version
+echo "$1" >> /etc/clever_version
+# Origin image file name
+echo "${2%.*}" >> /etc/clever_origin
+
+echo -e "\033[0;31m\033[1m$(date) | #3 End initialisation of image\033[0m\033[0m"
--- a/image_builder/scripts/network_setup.sh
+++ b/image_builder/scripts/network_setup.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -e
+
+echo -e "\033[0;31m\033[1m$(date) | #1 Write to /etc/wpa_supplicant/wpa_supplicant.conf\033[0m\033[0m"
+
+# TODO: Use wpa_cli insted direct file edit
+echo "
+network={
+    ssid=\"CLEVER\"
+    mode=2
+    key_mgmt=WPA-PSK
+    psk=\"cleverwifi\"
+    frequency=2437
+}" >> /etc/wpa_supplicant/wpa_supplicant.conf
+
+echo -e "\033[0;31m\033[1m$(date) | #2 Write STATIC to /etc/dhcpcd.conf\033[0m\033[0m"
+
+echo "
+interface wlan0
+static ip_address=192.168.11.1/24" >> /etc/dhcpcd.conf
+
+echo -e "\033[0;31m\033[1m$(date) | #3 Write iface to /etc/default/isc-dhcp-server\033[0m\033[0m"
+
+# https://www.shellhacks.com/ru/sed-find-replace-string-in-file/
+sed -i 's/INTERFACESv4=\"\"/INTERFACESv4=\"wlan0\"/' /etc/default/isc-dhcp-server
+
+echo -e "\033[0;31m\033[1m$(date) | #4 Write dhcp declaration subnet to /etc/dhcp/dhcpd.conf\033[0m\033[0m"
+
+echo "subnet 192.168.11.0 netmask 255.255.255.0 {
+  range 192.168.11.11 192.168.11.254;
+  #option domain-name-servers 8.8.8.8;
+  #option domain-name "rpi.local";
+  option routers 192.168.11.1;
+  option broadcast-address 192.168.11.255;
+  default-lease-time 600;
+  max-lease-time 7200;
+}" >> /etc/dhcp/dhcpd.conf
+
+echo -e "\033[0;31m\033[1m$(date) | #5 Write start script for dhcpd to /etc/network/if-up.d/isc-dhcp-server\033[0m\033[0m"
+
+echo "#!/bin/sh
+if [ \"\$IFACE\" = \"--all\" ];
+then sleep 10 && systemctl start isc-dhcp-server.service &
+fi
+" > /etc/network/if-up.d/isc-dhcp-server \
+  && chmod +x /etc/network/if-up.d/isc-dhcp-server
+
+echo -e "\033[0;31m\033[1m$(date) | #6 Write magic script for rename SSID to /etc/rc.local\033[0m\033[0m"
+
+RENAME_SSID="sudo sed -i.OLD \"s/CLEVER/CLEVER-\$(head -c 100 /dev/urandom | xxd -ps -c 100 | sed -e 's/[^0-9]//g' | cut -c 1-4)/g\" /etc/wpa_supplicant/wpa_supplicant.conf && sudo sed -i '/sudo sed/d' /etc/rc.local && sudo reboot"
+
+sed -i "19a$RENAME_SSID" /etc/rc.local
+
+echo -e "\033[0;31m\033[1m$(date) | #7 End of network installation\033[0m\033[0m"
--- a/image_builder/scripts/ros_install.sh
+++ b/image_builder/scripts/ros_install.sh
@@ -0,0 +1,173 @@
+#!/bin/bash
+
+set -e
+
+##################################################################################################################################
+# ROS for user pi
+##################################################################################################################################
+
+# ros http://wiki.ros.org/action/fullsearch/ROSberryPi/Installing%20ROS%20Kinetic%20on%20the%20Raspberry%20Pi
+# maintainer @urpylka
+
+echo -e "\033[0;31m\033[1m$(date) | #0 Installing ROS\033[0m\033[0m"
+
+echo -e "\033[0;31m\033[1m$(date) | #1 Installing dirmngr & add key to apt-key\033[0m\033[0m"
+
+# Install a tool that apt-key uses to add ROS repository key
+# http://wpblogger.su/tags/apt/
+apt-get install dirmngr
+# setup keys
+apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-key 421C365BD9FF1F717815A3895523BAEEB01FA116
+
+# setup sources.list
+echo "deb http://packages.ros.org/ros/ubuntu stretch main" > /etc/apt/sources.list.d/ros-latest.list
+
+echo -e "\033[0;31m\033[1m$(date) | #2 apt update && apt upgrade\033[0m\033[0m"
+
+# install bootstrap tools
+apt-get update
+# && apt upgrade -y
+
+echo -e "\033[0;31m\033[1m$(date) | #3 Installing wget, unzip, python-rosdep, python-rosinstall-generator, python-wstool, python-rosinstall, build-essential, cmake\033[0m\033[0m"
+
+apt-get install --no-install-recommends -y \
+  wget \
+  unzip \
+  python-rosdep \
+  python-rosinstall-generator \
+  python-wstool \
+  python-rosinstall \
+  build-essential \
+  cmake \
+  libjpeg8-dev
+
+echo -e "\033[0;31m\033[1m$(date) | #4 rosdep init && rosdep update\033[0m\033[0m"
+
+# bootstrap rosdep
+rosdep init && rosdep update
+
+echo -e "\033[0;31m\033[1m$(date) | #5 Preparing ros_comm packages to kinetic-ros_comm-wet.rosinstall\033[0m\033[0m"
+
+# create catkin workspace
+mkdir -p /home/pi/ros_catkin_ws && cd /home/pi/ros_catkin_ws \
+    && rosinstall_generator ros_comm --rosdistro kinetic --deps --wet-only --tar > kinetic-ros_comm-wet.rosinstall \
+    && wstool init src kinetic-ros_comm-wet.rosinstall
+
+echo -e "\033[0;31m\033[1m$(date) | #6 Preparing other ROS-packages to kinetic-custom_ros.rosinstall\033[0m\033[0m"
+
+cd /home/pi/ros_catkin_ws \
+  && rosinstall_generator \
+  actionlib actionlib_msgs angles async_web_server_cpp bond bond_core bondcpp bondpy camera_calibration_parsers camera_info_manager catkin class_loader cmake_modules cpp_common cv_bridge cv_camera diagnostic_msgs diagnostic_updater dynamic_reconfigure eigen_conversions gencpp geneus genlisp genmsg gennodejs genpy geographic_msgs geometry_msgs geometry2 image_transport libmavconn mavlink mavros_msgs message_filters message_generation message_runtime mk nav_msgs nodelet orocos_kdl pluginlib python_orocos_kdl ros ros_comm rosapi rosauth rosbag rosbag_migration_rule rosbag_storage rosbash rosboost_cfg rosbridge_library rosbridge_server rosbridge_suite rosbuild rosclean rosconsole rosconsole_bridge roscpp roscpp_serialization roscpp_traits roscreate rosgraph rosgraph_msgs roslang roslaunch roslib roslint roslisp roslz4 rosmake rosmaster rosmsg rosnode rosout rospack rosparam rospy rospy_tutorials rosserial rosserial_client rosserial_msgs rosserial_python rosservice rostest rostime rostopic rosunit roswtf sensor_msgs smclib std_msgs std_srvs stereo_msgs tf tf2 tf2_bullet tf2_eigen tf2_geometry_msgs tf2_kdl tf2_msgs tf2_py tf2_ros tf2_sensor_msgs tf2_tools topic_tools trajectory_msgs urdf urdf_parser_plugin usb_cam uuid_msgs visualization_msgs web_video_server xmlrpcpp mavros opencv3 mavros_extras \
+  --rosdistro kinetic --deps --wet-only --tar > kinetic-custom_ros.rosinstall \
+  && wstool merge -t src kinetic-custom_ros.rosinstall \
+  && wstool update -t src
+
+echo -e "\033[0;31m\033[1m$(date) | #7 Installing dependencies apps with rosdep\033[0m\033[0m"
+cd /home/pi/ros_catkin_ws
+# There is a risk that umount will fail
+set +e
+# Successfull unmount flag (false at thismoment)
+install_ok=false
+# Repeat 5 times
+for i in {1..5}
+do
+  # Resolving Dependencies with rosdep
+  rosdep install -y --from-paths src --ignore-src --rosdistro kinetic -r --os=debian:stretch
+  # If no problems detected
+  if [[ $? == 0 ]]
+  then
+    echo -e "\033[0;31m\033[1m$(date) | Successfull rosdep install\033[0m\033[0m"
+    # Set flag
+    install_ok=true
+    # Exit loop
+    break
+  fi
+  # Unmount has failed
+  echo -e "\033[0;31m\033[1m$(date) | Rosdep installation failed\033[0m\033[0m"
+  # Wait for some time
+  sleep 2
+done
+set -e
+# Jenkins job will fail if this condition is not true
+[[ "$install_ok" == true ]]
+echo -e "\033[0;31m\033[1m$(date) | End of rosdep install\033[0m\033[0m"
+
+
+echo -e "\033[0;31m\033[1m$(date) | #8 Refactoring usb_cam in SRC\033[0m\033[0m"
+
+sed -i '/#define __STDC_CONSTANT_MACROS/a\#define PIX_FMT_RGB24 AV_PIX_FMT_RGB24\n#define PIX_FMT_YUV422P AV_PIX_FMT_YUV422P' /home/pi/ros_catkin_ws/src/usb_cam/src/usb_cam.cpp
+
+echo -e "\033[0;31m\033[1m$(date) | #9 Installing GeographicLib datasets\033[0m\033[0m"
+
+/home/pi/ros_catkin_ws/src/mavros/mavros/scripts/install_geographiclib_datasets.sh
+
+#echo -e "\033[0;31m\033[1m$(date) | #11 Building light packages on 2 threads\033[0m\033[0m"
+
+# Build the catkin Workspace
+#cd /home/pi/ros_catkin_ws && ./src/catkin/bin/catkin_make_isolated --install -DCMAKE_BUILD_TYPE=Release -j2 --install-space /opt/ros/kinetic --pkg actionlib actionlib_msgs angles async_web_server_cpp bond bond_core bondcpp bondpy camera_calibration_parsers camera_info_manager catkin class_loader cmake_modules cpp_common diagnostic_msgs diagnostic_updater dynamic_reconfigure eigen_conversions gencpp geneus genlisp genmsg gennodejs genpy geographic_msgs geometry_msgs geometry2 image_transport libmavconn mavlink mavros_msgs message_filters message_generation message_runtime mk nav_msgs nodelet orocos_kdl pluginlib python_orocos_kdl ros ros_comm rosapi rosauth rosbag rosbag_migration_rule rosbag_storage rosbash rosboost_cfg rosbridge_library rosbridge_server rosbridge_suite rosbuild rosclean rosconsole rosconsole_bridge roscpp roscpp_serialization roscpp_traits roscreate rosgraph rosgraph_msgs roslang roslaunch roslib roslint roslisp roslz4 rosmake rosmaster rosmsg rosnode rosout rospack rosparam rospy rospy_tutorials rosserial rosserial_client rosserial_msgs rosserial_python rosservice rostest rostime rostopic rosunit roswtf sensor_msgs smclib std_msgs std_srvs stereo_msgs tf tf2 tf2_bullet tf2_eigen tf2_geometry_msgs tf2_kdl tf2_msgs tf2_py tf2_ros tf2_sensor_msgs tf2_tools topic_tools trajectory_msgs urdf urdf_parser_plugin usb_cam uuid_msgs visualization_msgs xmlrpcpp
+
+#echo -e "\033[0;31m\033[1m$(date) | #12 Building heavy packages\033[0m\033[0m"
+
+# This command uses less threads to avoid Raspberry Pi freeze
+# Build the catkin Workspace
+#cd /home/pi/ros_catkin_ws && ./src/catkin/bin/catkin_make_isolated --install -DCMAKE_BUILD_TYPE=Release -j1 --install-space /opt/ros/kinetic --pkg mavros opencv3 cv_bridge cv_camera mavros_extras web_video_server
+
+echo -e "\033[0;31m\033[1m$(date) | #10 Building packages on 1 thread\033[0m\033[0m"
+
+# Install builded packages
+# WARNING: A major bug was found when using --pkg option (catkin_make_isolated doesn't install environment files)
+# TODO: Can we increase threads number with HDD swap?
+cd /home/pi/ros_catkin_ws && ./src/catkin/bin/catkin_make_isolated --install -j1 -DCMAKE_BUILD_TYPE=Release --install-space /opt/ros/kinetic
+
+echo -e "\033[0;31m\033[1m$(date) | #12 Creating catkin_ws\033[0m\033[0m"
+
+mkdir -p /home/pi/catkin_ws/src \
+  && cd /home/pi/catkin_ws \
+  && . /opt/ros/kinetic/setup.sh \
+  && catkin init \
+  && wstool init /home/pi/catkin_ws/src
+
+echo -e "\033[0;31m\033[1m$(date) | #13 Installing CLEVER-BUNDLE\033[0m\033[0m"
+
+cd /home/pi/catkin_ws/src \
+  && git clone https://github.com/CopterExpress/clever.git clever \
+  && pip install wheel \
+  && pip install -r /home/pi/catkin_ws/src/clever/clever/requirements.txt \
+  && cd /home/pi/catkin_ws \
+  && . /opt/ros/kinetic/setup.sh \
+  && catkin_make -j1 \
+  && systemctl enable /home/pi/catkin_ws/src/clever/deploy/roscore.service \
+  && systemctl enable /home/pi/catkin_ws/src/clever/deploy/clever.service
+
+echo -e "\033[0;31m\033[1m$(date) | #14 Adding mjpg-streamer at /home/pi\033[0m\033[0m"
+
+# https://github.com/jacksonliam/mjpg-streamer
+
+cd /home/pi \
+  && git clone https://github.com/jacksonliam/mjpg-streamer.git \
+  && cd /home/pi/mjpg-streamer/mjpg-streamer-experimental \
+  && make \
+  && make install
+
+echo -e "\033[0;31m\033[1m$(date) | #15 Adding ENV vars\033[0m\033[0m"
+
+# setup environment
+echo "LANG=C.UTF-8" >> /home/pi/.bashrc
+echo "LC_ALL=C.UTF-8" >> /home/pi/.bashrc
+echo "ROS_DISTRO=kinetic" >> /home/pi/.bashrc
+echo "export ROS_IP=192.168.11.1" >> /home/pi/.bashrc
+
+echo "source /opt/ros/kinetic/setup.bash" >> /home/pi/.bashrc \
+  && echo "source /home/pi/catkin_ws/devel/setup.bash" >> /home/pi/.bashrc
+
+chown -Rf pi:pi /home/pi
+
+#echo -e "\033[0;31m\033[1m$(date) | #16 Removing local apt mirror\033[0m\033[0m"
+# Restore original sources.list
+#mv /var/sources.list.bak /etc/apt/sources.list
+# Clean apt cache
+apt-get clean
+# Remove local mirror repository key
+#apt-key del COEX-MIRROR
+
+echo -e "\033[0;31m\033[1m$(date) | #16 END of ROS INSTALLATION\033[0m\033[0m"
--- a/image_builder/scripts/software_install.sh
+++ b/image_builder/scripts/software_install.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -e
+
+##################################################################################################################################
+# Image software installation
+##################################################################################################################################
+
+echo -e "\033[0;31m\033[1m$(date) | #1 Network installing\033[0m\033[0m"
+
+# TODO: Use dnsmasq instead of isc-dhcp-server
+apt-get install --no-install-recommends -y \
+  unzip \
+  zip \
+  ipython \
+  screen \
+  byobu \
+  nmap \
+  lsof \
+  python-pip \
+  git \
+  isc-dhcp-server \
+  tmux \
+  vim \
+  ipython3 \
+  python3-pip
+
+echo -e "\033[0;31m\033[1m$(date) | End of network installation\033[0m\033[0m"