Don’t show s3 progress

Instead of ROS mavlink

.github/workflows/main.yml

@@ -0,0 +1,22 @@
+name: Build image
+
+on:
+  push:
+    branches: [ '*' ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: macos-10.15 # Using macOS as https://github.com/actions/virtual-environments/issues/183
+    steps:
+      - uses: actions/checkout@v2
+      - name: Download base image
+        run: |
+          mkdir output-virtualbox-iso
+          wget --progress=dot:giga https://clovervm.ams3.digitaloceanspaces.com/base-focal.ova -O output-virtualbox-iso/clover-devel.ova
+      - name: Build
+        run: ./build.sh
+      - name: Upload
+        run: AWS_ACCESS_KEY_ID=${{secrets.S3_KEY}} AWS_SECRET_ACCESS_KEY=${{secrets.S3_SECRET_KEY}} aws s3 sync . s3://clovervm/ --acl public-read --endpoint-url https://ams3.digitaloceanspaces.com --no-progress
+        working-directory: output-virtualbox-ovf

.travis.yml

@@ -6,19 +6,19 @@ before_install:
   bionic contrib >> /etc/apt/sources.list" && sudo apt-get update && sudo apt-get
   install build-essential gcc make linux-headers-$(uname -r) virtualbox-6.0 && sudo
   usermod -aG vboxusers $(whoami)
-- mkdir output-virtualbox-iso && pushd output-virtualbox-iso && wget --progress=dot:giga https://github.com/sfalexrog/clever_vm/releases/download/v0.1_base_vm/clover-devel.ova
+- mkdir output-virtualbox-iso && pushd output-virtualbox-iso && wget --progress=dot:giga https://clovervm.ams3.digitaloceanspaces.com/base-focal.ova -O clover-devel.ova
   && popd
 - sudo apt install -y xvfb
 script:
 - "xvfb-run ./build.sh"
-deploy:
-  provider: s3
-  access_key_id: XOJXX4IOM66XN7VZAFYX
-  secret_access_key: $S3_SECRET_KEY
-  bucket: clovervm
-  local-dir: output-virtualbox-ovf
-  acl: public_read
-  on:
-    all_branches: true
-  skip_cleanup: true
-  endpoint: https://ams3.digitaloceanspaces.com
+# deploy:
+#   provider: s3
+#   access_key_id: XOJXX4IOM66XN7VZAFYX
+#   secret_access_key: $S3_SECRET_KEY
+#   bucket: clovervm
+#   local-dir: output-virtualbox-ovf
+#   acl: public_read
+#   on:
+#     all_branches: true
+#   skip_cleanup: true
+#   endpoint: https://ams3.digitaloceanspaces.com

assets/configs/xfce4-power-manager.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<channel name="xfce4-power-manager" version="1.0">
+  <property name="xfce4-power-manager" type="empty">
+    <property name="power-button-action" type="empty"/>
+    <property name="lock-screen-suspend-hibernate" type="empty"/>
+    <property name="logind-handle-lid-switch" type="empty"/>
+    <property name="blank-on-ac" type="empty"/>
+    <property name="blank-on-battery" type="empty"/>
+    <property name="dpms-enabled" type="bool" value="false"/>
+    <property name="dpms-on-ac-sleep" type="empty"/>
+    <property name="dpms-on-ac-off" type="empty"/>
+    <property name="dpms-on-battery-sleep" type="empty"/>
+    <property name="dpms-on-battery-off" type="empty"/>
+    <property name="show-panel-label" type="empty"/>
+    <property name="inactivity-sleep-mode-on-ac" type="empty"/>
+    <property name="inactivity-sleep-mode-on-battery" type="empty"/>
+    <property name="show-tray-icon" type="bool" value="false"/>
+  </property>
+</channel>

assets/patches/sitl_gazebo.patch

@@ -1,59 +1,28 @@
-diff --git a/include/gazebo_opticalflow_plugin.h b/include/gazebo_opticalflow_plugin.h
-index 4fbeab2..18a192b 100644
---- a/include/gazebo_opticalflow_plugin.h
-+++ b/include/gazebo_opticalflow_plugin.h
-@@ -40,7 +40,7 @@
- #include "flow_px4.hpp"
+diff --git a/cmake/FindMAVLink.cmake b/cmake/FindMAVLink.cmake
+index 8de3079..3fdae02 100644
+--- a/cmake/FindMAVLink.cmake
++++ b/cmake/FindMAVLink.cmake
+@@ -19,6 +19,7 @@ set(_MAVLINK_EXTRA_SEARCH_HINTS
+     ${CMAKE_SOURCE_DIR}/mavlink/
+     ../../mavlink/
+     ../mavlink/
++    $ENV{HOME}/Firmware/mavlink/include
+     ${CATKIN_DEVEL_PREFIX}/
+     )
  
- #define DEFAULT_RATE 20
--#define HAS_GYRO TRUE
-+#define HAS_GYRO true
  
- using namespace cv;
- using namespace std;
-diff --git a/src/gazebo_geotagged_images_plugin.cpp b/src/gazebo_geotagged_images_plugin.cpp
-index 050558f..7029199 100644
---- a/src/gazebo_geotagged_images_plugin.cpp
-+++ b/src/gazebo_geotagged_images_plugin.cpp
-@@ -581,7 +581,8 @@ void GeotaggedImagesPlugin::_send_capture_status(struct sockaddr* srcaddr)
-         0,                                      // video status (Idle)
-         interval,                               // image interval
-         0,                                      // recording_time_s
--        available_mib);                         // available_capacity
-+        available_mib,                          // available_capacity
-+        _imageCounter);
-     _send_mavlink_message(&msg, srcaddr);
- }
- 
-@@ -591,6 +592,7 @@ void GeotaggedImagesPlugin::_handle_storage_info(const mavlink_message_t *pMsg,
-     float total_mib     = 0.0f;
-     float available_mib = 0.0f;
-     boost::filesystem::space_info si = boost::filesystem::space(".");
-+    const std::string storage_name = "SITL Camera Storage";
-     available_mib = (float)((double)si.available / (1024.0 * 1024.0));
-     total_mib     = (float)((double)si.capacity  / (1024.0 * 1024.0));
-     _send_cmd_ack(pMsg->sysid, pMsg->compid, MAV_CMD_REQUEST_STORAGE_INFORMATION, MAV_RESULT_ACCEPTED, srcaddr);
-@@ -608,7 +610,9 @@ void GeotaggedImagesPlugin::_handle_storage_info(const mavlink_message_t *pMsg,
-         total_mib - available_mib,          // used_capacity,
-         available_mib,
-         NAN,                                // read_speed,
--        NAN                                 // write_speed
-+        NAN,                                // write_speed
-+        STORAGE_TYPE_OTHER,                 // storage type
-+        storage_name.c_str()                // storage name
-     );
-     _send_mavlink_message(&msg, srcaddr);
- }
-diff --git a/src/gazebo_mavlink_interface.cpp b/src/gazebo_mavlink_interface.cpp
-index 5d29017..d4dd6a7 100644
---- a/src/gazebo_mavlink_interface.cpp
-+++ b/src/gazebo_mavlink_interface.cpp
-@@ -1039,7 +1039,7 @@ void GazeboMavlinkInterface::VisionCallback(OdomPtr& odom_message) {
- 
-     odom.time_usec = odom_message->time_usec();
- 
--    odom.frame_id = MAV_FRAME_VISION_NED;
-+    odom.frame_id = 16 /*MAV_FRAME_VISION_NED*/;
-     odom.child_frame_id = MAV_FRAME_BODY_FRD;
- 
-     odom.x = position.X();
+diff --git a/package.xml b/package.xml
+index ae0fb34..12f17b8 100644
+--- a/package.xml
++++ b/package.xml
+@@ -50,8 +50,8 @@
+   <build_depend>roscpp</build_depend>
+   <build_depend>sensor_msgs</build_depend>
+   <build_depend>std_msgs</build_depend>
+-  <build_depend>python-rospkg</build_depend>
+-  <build_depend>python-jinja2</build_depend>
++  <build_depend>python3-rospkg</build_depend>
++  <build_depend>python3-jinja2</build_depend>
+   <run_depend>eigen</run_depend>
+   <run_depend>gazebo_ros</run_depend>
+   <run_depend>geometry_msgs</run_depend>

assets/vscode/settings.json

@@ -1,3 +0,0 @@
-{
-    "python.pythonPath": "/usr/bin/python2.7"
-}

base_vm.json

@@ -17,10 +17,10 @@
       ["storageattach", "{{.Name}}", "--storagectl", "SATA Controller", "--port", "0", "--nonrotational", "on", "--discard", "on"]
     ],
 
-    "headless": "true",
+    "headless": "false",
 
-    "iso_url": "http://archive.ubuntu.com/ubuntu/dists/bionic/main/installer-amd64/current/images/netboot/mini.iso",
-    "iso_checksum": "bed8a55ae2a657f8349fe3271097cff3a5b8c3d1048cf258568f1601976fa30d",
+    "iso_url": "http://archive.ubuntu.com/ubuntu/dists/focal/main/installer-amd64/current/legacy-images/netboot/mini.iso",
+    "iso_checksum": "0e79e00bf844929d40825b1f0e8634415cda195ba23bae0b041911fde4dfe018",
     "iso_checksum_type": "sha256",
 
     "ssh_password": "{{user `password`}}",

build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -e
+set -ex
 
 # FIXME: Use system Packer if possible
 #PACKER=$(which packer)
@@ -9,9 +9,14 @@ set -e
 PACKER="./packer"
 if [ ! -f ${PACKER} ]; then
     echo "Packer not found; downloading v1.5.4 from Hashicorp"
-    wget --progress=dot:giga https://releases.hashicorp.com/packer/1.5.4/packer_1.5.4_linux_amd64.zip
-    unzip packer_1.5.4_linux_amd64.zip
-    rm packer_1.5.4_linux_amd64.zip
+    if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+        OS=linux
+    elif [[ "$OSTYPE" == "darwin"* ]]; then
+        OS=darwin
+    fi
+    wget --progress=dot:giga https://releases.hashicorp.com/packer/1.5.4/packer_1.5.4_${OS}_amd64.zip
+    unzip packer_1.5.4_${OS}_amd64.zip
+    rm packer_1.5.4_${OS}_amd64.zip
 fi
 
 echo "--- Using Packer version $(${PACKER} --version)"
@@ -28,9 +33,10 @@ ${PACKER} build ros_ide_vm.json
 echo "--- Marking the VM"
 
 GIT_REV=$(git rev-parse --short HEAD)
-GIT_DESCRIBE=$(git describe --tags)
+GIT_DESCRIBE=$(git describe --always)
 VM_NAME="clover-devel_${GIT_DESCRIBE}.ova"
 mv ./output-virtualbox-ovf/clover-devel.ova ./output-virtualbox-ovf/${VM_NAME}
+ls -l output-virtualbox-ovf
 
 echo "--- All done!"
 

http/preseed.cfg

@@ -7,7 +7,7 @@ d-i debconf debconf/frontend select Noninteractive
 d-i debian-installer/locale string en_US.utf8
 
 ### Hardware enablement kernel
-d-i base-installer/kernel/override-image string linux-generic-hwe-18.04
+d-i base-installer/kernel/override-image string linux-generic-hwe-20.04
 
 ### Keyboard selection
 d-i console-setup/ask_detect boolean false

ros_ide_vm.json

@@ -67,10 +67,5 @@
     "type": "shell",
     "script": "scripts/installSoftware.sh",
     "execute_command": "PASSWORD='{{user `password`}}' bash '{{.Path}}'"
-  },
-  {
-    "type": "file",
-    "source": "{{user `assetsDir`}}/vscode/",
-    "destination": "/home/{{user `user`}}/.config/Code/User/"
   }]
 }

scripts/installSoftware.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -e
+set -ex
 
 echo "--- Current environment:"
 /usr/bin/env
@@ -13,26 +13,28 @@ sudo -E sh -c 'echo "APT::Acquire::Retries \"3\";" > /etc/apt/apt.conf.d/80-retr
 cat /etc/apt/apt.conf.d/80-retries
 
 echo "--- Allowing apt to perform its updates"
-sudo -E sh -c 'apt update; while fuser /var/lib/dpkg/lock ; do sleep 0.5 ; done'
+sudo -E sh -c 'apt-get update; while fuser /var/lib/dpkg/lock ; do sleep 0.5 ; done'
 
 echo "--- Installing open-vm-tools"
 
-sudo -E sh -c 'apt update; apt install -y open-vm-tools open-vm-tools-desktop'
+sudo -E sh -c 'apt-get update; apt-get install -y open-vm-tools open-vm-tools-desktop'
 
 echo "--- Installing ROS desktop packages"
 
 sudo -E sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list'
 sudo -E sh -c 'apt-key adv --keyserver "hkp://keyserver.ubuntu.com:80" --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654'
-sudo -E sh -c 'apt update; apt install -y python-rosdep python-rosinstall-generator python-wstool build-essential ros-melodic-desktop'
+sudo -E sh -c 'apt-get update; apt-get install -y python3-rosdep python3-rosinstall-generator python3-wstool build-essential ros-noetic-desktop'
 
 sudo -E sh -c 'rosdep init'
 rosdep update
 
+# FIXME: PX4 needs pip to be installed
+# FIXME: python2 dependencies?
 echo "--- Downloading PX4 and installing its dependencies"
-git clone -b v1.10.1-clover https://github.com/CopterExpress/Firmware ${HOME}/Firmware
-sudo -E -S sh -c '${HOME}/Firmware/Tools/setup/ubuntu.sh'
-sudo -E -S sh -c 'echo "2" | update-alternatives --config java'
-sudo -E -S sed -i -e '/^assistive_technologies=/s/^/#/' /etc/java-*-openjdk/accessibility.properties
+git clone --recursive -b v1.11.1-clover https://github.com/CopterExpress/Firmware ${HOME}/Firmware
+# PX4 v1.11.1 script will happily run sudo by itself
+${HOME}/Firmware/Tools/setup/ubuntu.sh
+# Ubuntu 20.04 no longer sets assistive_technologies, thankfully
 
 echo "--- Prebuilding PX4 SITL configuration"
 make -C ${HOME}/Firmware px4_sitl
@@ -41,16 +43,16 @@ cd ${HOME}/Firmware/Tools/sitl_gazebo
 patch -p1 < /tmp/patches/sitl_gazebo.patch
 echo 'export SVGA_VGPU10=0' >> ${HOME}/Firmware/Tools/setup_gazebo.bash
 
-echo "source /opt/ros/melodic/setup.bash" >> ~/.bashrc
+echo "source /opt/ros/noetic/setup.bash" >> ~/.bashrc
 
 echo "--- Installing Visual Studio Code"
 
-sudo -E sh -c 'apt update; apt install -y curl'
+sudo -E sh -c 'apt-get update; apt-get install -y curl'
 curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > ${HOME}/packages.microsoft.gpg
 sudo -E sh -c 'install -o root -g root -m 644 ${HOME}/packages.microsoft.gpg /usr/share/keyrings'
 rm ${HOME}/packages.microsoft.gpg
 sudo -E sh -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/vscode stable main" > /etc/apt/sources.list.d/vscode.list'
-sudo -E sh -c 'apt install -y apt-transport-https; apt update; apt install -y code'
+sudo -E sh -c 'apt-get install -y apt-transport-https; apt-get update; apt-get install -y code'
 code --install-extension ms-python.python
 code --install-extension DavidAnson.vscode-markdownlint
 code --install-extension ms-vscode.cmake-tools
@@ -58,32 +60,56 @@ code --install-extension ms-vscode.cpptools
 code --install-extension streetsidesoftware.code-spell-checker
 code --install-extension eamodio.gitlens
 echo "--- Installing pylint"
-/usr/bin/python2.7 -m pip install -U "pylint<2.0.0" --user
-/usr/bin/python3.6 -m pip install -U pylint --user
+/usr/bin/python3 -m pip install -U pylint --user
 
 echo "--- Cloning and installing Clover packages"
+sudo sh -c 'curl http://deb.coex.tech/aptly_repo_signing.key 2> /dev/null | apt-key add -'
+sudo sh -c 'echo "deb http://deb.coex.tech/ros xenial main" > /etc/apt/sources.list.d/coex.tech.list'
+sudo sh -c 'echo "yaml file:///etc/ros/rosdep/coex.yaml" > /etc/ros/rosdep/sources.list.d/99-coex.list'
+sudo sh -c 'cat <<EOF > /etc/ros/rosdep/coex.yaml
+led_msgs:
+  ubuntu:
+    focal: [ros-noetic-led-msgs]
+async_web_server_cpp:
+  ubuntu:
+    focal: [ros-noetic-async-web-server-cpp]
+ros_pytest:
+  ubuntu:
+    focal: [ros-noetic-ros-pytest]
+tf2_web_republisher:
+  ubuntu:
+    focal: [ros-noetic-tf2-web-republisher]
+web_video_server:
+  ubuntu:
+    focal: [ros-noetic-web-video-server]
+ws281x:
+  ubuntu:
+    focal: [ros-noetic-ws281x]
+EOF'
+sudo apt-get update
+rosdep update
 mkdir -p ${HOME}/catkin_ws/src
 git clone https://github.com/CopterExpress/clover ${HOME}/catkin_ws/src/clover
 git clone https://github.com/CopterExpress/ros_led ${HOME}/catkin_ws/src/ros_led
+# These packages are missing from Noetic release, but are required for sitl_gazebo
+git clone https://github.com/ethz-asl/mav_comm ${HOME}/catkin_ws/src/mav_comm
 # Make PX4 and Gazebo plugins visible in the workspace
 ln -s ${HOME}/Firmware ${HOME}/catkin_ws/src/Firmware
 ln -s ${HOME}/Firmware/Tools/sitl_gazebo ${HOME}/catkin_ws/src/sitl_gazebo
-rosdep install --from-paths ${HOME}/catkin_ws/src --ignore-src --rosdistro melodic -y
-curl https://raw.githubusercontent.com/mavlink/mavros/master/mavros/scripts/install_geographiclib_datasets.sh -o ${HOME}/install_geographiclib_datasets.sh
-chmod a+x ${HOME}/install_geographiclib_datasets.sh
-sudo -E sh -c '${HOME}/install_geographiclib_datasets.sh'
-sudo /usr/bin/python2.7 -m pip install -r ${HOME}/catkin_ws/src/clover/clover/requirements.txt
-source /opt/ros/melodic/setup.bash
+rosdep install --from-paths ${HOME}/catkin_ws/src --ignore-src --rosdistro noetic -y
+sudo /opt/ros/noetic/lib/mavros/install_geographiclib_datasets.sh
+sudo /usr/bin/python3 -m pip install -r ${HOME}/catkin_ws/src/clover/clover/requirements.txt
+source /opt/ros/noetic/setup.bash
 cd ${HOME}/catkin_ws && catkin_make
 echo "source ${HOME}/catkin_ws/devel/setup.bash" >> ~/.bashrc
 
 echo "--- Installing npm"
 cd ${HOME}
-wget --progress=dot:giga https://nodejs.org/dist/v10.15.0/node-v10.15.0-linux-x64.tar.gz
-tar -xzf node-v10.15.0-linux-x64.tar.gz
-sudo cp -R node-v10.15.0-linux-x64/* /usr/local/
-rm -rf node-v10.15.0-linux-x64
-rm node-v10.15.0-linux-x64.tar.gz
+NODE_VERSION=v10.15.0 # GitBook won't install on newer version
+wget --progress=dot:giga https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-linux-x64.tar.gz
+tar -xzf node-$NODE_VERSION-linux-x64.tar.gz
+sudo cp -R node-$NODE_VERSION-linux-x64/* /usr/local/
+rm -rf node-$NODE_VERSION-linux-x64 node-$NODE_VERSION-linux-x64.tar.gz
 echo "--- Reconfiguring npm to use local prefix"
 mkdir ${HOME}/.npm-global
 npm config set prefix "${HOME}/.npm-global"
@@ -91,13 +117,13 @@ export PATH=${HOME}/.npm-global/bin:$PATH
 echo 'export PATH='${HOME}'/.npm-global/bin:$PATH' >> ${HOME}/.bashrc
 echo "--- Installing gitbook and building docs"
 cd ${HOME}/catkin_ws/src/clover
-NPM_CONFIG_UNSAFE_PERM=true npm install gitbook-cli -g
-NPM_CONFIG_UNSAFE_PERM=true gitbook install
+builder/assets/install_gitbook.sh
+gitbook install
 gitbook build
 touch node_modules/CATKIN_IGNORE docs/CATKIN_IGNORE _book/CATKIN_IGNORE clover/www/CATKIN_IGNORE # ignore documentation files by catkin
 
 echo "--- Exposing examples"
-cp -R ${HOME}/catkin_ws/src/clover/builder/assets/examples ${HOME}/
+ln -s ${HOME}/catkin_ws/src/clover/builder/assets/examples ${HOME}/
 
 echo "--- Enabling roscore service"
 sed -i "s/pi/${USER}/g" ${HOME}/catkin_ws/src/clover/builder/assets/roscore.service
@@ -106,24 +132,23 @@ sudo systemctl enable roscore.service
 
 echo "--- Installing QGroundControl"
 sudo -E sh -c "usermod -a -G dialout $USER"
-sudo -E sh -c 'apt remove -y modemmanager; apt install -y gstreamer1.0-plugins-bad gstreamer1.0-libav'
+sudo -E sh -c 'apt-get remove -y modemmanager; apt-get install -y gstreamer1.0-plugins-bad gstreamer1.0-libav'
 curl https://s3-us-west-2.amazonaws.com/qgroundcontrol/latest/QGroundControl.AppImage -o ${HOME}/QGroundControl.AppImage
 chmod a+x ${HOME}/QGroundControl.AppImage
 
 echo "--- Installing Firefox web browser"
-sudo -E sh -c 'apt update; apt install -y firefox'
+sudo -E sh -c 'apt-get update; apt-get install -y firefox'
 
 echo "--- Installing Monkey web server"
-sudo apt install -y /tmp/packages/monkey_1.6.9-1_amd64.deb
+sudo apt-get install -y /tmp/packages/monkey_1.6.9-1_amd64.deb
 sed "s/pi/${USER}/g" ${HOME}/catkin_ws/src/clover/builder/assets/monkey | sudo tee /etc/monkey/sites/default
 sudo cp ${HOME}/catkin_ws/src/clover/builder/assets/monkey.service /etc/systemd/system/monkey.service
 sudo systemctl enable monkey
 
 echo "--- Installing additional packages"
-sudo -E sh -c 'apt update; apt install -y sshfs gvfs-fuse gvfs-backends python3-opencv byobu ipython ipython3 byobu nmap lsof tmux vim ros-melodic-rqt-multiplot'
+sudo -E sh -c 'apt-get update; apt-get install -y sshfs gvfs-fuse gvfs-backends python3-opencv byobu ipython3 byobu nmap lsof tmux vim ros-noetic-rqt-multiplot'
 
 echo "--- Personalizing VM"
-sudo -E sh -c 'mv /etc/xdg/autostart/light-locker.desktop /etc/xdg/autostart/light-locker.desktop.old'
 sudo -E sh -c 'cp /usr/share/xfce4/backdrops/xubuntu-wallpaper.png /usr/share/xfce4/backdrops/xubuntu-wallpaper-old.png; cp ${HOME}/Pictures/Logo_COEX_2019_white_on_black.png /usr/share/xfce4/backdrops/xubuntu-wallpaper.png'
 sudo -E sh -c 'hostnamectl set-hostname clover-dev; sed -i "s/ubuntu/clover-dev clover-dev.local/g" /etc/hosts'
 echo "export ROS_HOSTNAME=\`hostname\`.local" >> ${HOME}/.bashrc