Upload Parallels image

https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners

.github/workflows/main.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   build:
-    runs-on: macos-10.15 # Using macOS as https://github.com/actions/virtual-environments/issues/183
+    runs-on: macos-12 # Using macOS as https://github.com/actions/virtual-environments/issues/183
     steps:
       - uses: actions/checkout@v2
       - name: Download base image
@@ -19,6 +19,8 @@ jobs:
           wget --progress=dot:giga https://clovervm.ams3.digitaloceanspaces.com/base-focal.ova -O output-virtualbox-iso/clover-devel.ova
       - name: Build
         run: ./build.sh
+      - name: Image size
+        run: ls -lh output-virtualbox-ovf
       - name: Upload
         if: ${{ github.event_name == 'release' }}
         # Not using ready S3 actions, because they either don't run on macOS machines or don't support changing the endpoint
@@ -26,3 +28,15 @@ jobs:
           AWS_ACCESS_KEY_ID=${{secrets.S3_KEY}} AWS_SECRET_ACCESS_KEY=${{secrets.S3_SECRET_KEY}} aws s3 sync . s3://clovervm/ --acl public-read --endpoint-url https://ams3.digitaloceanspaces.com --no-progress
           echo Uploaded https://clovervm.ams3.digitaloceanspaces.com/$(ls)
         working-directory: output-virtualbox-ovf
+      - name: Install ovftool
+        run: sudo installer -pkg "VMware OVF Tool.pkg" -target /
+      - name: Convert to Parallels
+        run: |
+          mkdir parallels
+          /Applications/VMware\ OVF\ Tool/ovftool --lax output-virtualbox-ovf/*.ova parallels/clover-devel.vmx
+          ls -lh parallels
+      - name: Upload Parallels image
+        run: |
+          AWS_ACCESS_KEY_ID=${{secrets.S3_KEY}} AWS_SECRET_ACCESS_KEY=${{secrets.S3_SECRET_KEY}} aws s3 sync . s3://clovervm/ --acl public-read --endpoint-url https://ams3.digitaloceanspaces.com --no-progress
+          echo Uploaded https://clovervm.ams3.digitaloceanspaces.com/$(ls)
+        working-directory: parallels

assets/clover_vm_version

@@ -0,0 +1 @@
+unknown

assets/desktop/Gazebo Clover.desktop

@@ -1,8 +1,8 @@
 [Desktop Entry]
 Version=1.0
 Type=Application
-Name=Gazebo PX4
+Name=Gazebo Clover
-Comment=Fully-featured Gazebo simulation with default PX4 settings
+Comment=Run Clover drone simulation in Gazebo
 Exec=/bin/bash -c 'source /home/clover/catkin_ws/devel/setup.bash; roslaunch clover_simulation simulator.launch'
 Icon=gazebo
 Path=

assets/desktop/jMAVSim.desktop

@@ -1,10 +0,0 @@
-[Desktop Entry]
-Version=1.0
-Type=Application
-Name=jMAVSim
-Comment=A lightweight PX4 simulator
-Exec=/bin/bash -c 'cd ~/PX4-Autopilot; make px4_sitl jmavsim'
-Icon=openjdk-8
-Path=
-Terminal=true
-StartupNotify=false

assets/patches/sitl_gazebo.patch

@@ -1,28 +1,36 @@
-diff --git a/cmake/FindMAVLink.cmake b/cmake/FindMAVLink.cmake
+diff --git a/src/gazebo_gps_plugin.cpp b/src/gazebo_gps_plugin.cpp
-index 8de3079..3fdae02 100644
+index 1195f5e..6540596 100644
---- a/cmake/FindMAVLink.cmake
+--- a/src/gazebo_gps_plugin.cpp
-+++ b/cmake/FindMAVLink.cmake
++++ b/src/gazebo_gps_plugin.cpp
-@@ -19,6 +19,7 @@ set(_MAVLINK_EXTRA_SEARCH_HINTS
+@@ -41,7 +41,6 @@ GpsPlugin::~GpsPlugin()
-     ${CMAKE_SOURCE_DIR}/mavlink/
+   if (updateSensorConnection_)
-     ../../mavlink/
+     updateSensorConnection_->~Connection();
-     ../mavlink/
+   parentSensor_.reset();
-+    $ENV{HOME}/PX4-Autopilot/mavlink/include
+-  world_->Reset();
-     ${CATKIN_DEVEL_PREFIX}/
+ }
-     )
  
+ void GpsPlugin::Load(sensors::SensorPtr _parent, sdf::ElementPtr _sdf)
+diff --git a/src/gazebo_groundtruth_plugin.cpp b/src/gazebo_groundtruth_plugin.cpp
+index 441ebd9..dd66e12 100644
+--- a/src/gazebo_groundtruth_plugin.cpp
++++ b/src/gazebo_groundtruth_plugin.cpp
+@@ -51,7 +51,6 @@ GroundtruthPlugin::~GroundtruthPlugin()
+ {
+   if (updateConnection_)
+     updateConnection_->~Connection();
+-  world_->Reset();
+ }
  
-diff --git a/package.xml b/package.xml
+ void GroundtruthPlugin::Load(physics::ModelPtr _model, sdf::ElementPtr _sdf)
-index ae0fb34..12f17b8 100644
+diff --git a/src/gazebo_lidar_plugin.cpp b/src/gazebo_lidar_plugin.cpp
---- a/package.xml
+index 38374e6..1b2fbf8 100644
-+++ b/package.xml
+--- a/src/gazebo_lidar_plugin.cpp
-@@ -50,8 +50,8 @@
++++ b/src/gazebo_lidar_plugin.cpp
-   <build_depend>roscpp</build_depend>
+@@ -47,7 +47,6 @@ LidarPlugin::~LidarPlugin()
-   <build_depend>sensor_msgs</build_depend>
+   newLaserScansConnection_->~Connection();
-   <build_depend>std_msgs</build_depend>
+   newLaserScansConnection_.reset();
--  <build_depend>python-rospkg</build_depend>
+   parentSensor_.reset();
--  <build_depend>python-jinja2</build_depend>
+-  world_->Reset();
-+  <build_depend>python3-rospkg</build_depend>
+ }
-+  <build_depend>python3-jinja2</build_depend>
+ 
-   <run_depend>eigen</run_depend>
+ /////////////////////////////////////////////////
-   <run_depend>gazebo_ros</run_depend>
-   <run_depend>geometry_msgs</run_depend>

build.sh

@@ -5,6 +5,18 @@ set -ex
 # FIXME: Use system Packer if possible
 #PACKER=$(which packer)
 
+echo "--- Fill version asset"
+# if [[ $GITHUB_REF == refs/tags/*-rc* ]]; then
+    # remove rc label
+    # VERSION=${GITHUB_REF#refs/tags/}
+    # VERSION=${VERSION/-rc*/}
+if [[ $GITHUB_REF == refs/tags/* ]]; then
+    VERSION=${GITHUB_REF#refs/tags/}
+else
+    VERSION=$(git describe --always)
+fi
+echo $VERSION > assets/clover_vm_version
+
 #if [ "x${PACKER}" == "x" ]; then
 PACKER="./packer"
 if [ ! -f ${PACKER} ]; then
@@ -32,16 +44,6 @@ ${PACKER} build ros_ide_vm.json
 
 echo "--- Marking the VM"
 
-# if [[ $GITHUB_REF == refs/tags/*-rc* ]]; then
-    # remove rc label
-    # VERSION=${GITHUB_REF#refs/tags/}
-    # VERSION=${VERSION/-rc*/}
-if [[ $GITHUB_REF == refs/tags/* ]]; then
-    VERSION=${GITHUB_REF#refs/tags/}
-else
-    VERSION=$(git describe --always)
-fi
-
 VM_NAME="clover-devel_${VERSION}.ova"
 mv ./output-virtualbox-ovf/clover-devel.ova ./output-virtualbox-ovf/${VM_NAME}
 ls -l output-virtualbox-ovf

ros_ide_vm.json

@@ -63,6 +63,11 @@
     "source": "{{user `assetsDir`}}/patches",
     "destination": "/tmp"
   },
+  {
+    "type": "file",
+    "source": "{{user `assetsDir`}}/clover_vm_version",
+    "destination": "/tmp/clover_vm_version"
+  },
   {
     "type": "shell",
     "script": "scripts/install_software.sh",

scripts/install_software.sh

@@ -12,45 +12,84 @@ echo "--- Increasing apt retries"
 sudo -E sh -c 'echo "APT::Acquire::Retries \"3\";" > /etc/apt/apt.conf.d/80-retries'
 cat /etc/apt/apt.conf.d/80-retries
 
+echo "--- Updating apt"
+sudo -E sh -c 'apt-get update'
+
 echo "--- Allowing apt to perform its updates"
-sudo -E sh -c 'apt-get update; while fuser /var/lib/dpkg/lock ; do sleep 0.5 ; done'
+sudo -E sh -c 'while fuser /var/lib/dpkg/lock ; do sleep 0.5 ; done'
 
 echo "--- Installing open-vm-tools"
+sudo -E sh -c 'apt-get install -y open-vm-tools open-vm-tools-desktop'
 
-sudo -E sh -c 'apt-get update; apt-get install -y open-vm-tools open-vm-tools-desktop'
+echo "--- Installing ROS desktop"
+sudo -E sh -c 'apt-get install -y curl'
+sudo sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list'
+curl -s https://raw.githubusercontent.com/ros/rosdistro/master/ros.asc | sudo apt-key add -
+sudo -E sh -c 'apt-get update; apt-get install -y python3-pip python3-rosdep python3-rosinstall-generator python3-wstool build-essential ros-noetic-desktop'
 
-echo "--- Installing ROS desktop packages"
+echo "source /opt/ros/noetic/setup.bash" >> ~/.bashrc
-
+source /opt/ros/noetic/setup.bash
-sudo -E sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list'
-sudo -E sh -c 'apt-key adv --keyserver "hkp://keyserver.ubuntu.com:80" --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654'
-sudo -E sh -c 'apt-get update; apt-get install -y python3-rosdep python3-rosinstall-generator python3-wstool build-essential ros-noetic-desktop'
 
+echo "--- Updating rosdep"
 sudo -E sh -c 'rosdep init'
 rosdep update
 
-# FIXME: PX4 needs pip to be installed
+echo "--- Creating Catkin workspace"
-# FIXME: python2 dependencies?
+mkdir -p ~/catkin_ws/src
-echo "--- Downloading PX4 and installing its dependencies"
+cd ~/catkin_ws
-git clone --recursive --depth 1 --branch v1.12.0 https://github.com/PX4/PX4-Autopilot.git ${HOME}/PX4-Autopilot
+catkin_make
-# PX4 v1.11.1 script will happily run sudo by itself
+echo "source ~/catkin_ws/devel/setup.bash" >> ~/.bashrc
-${HOME}/PX4-Autopilot/Tools/setup/ubuntu.sh
+source ~/catkin_ws/devel/setup.bash
-# Ubuntu 20.04 no longer sets assistive_technologies, thankfully
-sudo -E sh -c 'apt-get install -y ant openjdk-11-jdk' # Additional packages for jMAVSim
 
-echo "--- Prebuilding PX4 SITL configuration"
+echo "--- Getting Clover sources"
-make -C ${HOME}/PX4-Autopilot px4_sitl
+cd ~/catkin_ws/src
-echo "--- Patching gazebo plugins for SITL"
+git clone --depth 1 https://github.com/CopterExpress/clover
-# cd ${HOME}/PX4-Autopilot/Tools/sitl_gazebo
+git clone --depth 1 https://github.com/CopterExpress/ros_led
-# patch -p1 < /tmp/patches/sitl_gazebo.patch
+git clone --depth 1 https://github.com/ethz-asl/mav_comm
-echo 'export SVGA_VGPU10=0' >> ${HOME}/PX4-Autopilot/Tools/setup_gazebo.bash
 
+echo "--- Installing dependencies with rosdep"
+cd ~/catkin_ws
+rosdep install --from-paths src --ignore-src -y
+
+echo "--- Installing Clover's Python dependencies"
+sudo -E sh -c '/usr/bin/python3 -m pip install -r ~/catkin_ws/src/clover/clover/requirements.txt'
+
+echo "--- Downloading PX4"
+git clone --recursive --depth 1 --branch v1.13.0 https://github.com/PX4/PX4-Autopilot.git ~/PX4-Autopilot
+ln -s ~/PX4-Autopilot ~/catkin_ws/src/
+ln -s ~/PX4-Autopilot/Tools/sitl_gazebo ~/catkin_ws/src/
+
+echo "--- Installing PX4 dependencies"
+echo "progress=dot:giga" > ~/.wgetrc # make wget don't spam to log
+~/PX4-Autopilot/Tools/setup/ubuntu.sh --no-nuttx
+rm ~/.wgetrc
+pip3 install --user toml
+# sudo -E sh -c 'apt-get install -y ant openjdk-11-jdk' # Additional packages for jMAVSim
+
+echo "--- Patching mavlink_sitl_gazebo"
+# See https://github.com/PX4/PX4-SITL_gazebo/pull/872
+cd ~/PX4-Autopilot/Tools/sitl_gazebo
+patch -p1 < /tmp/patches/sitl_gazebo.patch
+
+echo "--- Addding Gazebo initialization to bashrc"
 echo "source /usr/share/gazebo/setup.sh" >> ~/.bashrc
 echo "export SVGA_VGPU10=0" >> ~/.bashrc
 
-echo "source /opt/ros/noetic/setup.bash" >> ~/.bashrc
+echo "--- Addding Clover airframe"
+ln -s ~/catkin_ws/src/clover/clover_simulation/airframes/* ~/PX4-Autopilot/ROMFS/px4fmu_common/init.d-posix/airframes/
+
+echo "--- Installing geographiclib datasets"
+sudo -E sh -c '/opt/ros/noetic/lib/mavros/install_geographiclib_datasets.sh'
+
+echo "--- Build mavlink"
+cd ~/catkin_ws
+catkin_make mavlink_c_generate -DCATKIN_WHITELIST_PACKAGES="px4"  # at first build PX4's mavlink to enforce mavlink_sitl_gazebo using it
+ln -s "." build/mavlink/mavlink  # fix https://github.com/PX4/PX4-Autopilot/pull/19964
+
+echo "--- Building the workspace"
+catkin_make -DCATKIN_WHITELIST_PACKAGES=""
 
 echo "--- Installing Visual Studio Code"
-
 sudo -E sh -c 'apt-get update; apt-get install -y curl'
 curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > ${HOME}/packages.microsoft.gpg
 sudo -E sh -c 'install -o root -g root -m 644 ${HOME}/packages.microsoft.gpg /usr/share/keyrings'
@@ -63,56 +102,15 @@ code --install-extension ms-vscode.cmake-tools
 code --install-extension ms-vscode.cpptools
 code --install-extension streetsidesoftware.code-spell-checker
 code --install-extension eamodio.gitlens
-echo "--- Installing pylint"
-/usr/bin/python3 -m pip install -U pylint --user
 
-echo "--- Cloning and installing Clover packages"
+echo "--- Installing additional Python packages"
-sudo sh -c 'curl http://deb.coex.tech/aptly_repo_signing.key 2> /dev/null | apt-key add -'
+/usr/bin/python3 -m pip install -U pylint pyzbar --user
-sudo sh -c 'echo "deb http://deb.coex.tech/ros xenial main" > /etc/apt/sources.list.d/coex.tech.list'
-sudo sh -c 'echo "yaml file:///etc/ros/rosdep/coex.yaml" > /etc/ros/rosdep/sources.list.d/99-coex.list'
-sudo sh -c 'cat <<EOF > /etc/ros/rosdep/coex.yaml
-led_msgs:
-  ubuntu:
-    focal: [ros-noetic-led-msgs]
-async_web_server_cpp:
-  ubuntu:
-    focal: [ros-noetic-async-web-server-cpp]
-ros_pytest:
-  ubuntu:
-    focal: [ros-noetic-ros-pytest]
-tf2_web_republisher:
-  ubuntu:
-    focal: [ros-noetic-tf2-web-republisher]
-web_video_server:
-  ubuntu:
-    focal: [ros-noetic-web-video-server]
-ws281x:
-  ubuntu:
-    focal: [ros-noetic-ws281x]
-EOF'
-sudo apt-get update
-rosdep update
-mkdir -p ${HOME}/catkin_ws/src
-git clone https://github.com/CopterExpress/clover ${HOME}/catkin_ws/src/clover
-git clone https://github.com/CopterExpress/ros_led ${HOME}/catkin_ws/src/ros_led
-# These packages are missing from Noetic release, but are required for sitl_gazebo
-git clone https://github.com/ethz-asl/mav_comm ${HOME}/catkin_ws/src/mav_comm
-# Make PX4 and Gazebo plugins visible in the workspace
-ln -s ${HOME}/PX4-Autopilot ${HOME}/catkin_ws/src/PX4-Autopilot
-ln -s ${HOME}/PX4-Autopilot/Tools/sitl_gazebo ${HOME}/catkin_ws/src/sitl_gazebo
-rosdep install --from-paths ${HOME}/catkin_ws/src --ignore-src --rosdistro noetic -y
-sudo /opt/ros/noetic/lib/mavros/install_geographiclib_datasets.sh
-sudo /usr/bin/python3 -m pip install -r ${HOME}/catkin_ws/src/clover/clover/requirements.txt
-source /opt/ros/noetic/setup.bash
-cd ${HOME}/catkin_ws && catkin_make
-echo "source ${HOME}/catkin_ws/devel/setup.bash" >> ~/.bashrc
 
-source $HOME/catkin_ws/devel/setup.bash
+echo "--- Exposing examples"
+ln -s ${HOME}/catkin_ws/src/clover/clover/examples ${HOME}/
+[[ -d ${HOME}/examples ]] # test symlink is valid
 
-echo "--- Adding Clover vehicle startup script to PX4"
+echo "--- Installing npm and building documentation"
-ln -s "$(catkin_find clover_simulation airframes)"/* $HOME/PX4-Autopilot/ROMFS/px4fmu_common/init.d-posix/airframes/
-
-echo "--- Installing npm"
 cd ${HOME}
 NODE_VERSION=v10.15.0 # GitBook won't install on newer version
 wget --progress=dot:giga https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-linux-x64.tar.gz
@@ -131,10 +129,6 @@ gitbook install
 gitbook build
 touch node_modules/CATKIN_IGNORE docs/CATKIN_IGNORE _book/CATKIN_IGNORE clover/www/CATKIN_IGNORE # ignore documentation files by catkin
 
-echo "--- Exposing examples"
-ln -s ${HOME}/catkin_ws/src/clover/clover/examples ${HOME}/
-[[ -d ${HOME}/examples ]] # test symlink is valid
-
 echo "--- Enabling roscore service"
 sed -i "s/pi/${USER}/g" ${HOME}/catkin_ws/src/clover/builder/assets/roscore.service
 sudo cp ${HOME}/catkin_ws/src/clover/builder/assets/roscore.service /etc/systemd/system
@@ -152,11 +146,12 @@ sudo -E sh -c 'apt-get update; apt-get install -y firefox'
 echo "--- Installing Monkey web server"
 sudo apt-get install -y /tmp/packages/monkey_1.6.9-1_amd64.deb
 sed "s/pi/${USER}/g" ${HOME}/catkin_ws/src/clover/builder/assets/monkey | sudo tee /etc/monkey/sites/default
+sudo -E sh -c "sed -i 's/SymLink Off/SymLink On/' /etc/monkey/monkey.conf"
 sudo cp ${HOME}/catkin_ws/src/clover/builder/assets/monkey.service /etc/systemd/system/monkey.service
 sudo systemctl enable monkey
 
 echo "--- Installing additional packages"
-sudo -E sh -c 'apt-get update; apt-get install -y sshfs gvfs-fuse gvfs-backends python3-opencv byobu ipython3 byobu nmap lsof tmux vim ros-noetic-rqt-multiplot'
+sudo -E sh -c 'apt-get update; apt-get install -y sshfs gvfs-fuse gvfs-backends python3-opencv byobu ipython3 byobu nmap lsof tmux vim ros-noetic-rqt-multiplot ros-noetic-image-view'
 
 echo "--- Personalizing VM"
 sudo -E sh -c 'cp /usr/share/xfce4/backdrops/xubuntu-wallpaper.png /usr/share/xfce4/backdrops/xubuntu-wallpaper-old.png; cp ${HOME}/Pictures/Logo_COEX_2019_white_on_black.png /usr/share/xfce4/backdrops/xubuntu-wallpaper.png'
@@ -164,6 +159,10 @@ sudo -E sh -c 'hostnamectl set-hostname clover-dev; sed -i "s/ubuntu/clover-dev
 echo "export ROS_HOSTNAME=\`hostname\`.local" >> ${HOME}/.bashrc
 chmod a+x ${HOME}/Desktop/*
 
+echo "--- Creating /etc/clover_vm_version"
+sudo -E sh -c 'mv /tmp/clover_vm_version /etc/clover_vm_version'
+cat /etc/clover_vm_version
+
 echo "--- Cleaning up"
 sudo -E sh -c 'apt-get -y autoremove; apt-get -y autoclean; apt-get -y clean; fstrim -v /'
 
@@ -200,10 +199,31 @@ rosversion tf2_web_republisher
 rosversion cv_camera
 rosversion web_video_server
 rosversion nodelet
+rosversion image_view
 
-echo "Trying running the Gazebo simulator, check the output"
+echo "--- Validating PX4 builds"
+cd ~/PX4-Autopilot
+make px4_sitl  # regular sitl build
+# sudo -E sh -c 'apt-get install -y gcc-arm-none-eabi'
+# make px4_fmu-v4_default
+make clean
+
+echo "--- Run Clover's Python libraries validation"
+$HOME/catkin_ws/src/clover/builder/test/tests_py3.py
+
+echo "--- Versions of all installed ROS packages"
+set +x
+rospack list-names | while read line; do echo $line `rosversion $line`; done
+set -x
+
+echo "--- Largest packages installed"
+sudo -E sh -c 'apt-get install -y debian-goodies'
+dpigs -H -n 30
+
+echo "--- Trying running the Gazebo simulator, check the output"
 timeout --preserve-status 30 roslaunch clover_simulation simulator.launch gui:=false --screen
 
-echo "Trying running jMAVSim, check the output"
+echo "--- Trying running jMAVSim, check the output"
-cd ~/PX4-Autopilot
+# cd ~/PX4-Autopilot
-HEADLESS=1 timeout --preserve-status 30 make px4_sitl jmavsim
+# HEADLESS=1 timeout --preserve-status 30 make px4_sitl jmavsim
+HEADLESS=1 timeout --preserve-status 30 roslaunch clover_simulation simulator.launch type:=jmavsim gui:=false --screen