Update PX4 to 1.13.3

To allow triggering when Clover package gets updated

.github/workflows/main.yml

@@ -7,10 +7,11 @@ on:
     branches: [ master ]
   release:
     types: [ created ]
+  workflow_dispatch:
 
 jobs:
   build:
-    runs-on: macos-10.15 # Using macOS as https://github.com/actions/virtual-environments/issues/183
+    runs-on: macos-12 # Using macOS as https://github.com/actions/virtual-environments/issues/183
     steps:
       - uses: actions/checkout@v2
       - name: Download base image
@@ -19,6 +20,8 @@ jobs:
           wget --progress=dot:giga https://clovervm.ams3.digitaloceanspaces.com/base-focal.ova -O output-virtualbox-iso/clover-devel.ova
       - name: Build
         run: ./build.sh
+      - name: Image size
+        run: ls -lh output-virtualbox-ovf
       - name: Upload
         if: ${{ github.event_name == 'release' }}
         # Not using ready S3 actions, because they either don't run on macOS machines or don't support changing the endpoint

assets/clover_vm_version

@@ -0,0 +1 @@
+unknown

build.sh

@@ -5,6 +5,18 @@ set -ex
 # FIXME: Use system Packer if possible
 #PACKER=$(which packer)
 
+echo "--- Fill version asset"
+# if [[ $GITHUB_REF == refs/tags/*-rc* ]]; then
+    # remove rc label
+    # VERSION=${GITHUB_REF#refs/tags/}
+    # VERSION=${VERSION/-rc*/}
+if [[ $GITHUB_REF == refs/tags/* ]]; then
+    VERSION=${GITHUB_REF#refs/tags/}
+else
+    VERSION=$(git describe --always)
+fi
+echo $VERSION > assets/clover_vm_version
+
 #if [ "x${PACKER}" == "x" ]; then
 PACKER="./packer"
 if [ ! -f ${PACKER} ]; then
@@ -32,16 +44,6 @@ ${PACKER} build ros_ide_vm.json
 
 echo "--- Marking the VM"
 
-# if [[ $GITHUB_REF == refs/tags/*-rc* ]]; then
-    # remove rc label
-    # VERSION=${GITHUB_REF#refs/tags/}
-    # VERSION=${VERSION/-rc*/}
-if [[ $GITHUB_REF == refs/tags/* ]]; then
-    VERSION=${GITHUB_REF#refs/tags/}
-else
-    VERSION=$(git describe --always)
-fi
-
 VM_NAME="clover-devel_${VERSION}.ova"
 mv ./output-virtualbox-ovf/clover-devel.ova ./output-virtualbox-ovf/${VM_NAME}
 ls -l output-virtualbox-ovf

ros_ide_vm.json

@@ -63,6 +63,11 @@
     "source": "{{user `assetsDir`}}/patches",
     "destination": "/tmp"
   },
+  {
+    "type": "file",
+    "source": "{{user `assetsDir`}}/clover_vm_version",
+    "destination": "/tmp/clover_vm_version"
+  },
   {
     "type": "shell",
     "script": "scripts/install_software.sh",

scripts/install_software.sh

@@ -55,15 +55,16 @@ echo "--- Installing Clover's Python dependencies"
 sudo -E sh -c '/usr/bin/python3 -m pip install -r ~/catkin_ws/src/clover/clover/requirements.txt'
 
 echo "--- Downloading PX4"
-git clone --recursive --depth 1 --branch v1.12.3 https://github.com/PX4/PX4-Autopilot.git ~/PX4-Autopilot
+git clone --recursive --depth 1 --branch v1.13.3 https://github.com/PX4/PX4-Autopilot.git ~/PX4-Autopilot
 ln -s ~/PX4-Autopilot ~/catkin_ws/src/
 ln -s ~/PX4-Autopilot/Tools/sitl_gazebo ~/catkin_ws/src/
-ln -s ~/PX4-Autopilot/mavlink ~/catkin_ws/src/
 
 echo "--- Installing PX4 dependencies"
-~/PX4-Autopilot/Tools/setup/ubuntu.sh
+echo "progress=dot:giga" > ~/.wgetrc # make wget don't spam to log
+~/PX4-Autopilot/Tools/setup/ubuntu.sh --no-nuttx
+rm ~/.wgetrc
 pip3 install --user toml
-sudo -E sh -c 'apt-get install -y ant openjdk-11-jdk' # Additional packages for jMAVSim
+# sudo -E sh -c 'apt-get install -y ant openjdk-11-jdk' # Additional packages for jMAVSim
 
 echo "--- Patching mavlink_sitl_gazebo"
 # See https://github.com/PX4/PX4-SITL_gazebo/pull/872
@@ -80,9 +81,16 @@ ln -s ~/catkin_ws/src/clover/clover_simulation/airframes/* ~/PX4-Autopilot/ROMFS
 echo "--- Installing geographiclib datasets"
 sudo -E sh -c '/opt/ros/noetic/lib/mavros/install_geographiclib_datasets.sh'
 
-echo "--- Building the workspace"
+echo "--- Build mavlink"
 cd ~/catkin_ws
-catkin_make
+catkin_make mavlink_c_generate -DCATKIN_WHITELIST_PACKAGES="px4"  # at first build PX4's mavlink to enforce mavlink_sitl_gazebo using it
+ln -s "." build/mavlink/mavlink  # fix https://github.com/PX4/PX4-Autopilot/pull/19964
+
+echo "--- Building the workspace"
+catkin_make -DCATKIN_WHITELIST_PACKAGES=""
+
+echo "--- Running tests"
+catkin_make run_tests && catkin_test_results
 
 echo "--- Installing Visual Studio Code"
 sudo -E sh -c 'apt-get update; apt-get install -y curl'
@@ -144,9 +152,10 @@ sed "s/pi/${USER}/g" ${HOME}/catkin_ws/src/clover/builder/assets/monkey | sudo t
 sudo -E sh -c "sed -i 's/SymLink Off/SymLink On/' /etc/monkey/monkey.conf"
 sudo cp ${HOME}/catkin_ws/src/clover/builder/assets/monkey.service /etc/systemd/system/monkey.service
 sudo systemctl enable monkey
+rosrun clover www # create directory to serve
 
 echo "--- Installing additional packages"
-sudo -E sh -c 'apt-get update; apt-get install -y sshfs gvfs-fuse gvfs-backends python3-opencv byobu ipython3 byobu nmap lsof tmux vim ros-noetic-rqt-multiplot'
+sudo -E sh -c 'apt-get update; apt-get install -y sshfs gvfs-fuse gvfs-backends python3-opencv byobu ipython3 byobu nmap lsof tmux vim ros-noetic-usb-cam ros-noetic-rqt-multiplot ros-noetic-image-view'
 
 echo "--- Personalizing VM"
 sudo -E sh -c 'cp /usr/share/xfce4/backdrops/xubuntu-wallpaper.png /usr/share/xfce4/backdrops/xubuntu-wallpaper-old.png; cp ${HOME}/Pictures/Logo_COEX_2019_white_on_black.png /usr/share/xfce4/backdrops/xubuntu-wallpaper.png'
@@ -154,6 +163,10 @@ sudo -E sh -c 'hostnamectl set-hostname clover-dev; sed -i "s/ubuntu/clover-dev
 echo "export ROS_HOSTNAME=\`hostname\`.local" >> ${HOME}/.bashrc
 chmod a+x ${HOME}/Desktop/*
 
+echo "--- Creating /etc/clover_vm_version"
+sudo -E sh -c 'mv /tmp/clover_vm_version /etc/clover_vm_version'
+cat /etc/clover_vm_version
+
 echo "--- Cleaning up"
 sudo -E sh -c 'apt-get -y autoremove; apt-get -y autoclean; apt-get -y clean; fstrim -v /'
 
@@ -173,7 +186,6 @@ pip --version
 pip3 --version
 monkey --version
 systemctl --version
-# TODO: add Python tests
 
 roscore -h
 rosversion px4
@@ -190,19 +202,35 @@ rosversion tf2_web_republisher
 rosversion cv_camera
 rosversion web_video_server
 rosversion nodelet
+rosversion image_view
 
-echo "--- Run Clover's Python libraries validation"
+echo "--- Validating PX4 builds"
+cd ~/PX4-Autopilot
+make px4_sitl  # regular sitl build
+# sudo -E sh -c 'apt-get install -y gcc-arm-none-eabi'
+# make px4_fmu-v4_default
+make clean
+
+echo "--- Run Clover's Python and shell tests"
+source ~/catkin_ws/devel/setup.bash
+export VM=1
+$HOME/catkin_ws/src/clover/builder/test/tests.py
 $HOME/catkin_ws/src/clover/builder/test/tests_py3.py
+$HOME/catkin_ws/src/clover/builder/test/tests.sh
 
 echo "--- Versions of all installed ROS packages"
 set +x
 rospack list-names | while read line; do echo $line `rosversion $line`; done
 set -x
 
-echo "Trying running the Gazebo simulator, check the output"
+echo "--- Largest packages installed"
+sudo -E sh -c 'apt-get install -y debian-goodies'
+dpigs -H -n 30
+
+echo "--- Trying running the Gazebo simulator, check the output"
 timeout --preserve-status 30 roslaunch clover_simulation simulator.launch gui:=false --screen
 
-echo "Trying running jMAVSim, check the output"
+echo "--- Trying running jMAVSim, check the output"
 # cd ~/PX4-Autopilot
 # HEADLESS=1 timeout --preserve-status 30 make px4_sitl jmavsim
 HEADLESS=1 timeout --preserve-status 30 roslaunch clover_simulation simulator.launch type:=jmavsim gui:=false --screen