selfcheck: Check logs for errors

builder/image-software.sh

@@ -103,6 +103,7 @@ epseak speak-data python-espeak \
 ntpdate \
 python-dev \
 python3-dev \
+python-systemd \
 && echo_stamp "Everything was installed!" "SUCCESS" \
 || (echo_stamp "Some packages wasn't installed!" "ERROR"; exit 1)
 

clever/src/selfcheck.py

@@ -13,10 +13,9 @@ from mavros_msgs.srv import ParamGet
 from geometry_msgs.msg import PoseStamped, TwistStamped, PoseWithCovarianceStamped
 import tf.transformations as t
 from aruco_pose.msg import MarkerArray
+from systemd import journal
 
 
-# TODO: roscore is running
-# TODO: clever.service is running
 # TODO: check attitude is present
 # TODO: disk free space
 # TODO: map, base_link, body
@@ -377,7 +376,47 @@ def check_cpu_usage():
                     cpu.strip(), cmd.strip(), pid.strip())
 
 
+def _check_systemd_service(service_name):
+    CMD = 'systemctl show -p ActiveState --value %s' % service_name
+    proc = Popen(CMD.split(), stdout=PIPE)
+    proc.wait()
+    output = proc.communicate()[0]
+    if 'inactive' in output:
+        failure('%s.service is not running' % service_name)
+
+
+@check('Clever service')
+def check_clever_service():
+    _check_systemd_service('clever')
+
+
+# FIXME: Do we really need to check for roscore presence?
+@check('roscore service')
+def check_roscore_service():
+    _check_systemd_service('roscore')
+
+
+@check('Clever logs')
+def check_clever_logs():
+    j = journal.Reader()
+    j.this_boot()
+    j.add_match(_SYSTEMD_UNIT='clever.service')
+    j.add_disjunction()
+    j.add_match(UNIT='clever.service')
+    node_errors = []
+    for event in j:
+        msg = event['MESSAGE']
+        if ('Stopped Clever ROS package' in msg) or ('Started Clever ROS package' in msg):
+            node_errors = []
+        elif ('[ERROR]' in msg) or ('[FATAL]' in msg):
+            node_errors.append(msg)
+    if len(node_errors) > 0:
+        failure('Log contains node errors:\n%s\nRun `journalctl -u clever` for more info', '\n'.join(node_errors))
+
+
 def selfcheck():
+    check_roscore_service()
+    check_clever_service()
     check_fcu()
     check_imu()
     check_local_position()
@@ -391,6 +430,7 @@ def selfcheck():
     check_rangefinder()
     check_cpu_usage()
     check_boot_duration()
+    check_clever_logs()
 
 
 if __name__ == '__main__':