acid: Manage persistent build logs

The output from build jobs are now stored in dedicated log files instead
of being dumped to stdout.  Files are saved in the new volume at /logs
and intended to be accessed by the web interface for auditing there.

Files are named like "X_Y_Z.log" where X = a unique build number, Y = a
timestamp in UNIX epoch seconds, and Z = the corresponding service name.

Log files are simply closed when the job is finished, so the UI can
check for the existence of a file "/logs/active" which indicates that
the highest-numbered job is still in progress.  This file is removed
afterward.

Failed jobs have a "JOB FAILED" message appended at the end of their
log.  Successful jobs simply lack this message.  One reason this is the
case is due to the build for cychedelic (--self) losing execution before
success can be affirmatively determined.

Signed-off-by: Malfurious <m@lfurio.us>

1 files changed, 43 insertions, 6 deletions

diff --git a/acid/acid b/acid/acid
index b4c0e1b..c675fa1 100755
--- a/acid/acid
+++ b/acid/acid
@@ -16,20 +16,51 @@ should_restart() {
     [ $runtime -gt $CYCHE_MAINT_TIME ]
 }
 
+log_filename() {
+    latest=$(find /logs -type f | sort -rn | sed 's/^\/logs\/0*//;s/_.*$//;1q')
+    [ -z "$latest" ] && number="1" || number=$((latest+1))
+    printf '/logs/%010i_%s_%s.log\n' "$number" "$(now)" "$1"
+}
+
+do_job() {
+    hash=$(acid-source "$1" "$2" "$3")
+    res=$?
+
+    if [ $res -eq 2 ]; then
+        echo -e "\n\nJOB FAILED"
+        return 1
+    fi
+
+    if [ $res -eq 1 ] || [ "$5" == "--force" ]; then
+        if ! acid-build "$1" "$4" "$hash" "$6"; then
+            echo -e "\n\nJOB FAILED"
+        fi
+        return 1
+    fi
+
+    return 0
+}
+
 update() {
     forceopt="--force-self"
     selfopt="--self"
 
     for line in "${CYCHE_SERVICES[@]}"; do
         read -r -a arr <<< "$line"
-        hash=$(acid-source ${arr[@]:0:3})
+
+        log=$(log_filename "${arr[0]}")
+        [ "$1" == "$forceopt" ] && f="--force" || f=""
+
+        touch /logs/active
+        do_job $line "$f" "$selfopt" >"$log" 2>&1
         res=$?
+        rm -f /logs/active
 
-        if [ $res -ne 2 ]; then
-            if [ $res -eq 1 ] || [ "$1" == "$forceopt" ]; then
-                acid-build "${arr[0]}" "${arr[3]}" "$hash" $selfopt
-                sleep "$CYCHE_JOB_TIME"
-            fi
+        if [ $res -ne 0 ]; then
+            echo "Updated ${arr[0]}"
+            sleep "$CYCHE_JOB_TIME"
+        else
+            rm -f "$log"
         fi
 
         forceopt="--force"
@@ -44,6 +75,7 @@ remove() {
 
     for name in $removed; do
         acid-remove "$name"
+        echo "Removed $name"
         sleep "$CYCHE_JOB_TIME"
     done
 }
@@ -52,7 +84,11 @@ remove() {
 # $CYCHE_MAINT_TIME period, so rebuild services and perform cleanup now.  We
 # sleep briefly at the beginning to allow the previous cychedelic instance time
 # to die.
+echo "cychedelic ACID kicking in ..."
+rm -f /logs/active
+
 sleep "$CYCHE_STARTUP_TIME"
+
 remove
 update --force
 docker system prune --force --all
@@ -65,4 +101,5 @@ while ! should_restart; do
 done
 
 # force self-upgrade to trigger maintenance activities
+echo "Forcing daemon restart"
 update --force-self