playbook/vifib-server, upgrader: add alternative anacron playbook scheduler

playbook/ors.yml

@@ -3,6 +3,8 @@
   connection: local
 
   vars_files:
+    - settings/vifib.yml
+    - settings/slapos-master.yml
     - settings/ors.yml
 
   roles:

playbook/roles/upgrader/defaults/main.yml

 period_minute: '0'
 period_hour: '*/3'
+extra_playbook: ''
+repeat_until_success: False

playbook/roles/upgrader/tasks/main.yml

 ---
+  - name: Set Cron to repeat until success
+    cron: name="Repeat until success" minute="*/5" hour="*" job="curl -fIs $(grep "^deb" /etc/apt/sources.list| cut -f2 -d' ' | head -n1) && flock -n /opt/upgrader/ansible.lock -c \"rm -rf /opt/upgrader/playbook && cp -R /opt/upgrader/playbook-tmp /opt/upgrader/playbook && cd /opt/upgrader/playbook && PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin ansible-playbook upgrader-run.yml --extra-vars 'extra_playbook={{ extra_playbook }} upgrader_playbook={{ upgrader_playbook }} repeat_until_success={{ repeat_until_success }} upgrade_kernel={{ upgrade_kernel | default(False) == True }}' -i hosts 2>>/opt/upgrader/latest_repeat_upgrade.log >> /opt/upgrader/latest_repeat_upgrade.log\""
+    when: repeat_until_success | bool
+
   - file: path=/opt/upgrader state=directory mode=0755
 
   - stat: path=/opt/upgrader/playbook
@@ -64,21 +68,34 @@
       current_upgrade_signature
 
   - name: Do upgrade
-    shell: ansible-playbook {{ upgrader_playbook }} -i hosts 2>>/opt/upgrader/latest_upgrade.log  >> /opt/upgrader/latest_upgrade.log
-    when: (upgrade_after > lookup('file', '/opt/upgrader/last-upgrade')|int ) and (lookup('file', '/opt/upgrader/next-upgrade-signature') != lookup('file', '/opt/upgrader/current-upgrade-signature'))
+    shell: ansible-playbook {{ upgrader_playbook }} --extra-vars 'repeat_until_success={{ repeat_until_success }}' -i hosts 2>>/opt/upgrader/latest_upgrade.log  >> /opt/upgrader/latest_upgrade.log
+    when: (lookup('file', '/opt/upgrader/playbook-tmp/upgrade_after')|int > lookup('file', '/opt/upgrader/last-upgrade')|int ) and (lookup('file', '/opt/upgrader/next-upgrade-signature') != lookup('file', '/opt/upgrader/current-upgrade-signature'))
     args:
       chdir: /opt/upgrader/playbook-tmp/
 
+  - name: Do extra playbook
+    shell: ansible-playbook {{ extra_playbook }} --extra-vars 'launch_upgrader=False' -i hosts 2>>/opt/upgrader/latest_extra_playbook.log  >> /opt/upgrader/latest_extra_playbook.log
+    when: extra_playbook | length > 0
+
   - name: Set Cron
     cron: name="Launch Upgrader with ansible" minute="{{ period_minute }}" hour="{{ period_hour }}" job="flock -n /opt/upgrader/ansible.lock -c \"rm -rf /opt/upgrader/playbook && cp -R /opt/upgrader/playbook-tmp /opt/upgrader/playbook && cd /opt/upgrader/playbook && PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin ansible-playbook upgrader-run.yml --extra-vars 'period_minute={{ period_minute }} period_hour={{ period_hour }} upgrader_playbook={{ upgrader_playbook }} upgrade_kernel={{ upgrade_kernel | default(False) == True }}' -i hosts 2>>/opt/upgrader/latest_upgrade.log >> /opt/upgrader/latest_upgrade.log\""
+    when: repeat_until_success | bool == False
 
   - cron: name="Launch Upgrader" state=absent
 
   - name: Upgrade Current Upgrade Signature
     shell: cat /opt/upgrader/next-upgrade-signature > /opt/upgrader/current-upgrade-signature
-    when: (upgrade_after > lookup('file', '/opt/upgrader/last-upgrade')|int ) and (lookup('file', '/opt/upgrader/next-upgrade-signature') != lookup('file', '/opt/upgrader/current-upgrade-signature'))
+    when: (lookup('file', '/opt/upgrader/playbook-tmp/upgrade_after')|int > lookup('file', '/opt/upgrader/last-upgrade')|int ) and (lookup('file', '/opt/upgrader/next-upgrade-signature') != lookup('file', '/opt/upgrader/current-upgrade-signature'))
 
   - shell: echo {{ ansible_date_time.epoch }} > /opt/upgrader/last-upgrade
-    when: (upgrade_after > lookup('file', '/opt/upgrader/last-upgrade')|int )
+    when: (lookup('file', '/opt/upgrader/playbook-tmp/upgrade_after')|int > lookup('file', '/opt/upgrader/last-upgrade')|int )
 
   - include: remove-legacy.yml
+
+  - name: Remove cron after successful playbook run
+    cron: name="Repeat until success" state=absent
+    when: repeat_until_success | bool
+
+  - name: Set Anacron after successful playbook run
+    lineinfile: dest=/etc/anacrontab regexp="^[0-9]+ [0-9]+ extra.playbook (.*)" line="1 10 extra.playbook curl -fIs $(grep "^deb" /etc/apt/sources.list| cut -f2 -d' ' | head -n1) && flock -n /opt/upgrader/ansible.lock -c \"rm -rf /opt/upgrader/playbook && cp -R /opt/upgrader/playbook-tmp /opt/upgrader/playbook && cd /opt/upgrader/playbook && PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin ansible-playbook upgrader-run.yml --extra-vars 'extra_playbook={{ extra_playbook }} upgrader_playbook={{ upgrader_playbook }} repeat_until_success={{ repeat_until_success }} upgrade_kernel={{ upgrade_kernel | default(False) == True }}' -i hosts 2>>/opt/upgrader/latest_daily_upgrade.log >> /opt/upgrader/latest_daily_upgrade.log\"" state=present
+    when: repeat_until_success | bool

playbook/roles/vifib-server/defaults/main.yml

+repeat_until_success: False

playbook/roles/vifib-server/tasks/main.yml

@@ -38,6 +38,7 @@
 
   - name: Set Cron
     cron: name="Launch Startup with ansible" special_time=reboot job='flock /opt/upgrader/ansible.lock -c "cd /opt/upgrader/playbook && ansible-playbook vifib-startup.yml -i hosts 2>>/opt/upgrader/startup.log >> /opt/upgrader/startup.log"'
+    when: repeat_until_success | bool == False
 
   - name: Get systemd version
     shell: systemd --version | head -n 1 | cut -d " " -f 2

playbook/roles/vm-bootstrap/tasks/upgrader.yml

@@ -30,8 +30,8 @@
 
   - name: Copy slapos.playbook
     unarchive: src=/opt/upgrader/archive.tar.gz dest=/opt/slapos.playbook
-    when: (upgrade_after|int > lookup('file', '/opt/upgrader/last-upgrade')|int ) and (lookup('file', '/opt/upgrader/next-upgrade-signature') != lookup('file', '/opt/upgrader/current-upgrade-signature'))
+    when: (lookup('file', '/opt/upgrader/playbook-tmp/upgrade_after')|int > lookup('file', '/opt/upgrader/last-upgrade')|int ) and (lookup('file', '/opt/upgrader/next-upgrade-signature') != lookup('file', '/opt/upgrader/current-upgrade-signature'))
 
   - name: Upgrade Current Upgrade Signature
     shell: echo /opt/upgrader/next-upgrade-signature > /opt/upgrader/current-upgrade-signature && echo {{ ansible_date_time.epoch }} > /opt/upgrader/last-upgrade
-    when: (upgrade_after|int > lookup('file', '/opt/upgrader/last-upgrade')|int )
+    when: (lookup('file', '/opt/upgrader/playbook-tmp/upgrade_after')|int > lookup('file', '/opt/upgrader/last-upgrade')|int )

playbook/settings/upgrader.yml

----
-upgrade_after: 1688117268

playbook/sha256sum

-32d040685a3d3cbc9b2a368a9f38a3d1bc47e4289ea14f7395bbf64121e3d08e  -
+572976325d64d618608d19c42dc6e2879086da332887672fc6a7f42c640e2472  -

playbook/upgrade_after

+1688117268

playbook/upgrader-run.yml

@@ -2,10 +2,6 @@
   hosts: 127.0.0.1
   connection: local
 
-  vars_files:
-    - settings/upgrader.yml
-
-
   roles:
     - upgrader
 

playbook/util/update-upgrader-timestamp

 cd $(dirname $(realpath $0))/..
-sed -i "s/upgrade_after:.*/upgrade_after: `date +%s`/g" settings/upgrader.yml 
+date +%s > upgrade_after

playbook/vifib-base.yml

@@ -9,7 +9,6 @@
 
   vars_files:
     - settings/vifib.yml
-    - settings/upgrader.yml
     - settings/slapos-master.yml
 
   roles:

playbook/vifib-upgrader-install.yml

@@ -5,9 +5,6 @@
   vars:
     - upgrader_playbook: vifib-upgrade.yml
 
-  vars_files:
-    - settings/upgrader.yml
-
   roles:
     - upgrader
 

playbook/vifib-without-re6stnet.yml

@@ -7,7 +7,6 @@
 
   vars_files:
     - settings/vifib.yml
-    - settings/upgrader.yml
     - settings/slapos-master.yml
 
   vars_prompt:

playbook/vifib.yml

@@ -7,7 +7,6 @@
 
   vars_files:
     - settings/vifib.yml
-    - settings/upgrader.yml
     - settings/slapos-master.yml
 
   vars_prompt: