[pve-devel] [RFC ha-manager v2 3/3] add regression test for 'stopped' state

Thomas Lamprecht t.lamprecht at proxmox.com
Tue Nov 15 11:13:57 CET 2016


Test that:
* stopped service will not start
* stopped service will be recovered
* stopped service can me relocated

Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
---
 src/test/test-service-stopped1/README          |  5 ++++
 src/test/test-service-stopped1/cmdlist         |  3 ++
 src/test/test-service-stopped1/hardware_status |  5 ++++
 src/test/test-service-stopped1/log.expect      | 22 ++++++++++++++
 src/test/test-service-stopped1/manager_status  |  1 +
 src/test/test-service-stopped1/service_config  |  3 ++
 src/test/test-service-stopped2/README          |  5 ++++
 src/test/test-service-stopped2/cmdlist         |  5 ++++
 src/test/test-service-stopped2/hardware_status |  5 ++++
 src/test/test-service-stopped2/log.expect      | 41 ++++++++++++++++++++++++++
 src/test/test-service-stopped2/manager_status  |  1 +
 src/test/test-service-stopped2/service_config  |  3 ++
 src/test/test-service-stopped3/README          |  6 ++++
 src/test/test-service-stopped3/cmdlist         |  4 +++
 src/test/test-service-stopped3/hardware_status |  5 ++++
 src/test/test-service-stopped3/log.expect      | 31 +++++++++++++++++++
 src/test/test-service-stopped3/manager_status  |  1 +
 src/test/test-service-stopped3/service_config  |  3 ++
 18 files changed, 149 insertions(+)
 create mode 100644 src/test/test-service-stopped1/README
 create mode 100644 src/test/test-service-stopped1/cmdlist
 create mode 100644 src/test/test-service-stopped1/hardware_status
 create mode 100644 src/test/test-service-stopped1/log.expect
 create mode 100644 src/test/test-service-stopped1/manager_status
 create mode 100644 src/test/test-service-stopped1/service_config
 create mode 100644 src/test/test-service-stopped2/README
 create mode 100644 src/test/test-service-stopped2/cmdlist
 create mode 100644 src/test/test-service-stopped2/hardware_status
 create mode 100644 src/test/test-service-stopped2/log.expect
 create mode 100644 src/test/test-service-stopped2/manager_status
 create mode 100644 src/test/test-service-stopped2/service_config
 create mode 100644 src/test/test-service-stopped3/README
 create mode 100644 src/test/test-service-stopped3/cmdlist
 create mode 100644 src/test/test-service-stopped3/hardware_status
 create mode 100644 src/test/test-service-stopped3/log.expect
 create mode 100644 src/test/test-service-stopped3/manager_status
 create mode 100644 src/test/test-service-stopped3/service_config

diff --git a/src/test/test-service-stopped1/README b/src/test/test-service-stopped1/README
new file mode 100644
index 0000000..8124238
--- /dev/null
+++ b/src/test/test-service-stopped1/README
@@ -0,0 +1,5 @@
+Test if a simulated template in the stopped state does not cause errors.
+
+The template gets simulated by using the VirtFail resource, the configuration
+fa:1501 tells the resource to fail to start and to fail to stop, same behavior
+as a template
diff --git a/src/test/test-service-stopped1/cmdlist b/src/test/test-service-stopped1/cmdlist
new file mode 100644
index 0000000..3bfad44
--- /dev/null
+++ b/src/test/test-service-stopped1/cmdlist
@@ -0,0 +1,3 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"]
+]
diff --git a/src/test/test-service-stopped1/hardware_status b/src/test/test-service-stopped1/hardware_status
new file mode 100644
index 0000000..451beb1
--- /dev/null
+++ b/src/test/test-service-stopped1/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
diff --git a/src/test/test-service-stopped1/log.expect b/src/test/test-service-stopped1/log.expect
new file mode 100644
index 0000000..6dc457b
--- /dev/null
+++ b/src/test/test-service-stopped1/log.expect
@@ -0,0 +1,22 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'fa:1501' on node 'node3'
+info     22    node2/crm: status change wait_for_quorum => slave
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     40    node1/crm: service 'fa:1501': state changed from 'request_stop' to 'stopped'
+info    620     hardware: exit simulation - done
diff --git a/src/test/test-service-stopped1/manager_status b/src/test/test-service-stopped1/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-service-stopped1/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-service-stopped1/service_config b/src/test/test-service-stopped1/service_config
new file mode 100644
index 0000000..8ae9079
--- /dev/null
+++ b/src/test/test-service-stopped1/service_config
@@ -0,0 +1,3 @@
+{
+    "fa:1501": { "node": "node3", "state": "stopped" }
+}
diff --git a/src/test/test-service-stopped2/README b/src/test/test-service-stopped2/README
new file mode 100644
index 0000000..06457bc
--- /dev/null
+++ b/src/test/test-service-stopped2/README
@@ -0,0 +1,5 @@
+Test if a simulated template in the stopped state recovers if its node fails.
+
+The template gets simulated by using the VirtFail resource, the configuration
+fa:1501 tells the resource to fail to start and to fail to stop, same behavior
+as a template
diff --git a/src/test/test-service-stopped2/cmdlist b/src/test/test-service-stopped2/cmdlist
new file mode 100644
index 0000000..43d56e5
--- /dev/null
+++ b/src/test/test-service-stopped2/cmdlist
@@ -0,0 +1,5 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "service fa:1501 stopped" ],
+    [ "network node3 off" ]
+]
diff --git a/src/test/test-service-stopped2/hardware_status b/src/test/test-service-stopped2/hardware_status
new file mode 100644
index 0000000..451beb1
--- /dev/null
+++ b/src/test/test-service-stopped2/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
diff --git a/src/test/test-service-stopped2/log.expect b/src/test/test-service-stopped2/log.expect
new file mode 100644
index 0000000..f8166ca
--- /dev/null
+++ b/src/test/test-service-stopped2/log.expect
@@ -0,0 +1,41 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'fa:1501' on node 'node3'
+info     22    node2/crm: status change wait_for_quorum => slave
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     40    node1/crm: service 'fa:1501': state changed from 'request_stop' to 'stopped'
+info    120      cmdlist: execute service fa:1501 stopped
+info    220      cmdlist: execute network node3 off
+info    220    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    224    node3/crm: status change slave => wait_for_quorum
+info    225    node3/lrm: status change active => lost_agent_lock
+info    260    node1/crm: service 'fa:1501': state changed from 'stopped' to 'fence'
+info    260    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+info    266     watchdog: execute power node3 off
+info    265    node3/crm: killed by poweroff
+info    266    node3/lrm: killed by poweroff
+info    266     hardware: server 'node3' stopped by poweroff (watchdog)
+info    340    node1/crm: got lock 'ha_agent_node3_lock'
+info    340    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
+info    340    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+info    340    node1/crm: recover service 'fa:1501' from fenced node 'node3' to node 'node1'
+info    340    node1/crm: service 'fa:1501': state changed from 'fence' to 'request_stop'  (node = node1)
+info    341    node1/lrm: got lock 'ha_agent_node1_lock'
+info    341    node1/lrm: status change wait_for_agent_lock => active
+info    360    node1/crm: service 'fa:1501': state changed from 'request_stop' to 'stopped'
+info    820     hardware: exit simulation - done
diff --git a/src/test/test-service-stopped2/manager_status b/src/test/test-service-stopped2/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-service-stopped2/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-service-stopped2/service_config b/src/test/test-service-stopped2/service_config
new file mode 100644
index 0000000..064553e
--- /dev/null
+++ b/src/test/test-service-stopped2/service_config
@@ -0,0 +1,3 @@
+{
+    "fa:1501": { "node": "node3", "state": "disabled" }
+}
diff --git a/src/test/test-service-stopped3/README b/src/test/test-service-stopped3/README
new file mode 100644
index 0000000..bb8e490
--- /dev/null
+++ b/src/test/test-service-stopped3/README
@@ -0,0 +1,6 @@
+Test if a simulated template in the stopped state can be relocated to other
+nodes without failure.
+
+The template gets simulated by using the VirtFail resource, the configuration
+fa:1501 tells the resource to fail to start and to fail to stop, same behavior
+as a template
diff --git a/src/test/test-service-stopped3/cmdlist b/src/test/test-service-stopped3/cmdlist
new file mode 100644
index 0000000..11e35f4
--- /dev/null
+++ b/src/test/test-service-stopped3/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "service fa:1501 migrate node2" ]
+]
diff --git a/src/test/test-service-stopped3/hardware_status b/src/test/test-service-stopped3/hardware_status
new file mode 100644
index 0000000..451beb1
--- /dev/null
+++ b/src/test/test-service-stopped3/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
diff --git a/src/test/test-service-stopped3/log.expect b/src/test/test-service-stopped3/log.expect
new file mode 100644
index 0000000..02ce703
--- /dev/null
+++ b/src/test/test-service-stopped3/log.expect
@@ -0,0 +1,31 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'fa:1501' on node 'node3'
+info     22    node2/crm: status change wait_for_quorum => slave
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     40    node1/crm: service 'fa:1501': state changed from 'request_stop' to 'stopped'
+info    120      cmdlist: execute service fa:1501 migrate node2
+info    120    node1/crm: got crm command: migrate fa:1501 node2
+info    120    node1/crm: service 'fa:1501': state changed from 'stopped' to 'migrate'  (node = node3, target = node2)
+info    125    node3/lrm: service fa:1501 - start migrate to node 'node2'
+info    125    node3/lrm: service fa:1501 - end migrate to node 'node2'
+info    140    node1/crm: service 'fa:1501': state changed from 'migrate' to 'request_stop'  (node = node2)
+info    143    node2/lrm: got lock 'ha_agent_node2_lock'
+info    143    node2/lrm: status change wait_for_agent_lock => active
+info    160    node1/crm: service 'fa:1501': state changed from 'request_stop' to 'stopped'
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-service-stopped3/manager_status b/src/test/test-service-stopped3/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-service-stopped3/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-service-stopped3/service_config b/src/test/test-service-stopped3/service_config
new file mode 100644
index 0000000..8ae9079
--- /dev/null
+++ b/src/test/test-service-stopped3/service_config
@@ -0,0 +1,3 @@
+{
+    "fa:1501": { "node": "node3", "state": "stopped" }
+}
-- 
2.1.4





More information about the pve-devel mailing list