[Pacemaker] Recovery after lost quorum

Denis Witt denis.witt at concepts-and-training.de
Wed Jun 5 01:55:20 UTC 2013


Am 05.06.2013 um 03:34 schrieb Andrew Beekhof <andrew at beekhof.net>:

>> But no resources are started, so I suspect there really is quorum.
> 
> Can you send me the output of cibadmin -Ql please?
> Perhaps those two resources are blocked for other reasons.

Hi Andrew,

here we go:

<cib epoch="66" num_updates="28" admin_epoch="0" validate-with="pacemaker-1.2" crm_feature_set="3.0.6" update-origin="test4" update-client="crmd" cib-last-written="Wed Jun  5 03:44:38 2013" have-quorum="1" dc-uuid="test4">
  <configuration>
    <crm_config>
      <cluster_property_set id="cib-bootstrap-options">
        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.7-ee0730e13d124c3d58f00016c3376a1de5323cff"/>
        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="openais"/>
        <nvpair id="cib-bootstrap-options-expected-quorum-votes" name="expected-quorum-votes" value="3"/>
        <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="stop"/>
        <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
        <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1370396678"/>
        <nvpair id="cib-bootstrap-options-default-resource-stickiness" name="default-resource-stickiness" value="100"/>
        <nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="false"/>
      </cluster_property_set>
    </crm_config>
    <nodes>
      <node id="test3" type="normal" uname="test3"/>
      <node id="test4" type="normal" uname="test4"/>
      <node id="backup3" type="normal" uname="backup3">
        <instance_attributes id="nodes-backup3">
          <nvpair id="nodes-backup3-standby" name="standby" value="on"/>
        </instance_attributes>
      </node>
    </nodes>
    <resources>
      <clone id="clone_pingtest">
        <primitive class="ocf" id="pingtest" provider="pacemaker" type="ping">
          <instance_attributes id="pingtest-instance_attributes">
            <nvpair id="pingtest-instance_attributes-multiplier" name="multiplier" value="1000"/>
            <nvpair id="pingtest-instance_attributes-host_list" name="host_list" value="192.168.100.19"/>
          </instance_attributes>
          <operations>
            <op id="pingtest-monitor-5" interval="5" name="monitor"/>
          </operations>
        </primitive>
      </clone>
      <master id="ms_drbd">
        <meta_attributes id="ms_drbd-meta_attributes">
          <nvpair id="ms_drbd-meta_attributes-master-max" name="master-max" value="1"/>
          <nvpair id="ms_drbd-meta_attributes-master-node-max" name="master-node-max" value="1"/>
          <nvpair id="ms_drbd-meta_attributes-clone-max" name="clone-max" value="2"/>
          <nvpair id="ms_drbd-meta_attributes-clone-node-max" name="clone-node-max" value="1"/>
          <nvpair id="ms_drbd-meta_attributes-notify" name="notify" value="true"/>
        </meta_attributes>
        <primitive class="ocf" id="drbd" provider="linbit" type="drbd">
          <instance_attributes id="drbd-instance_attributes">
            <nvpair id="drbd-instance_attributes-drbd_resource" name="drbd_resource" value="www_r0"/>
          </instance_attributes>
          <operations>
            <op id="drbd-monitor-10" interval="10" name="monitor"/>
          </operations>
        </primitive>
      </master>
      <group id="grp_all">
        <primitive class="ocf" id="sip" provider="heartbeat" type="IPaddr2">
          <instance_attributes id="sip-instance_attributes">
            <nvpair id="sip-instance_attributes-ip" name="ip" value="192.168.100.30"/>
            <nvpair id="sip-instance_attributes-nic" name="nic" value="eth0"/>
          </instance_attributes>
          <operations>
            <op id="sip-monitor-10" interval="10" name="monitor" timeout="20"/>
          </operations>
          <meta_attributes id="sip-meta_attributes">
            <nvpair id="sip-meta_attributes-target-role" name="target-role" value="Started"/>
          </meta_attributes>
        </primitive>
        <primitive class="ocf" id="fs_drbd" provider="heartbeat" type="Filesystem">
          <instance_attributes id="fs_drbd-instance_attributes">
            <nvpair id="fs_drbd-instance_attributes-device" name="device" value="/dev/drbd0"/>
            <nvpair id="fs_drbd-instance_attributes-directory" name="directory" value="/var/www"/>
            <nvpair id="fs_drbd-instance_attributes-fstype" name="fstype" value="ext4"/>
          </instance_attributes>
          <operations>
            <op id="fs_drbd-monitor-5" interval="5" name="monitor"/>
          </operations>
          <meta_attributes id="fs_drbd-meta_attributes">
            <nvpair id="fs_drbd-meta_attributes-target-role" name="target-role" value="Started"/>
          </meta_attributes>
        </primitive>
        <primitive class="lsb" id="apache" type="apache2">
          <operations>
            <op id="apache-monitor-10" interval="10" name="monitor" timeout="20"/>
          </operations>
          <meta_attributes id="apache-meta_attributes">
            <nvpair id="apache-meta_attributes-target-role" name="target-role" value="Started"/>
          </meta_attributes>
        </primitive>
      </group>
    </resources>
    <constraints>
      <rsc_colocation id="coloc_all_on_drbd" rsc="grp_all" score="INFINITY" with-rsc="ms_drbd" with-rsc-role="Master"/>
      <rsc_order first="ms_drbd" first-action="promote" id="order_all_after_drbd" score="INFINITY" then="grp_all" then-action="start"/>
      <rsc_location id="loc_all_on_best_ping" rsc="grp_all">
        <rule boolean-op="or" id="loc_all_on_best_ping-rule" score="-INFINITY">
          <expression attribute="pingd" id="loc_all_on_best_ping-expression" operation="not_defined"/>
          <expression attribute="pingd" id="loc_all_on_best_ping-expression-0" operation="lt" value="1000"/>
        </rule>
      </rsc_location>
    </constraints>
  </configuration>
  <status>
    <node_state id="test4" uname="test4" ha="active" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="post_cache_update" shutdown="0">
      <transient_attributes id="test4">
        <instance_attributes id="status-test4">
          <nvpair id="status-test4-probe_complete" name="probe_complete" value="true"/>
        </instance_attributes>
      </transient_attributes>
      <lrm id="test4">
        <lrm_resources>
          <lrm_resource id="pingtest:0" type="ping" class="ocf" provider="pacemaker">
            <lrm_rsc_op id="pingtest:0_last_0" operation_key="pingtest:0_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="8:15:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;8:15:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="48" rc-code="0" op-status="0" interval="0" op-digest="7d41580263769d86524e63413353cd0b"/>
          </lrm_resource>
          <lrm_resource id="pingtest:1" type="ping" class="ocf" provider="pacemaker">
            <lrm_rsc_op id="pingtest:1_last_0" operation_key="pingtest:1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="8:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;8:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="141" rc-code="0" op-status="0" interval="0" op-digest="7d41580263769d86524e63413353cd0b" last-run="1370396742" last-rc-change="1370396742" exec-time="90" queue-time="0"/>
            <lrm_rsc_op id="pingtest:1_monitor_5000" operation_key="pingtest:1_monitor_5000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="10:46:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;10:46:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="103" rc-code="0" op-status="0" interval="5000" op-digest="ea75753f1e9e0cac00432d83a812f4c8"/>
          </lrm_resource>
          <lrm_resource id="sip" type="IPaddr2" class="ocf" provider="heartbeat">
            <lrm_rsc_op id="sip_last_0" operation_key="sip_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="41:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;41:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="144" rc-code="0" op-status="0" interval="0" op-digest="b009a03f7cd980a54dff5e570c8de74d" last-run="1370396743" last-rc-change="1370396743" exec-time="40" queue-time="0"/>
            <lrm_rsc_op id="sip_monitor_10000" operation_key="sip_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="44:48:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;44:48:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="108" rc-code="0" op-status="0" interval="10000" op-digest="f92644ce891641f5d8a4019b289dbbd0"/>
          </lrm_resource>
          <lrm_resource id="drbd:0" type="drbd" class="ocf" provider="linbit">
            <lrm_rsc_op id="drbd:0_last_failure_0" operation_key="drbd:0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="8:59:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:8;8:59:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="120" rc-code="8" op-status="0" interval="0" op-digest="f2655e5354fe9acd9270c82254b11693"/>
            <lrm_rsc_op id="drbd:0_last_0" operation_key="drbd:0_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="10:84:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;10:84:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="148" rc-code="0" op-status="0" interval="0" last-run="1370396744" last-rc-change="1370396744" exec-time="110" queue-time="0" op-digest="f2655e5354fe9acd9270c82254b11693"/>
          </lrm_resource>
          <lrm_resource id="fs_drbd" type="Filesystem" class="ocf" provider="heartbeat">
            <lrm_rsc_op id="fs_drbd_last_failure_0" operation_key="fs_drbd_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="7:76:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;7:76:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="132" rc-code="0" op-status="0" interval="0" last-run="1370396658" last-rc-change="1370396658" exec-time="30" queue-time="0" op-digest="e0cde594906f5ecf8fa72d58000e0c45"/>
            <lrm_rsc_op id="fs_drbd_monitor_5000" operation_key="fs_drbd_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="46:77:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;46:77:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="133" rc-code="0" op-status="0" interval="5000" last-rc-change="1370396658" exec-time="30" queue-time="0" op-digest="58701fb56792cdf0fbe729ee4f043a87"/>
            <lrm_rsc_op id="fs_drbd_last_0" operation_key="fs_drbd_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="43:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;43:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="143" rc-code="0" op-status="0" interval="0" last-run="1370396743" last-rc-change="1370396743" exec-time="70" queue-time="0" op-digest="e0cde594906f5ecf8fa72d58000e0c45"/>
          </lrm_resource>
          <lrm_resource id="apache" type="apache2" class="lsb">
            <lrm_rsc_op id="apache_last_failure_0" operation_key="apache_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="7:81:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;7:81:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="138" rc-code="0" op-status="0" interval="0" last-run="1370396677" last-rc-change="1370396677" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
            <lrm_rsc_op id="apache_monitor_10000" operation_key="apache_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="48:82:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;48:82:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="139" rc-code="0" op-status="0" interval="10000" last-rc-change="1370396677" exec-time="20" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
            <lrm_rsc_op id="apache_last_0" operation_key="apache_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="45:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;45:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="140" rc-code="0" op-status="0" interval="0" last-run="1370396742" last-rc-change="1370396742" exec-time="1160" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
          </lrm_resource>
        </lrm_resources>
      </lrm>
    </node_state>
    <node_state id="backup3" uname="backup3" crmd="online" crm-debug-origin="post_cache_update" ha="active" in_ccm="true" join="member" expected="member" shutdown="0"/>
  </status>
</cib>

Quorum was lost, then I restarted corosync on backup3 without pacemaker.

When I enable pacemaker on backup3 the resources are started after a while. Then I got the following:

<cib epoch="66" num_updates="81" admin_epoch="0" validate-with="pacemaker-1.2" crm_feature_set="3.0.6" update-origin="test4" update-client="crmd" cib-last-written="Wed Jun  5 03:44:38 2013" have-quorum="1" dc-uuid="test4">
  <configuration>
    <crm_config>
      <cluster_property_set id="cib-bootstrap-options">
        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.7-ee0730e13d124c3d58f00016c3376a1de5323cff"/>
        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="openais"/>
        <nvpair id="cib-bootstrap-options-expected-quorum-votes" name="expected-quorum-votes" value="3"/>
        <nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="stop"/>
        <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
        <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1370396678"/>
        <nvpair id="cib-bootstrap-options-default-resource-stickiness" name="default-resource-stickiness" value="100"/>
        <nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="false"/>
      </cluster_property_set>
    </crm_config>
    <nodes>
      <node id="test3" type="normal" uname="test3"/>
      <node id="test4" type="normal" uname="test4"/>
      <node id="backup3" type="normal" uname="backup3">
        <instance_attributes id="nodes-backup3">
          <nvpair id="nodes-backup3-standby" name="standby" value="on"/>
        </instance_attributes>
      </node>
    </nodes>
    <resources>
      <clone id="clone_pingtest">
        <primitive class="ocf" id="pingtest" provider="pacemaker" type="ping">
          <instance_attributes id="pingtest-instance_attributes">
            <nvpair id="pingtest-instance_attributes-multiplier" name="multiplier" value="1000"/>
            <nvpair id="pingtest-instance_attributes-host_list" name="host_list" value="192.168.100.19"/>
          </instance_attributes>
          <operations>
            <op id="pingtest-monitor-5" interval="5" name="monitor"/>
          </operations>
        </primitive>
      </clone>
      <master id="ms_drbd">
        <meta_attributes id="ms_drbd-meta_attributes">
          <nvpair id="ms_drbd-meta_attributes-master-max" name="master-max" value="1"/>
          <nvpair id="ms_drbd-meta_attributes-master-node-max" name="master-node-max" value="1"/>
          <nvpair id="ms_drbd-meta_attributes-clone-max" name="clone-max" value="2"/>
          <nvpair id="ms_drbd-meta_attributes-clone-node-max" name="clone-node-max" value="1"/>
          <nvpair id="ms_drbd-meta_attributes-notify" name="notify" value="true"/>
        </meta_attributes>
        <primitive class="ocf" id="drbd" provider="linbit" type="drbd">
          <instance_attributes id="drbd-instance_attributes">
            <nvpair id="drbd-instance_attributes-drbd_resource" name="drbd_resource" value="www_r0"/>
          </instance_attributes>
          <operations>
            <op id="drbd-monitor-10" interval="10" name="monitor"/>
          </operations>
        </primitive>
      </master>
      <group id="grp_all">
        <primitive class="ocf" id="sip" provider="heartbeat" type="IPaddr2">
          <instance_attributes id="sip-instance_attributes">
            <nvpair id="sip-instance_attributes-ip" name="ip" value="192.168.100.30"/>
            <nvpair id="sip-instance_attributes-nic" name="nic" value="eth0"/>
          </instance_attributes>
          <operations>
            <op id="sip-monitor-10" interval="10" name="monitor" timeout="20"/>
          </operations>
          <meta_attributes id="sip-meta_attributes">
            <nvpair id="sip-meta_attributes-target-role" name="target-role" value="Started"/>
          </meta_attributes>
        </primitive>
        <primitive class="ocf" id="fs_drbd" provider="heartbeat" type="Filesystem">
          <instance_attributes id="fs_drbd-instance_attributes">
            <nvpair id="fs_drbd-instance_attributes-device" name="device" value="/dev/drbd0"/>
            <nvpair id="fs_drbd-instance_attributes-directory" name="directory" value="/var/www"/>
            <nvpair id="fs_drbd-instance_attributes-fstype" name="fstype" value="ext4"/>
          </instance_attributes>
          <operations>
            <op id="fs_drbd-monitor-5" interval="5" name="monitor"/>
          </operations>
          <meta_attributes id="fs_drbd-meta_attributes">
            <nvpair id="fs_drbd-meta_attributes-target-role" name="target-role" value="Started"/>
          </meta_attributes>
        </primitive>
        <primitive class="lsb" id="apache" type="apache2">
          <operations>
            <op id="apache-monitor-10" interval="10" name="monitor" timeout="20"/>
          </operations>
          <meta_attributes id="apache-meta_attributes">
            <nvpair id="apache-meta_attributes-target-role" name="target-role" value="Started"/>
          </meta_attributes>
        </primitive>
      </group>
    </resources>
    <constraints>
      <rsc_colocation id="coloc_all_on_drbd" rsc="grp_all" score="INFINITY" with-rsc="ms_drbd" with-rsc-role="Master"/>
      <rsc_order first="ms_drbd" first-action="promote" id="order_all_after_drbd" score="INFINITY" then="grp_all" then-action="start"/>
      <rsc_location id="loc_all_on_best_ping" rsc="grp_all">
        <rule boolean-op="or" id="loc_all_on_best_ping-rule" score="-INFINITY">
          <expression attribute="pingd" id="loc_all_on_best_ping-expression" operation="not_defined"/>
          <expression attribute="pingd" id="loc_all_on_best_ping-expression-0" operation="lt" value="1000"/>
        </rule>
      </rsc_location>
    </constraints>
  </configuration>
  <status>
    <node_state id="test4" uname="test4" ha="active" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="do_update_resource" shutdown="0">
      <transient_attributes id="test4">
        <instance_attributes id="status-test4">
          <nvpair id="status-test4-probe_complete" name="probe_complete" value="true"/>
          <nvpair id="status-test4-master-drbd.0" name="master-drbd:0" value="10000"/>
          <nvpair id="status-test4-pingd" name="pingd" value="1000"/>
        </instance_attributes>
      </transient_attributes>
      <lrm id="test4">
        <lrm_resources>
          <lrm_resource id="pingtest:0" type="ping" class="ocf" provider="pacemaker">
            <lrm_rsc_op id="pingtest:0_last_0" operation_key="pingtest:0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="5:90:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;5:90:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="149" rc-code="0" op-status="0" interval="0" op-digest="7d41580263769d86524e63413353cd0b" last-run="1370396943" last-rc-change="1370396943" exec-time="2080" queue-time="0"/>
            <lrm_rsc_op id="pingtest:0_monitor_5000" operation_key="pingtest:0_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="7:91:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;7:91:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="152" rc-code="0" op-status="0" interval="5000" last-rc-change="1370396945" exec-time="2060" queue-time="0" op-digest="ea75753f1e9e0cac00432d83a812f4c8"/>
          </lrm_resource>
          <lrm_resource id="pingtest:1" type="ping" class="ocf" provider="pacemaker">
            <lrm_rsc_op id="pingtest:1_last_0" operation_key="pingtest:1_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="8:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;8:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="141" rc-code="0" op-status="0" interval="0" op-digest="7d41580263769d86524e63413353cd0b"/>
          </lrm_resource>
          <lrm_resource id="fs_drbd" type="Filesystem" class="ocf" provider="heartbeat">
            <lrm_rsc_op id="fs_drbd_last_0" operation_key="fs_drbd_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="42:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;42:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="158" rc-code="0" op-status="0" interval="0" op-digest="e0cde594906f5ecf8fa72d58000e0c45" last-run="1370396950" last-rc-change="1370396950" exec-time="110" queue-time="0"/>
            <lrm_rsc_op id="fs_drbd_last_failure_0" operation_key="fs_drbd_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="7:76:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;7:76:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="132" rc-code="0" op-status="0" interval="0" op-digest="e0cde594906f5ecf8fa72d58000e0c45"/>
            <lrm_rsc_op id="fs_drbd_monitor_5000" operation_key="fs_drbd_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="43:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;43:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="159" rc-code="0" op-status="0" interval="5000" last-rc-change="1370396950" exec-time="100" queue-time="0" op-digest="58701fb56792cdf0fbe729ee4f043a87"/>
          </lrm_resource>
          <lrm_resource id="apache" type="apache2" class="lsb">
            <lrm_rsc_op id="apache_last_0" operation_key="apache_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="44:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;44:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="160" rc-code="0" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" last-run="1370396950" last-rc-change="1370396950" exec-time="50" queue-time="0"/>
            <lrm_rsc_op id="apache_last_failure_0" operation_key="apache_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="7:81:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;7:81:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="138" rc-code="0" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
            <lrm_rsc_op id="apache_monitor_10000" operation_key="apache_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="45:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;45:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="161" rc-code="0" op-status="0" interval="10000" last-rc-change="1370396950" exec-time="30" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
          </lrm_resource>
          <lrm_resource id="sip" type="IPaddr2" class="ocf" provider="heartbeat">
            <lrm_rsc_op id="sip_last_0" operation_key="sip_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="40:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;40:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="156" rc-code="0" op-status="0" interval="0" op-digest="b009a03f7cd980a54dff5e570c8de74d" last-run="1370396950" last-rc-change="1370396950" exec-time="120" queue-time="0"/>
            <lrm_rsc_op id="sip_monitor_10000" operation_key="sip_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="41:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;41:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="157" rc-code="0" op-status="0" interval="10000" last-rc-change="1370396950" exec-time="60" queue-time="0" op-digest="f92644ce891641f5d8a4019b289dbbd0"/>
          </lrm_resource>
          <lrm_resource id="drbd:0" type="drbd" class="ocf" provider="linbit">
            <lrm_rsc_op id="drbd:0_last_0" operation_key="drbd:0_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="15:91:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;15:91:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="154" rc-code="0" op-status="0" interval="0" op-digest="f2655e5354fe9acd9270c82254b11693" last-run="1370396945" last-rc-change="1370396945" exec-time="70" queue-time="0"/>
            <lrm_rsc_op id="drbd:0_last_failure_0" operation_key="drbd:0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="8:59:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:8;8:59:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="120" rc-code="8" op-status="0" interval="0" op-digest="f2655e5354fe9acd9270c82254b11693"/>
          </lrm_resource>
        </lrm_resources>
      </lrm>
    </node_state>
    <node_state id="backup3" uname="backup3" crmd="online" crm-debug-origin="do_update_resource" ha="active" in_ccm="true" join="member" expected="member" shutdown="0">
      <lrm id="backup3">
        <lrm_resources>
          <lrm_resource id="drbd:0" type="drbd" class="ocf" provider="linbit">
            <lrm_rsc_op id="drbd:0_last_failure_0" operation_key="drbd:0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="6:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:5;6:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="3" rc-code="5" op-status="0" interval="0" last-run="1370396962" last-rc-change="1370396962" exec-time="20" queue-time="0" op-digest="f2655e5354fe9acd9270c82254b11693"/>
          </lrm_resource>
          <lrm_resource id="pingtest:0" type="ping" class="ocf" provider="pacemaker">
            <lrm_rsc_op id="pingtest:0_last_0" operation_key="pingtest:0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="5:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:7;5:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1370396962" last-rc-change="1370396962" exec-time="60" queue-time="0" op-digest="7d41580263769d86524e63413353cd0b"/>
          </lrm_resource>
          <lrm_resource id="fs_drbd" type="Filesystem" class="ocf" provider="heartbeat">
            <lrm_rsc_op id="fs_drbd_last_failure_0" operation_key="fs_drbd_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="8:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:5;8:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="5" rc-code="5" op-status="0" interval="0" last-run="1370396962" last-rc-change="1370396962" exec-time="120" queue-time="0" op-digest="e0cde594906f5ecf8fa72d58000e0c45"/>
          </lrm_resource>
          <lrm_resource id="sip" type="IPaddr2" class="ocf" provider="heartbeat">
            <lrm_rsc_op id="sip_last_0" operation_key="sip_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="7:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:7;7:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1370396962" last-rc-change="1370396962" exec-time="140" queue-time="0" op-digest="b009a03f7cd980a54dff5e570c8de74d"/>
          </lrm_resource>
          <lrm_resource id="apache" type="apache2" class="lsb">
            <lrm_rsc_op id="apache_last_failure_0" operation_key="apache_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="9:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:5;9:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="6" rc-code="5" op-status="0" interval="0" last-run="1370396964" last-rc-change="1370396964" exec-time="0" queue-time="1010" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
          </lrm_resource>
        </lrm_resources>
      </lrm>
      <transient_attributes id="backup3">
        <instance_attributes id="status-backup3">
          <nvpair id="status-backup3-probe_complete" name="probe_complete" value="true"/>
        </instance_attributes>
      </transient_attributes>
    </node_state>
  </status>
</cib>

The differences (without last-rc-change):

root at test4:~# diff cibadmin1 cibadmin2 | grep -v last-rc-change
1c1
< <cib epoch="66" num_updates="28" admin_epoch="0" validate-with="pacemaker-1.2" crm_feature_set="3.0.6" update-origin="test4" update-client="crmd" cib-last-written="Wed Jun  5 03:44:38 2013" have-quorum="1" dc-uuid="test4">
---
> <cib epoch="66" num_updates="81" admin_epoch="0" validate-with="pacemaker-1.2" crm_feature_set="3.0.6" update-origin="test4" update-client="crmd" cib-last-written="Wed Jun  5 03:44:38 2013" have-quorum="1" dc-uuid="test4">
101c101
<     <node_state id="test4" uname="test4" ha="active" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="post_cache_update" shutdown="0">
---
>     <node_state id="test4" uname="test4" ha="active" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="do_update_resource" shutdown="0">
104a105,106
>           <nvpair id="status-test4-master-drbd.0" name="master-drbd:0" value="10000"/>
>           <nvpair id="status-test4-pingd" name="pingd" value="1000"/>
110c112,113
<             <lrm_rsc_op id="pingtest:0_last_0" operation_key="pingtest:0_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="8:15:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;8:15:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="48" rc-code="0" op-status="0" interval="0" op-digest="7d41580263769d86524e63413353cd0b"/>
---
113,114c116,126
<             <lrm_rsc_op id="pingtest:1_monitor_5000" operation_key="pingtest:1_monitor_5000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="10:46:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;10:46:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="103" rc-code="0" op-status="0" interval="5000" op-digest="ea75753f1e9e0cac00432d83a812f4c8"/>
---
>             <lrm_rsc_op id="pingtest:1_last_0" operation_key="pingtest:1_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="8:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;8:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="141" rc-code="0" op-status="0" interval="0" op-digest="7d41580263769d86524e63413353cd0b"/>
>           </lrm_resource>
>           <lrm_resource id="fs_drbd" type="Filesystem" class="ocf" provider="heartbeat">
>             <lrm_rsc_op id="fs_drbd_last_failure_0" operation_key="fs_drbd_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="7:76:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;7:76:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="132" rc-code="0" op-status="0" interval="0" op-digest="e0cde594906f5ecf8fa72d58000e0c45"/>
>           </lrm_resource>
>           <lrm_resource id="apache" type="apache2" class="lsb">
>             <lrm_rsc_op id="apache_last_failure_0" operation_key="apache_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="7:81:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;7:81:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="138" rc-code="0" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
117,118c129,130
<             <lrm_rsc_op id="sip_monitor_10000" operation_key="sip_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="44:48:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;44:48:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="108" rc-code="0" op-status="0" interval="10000" op-digest="f92644ce891641f5d8a4019b289dbbd0"/>
---
120a133
122c135,146
---
>           </lrm_resource>
>         </lrm_resources>
>       </lrm>
>     </node_state>
>     <node_state id="backup3" uname="backup3" crmd="online" crm-debug-origin="do_update_resource" ha="active" in_ccm="true" join="member" expected="member" shutdown="0">
>       <lrm id="backup3">
>         <lrm_resources>
>           <lrm_resource id="drbd:0" type="drbd" class="ocf" provider="linbit">
>           </lrm_resource>
>           <lrm_resource id="pingtest:0" type="ping" class="ocf" provider="pacemaker">
125,127c149,152
---
>           </lrm_resource>
>           <lrm_resource id="sip" type="IPaddr2" class="ocf" provider="heartbeat">
130,132c155
---
135a159,163
>       <transient_attributes id="backup3">
>         <instance_attributes id="status-backup3">
>           <nvpair id="status-backup3-probe_complete" name="probe_complete" value="true"/>
>         </instance_attributes>
>       </transient_attributes>
137d164
<     <node_state id="backup3" uname="backup3" crmd="online" crm-debug-origin="post_cache_update" ha="active" in_ccm="true" join="member" expected="member" shutdown="0"/>

Best regards
Denis Witt



More information about the Pacemaker mailing list