HACMP Procedures

The following provides an example script for configuring HACMP in a two-node mutual failover cluster with multiple resource groups and multiple volume groups.

This script assumes the appropriate entries and modifications have been made to the following files:


#!/usr/bin/ksh93
#################################################################
NODE1="mx1apora03"
NODE2="mx1apora04"

RG1="etapdtu0"
RG1APPLS="${RG1}_oradb01 ${RG1}_oradb02"
RG1VGS="${RG1}01vg ${RG1}02vg"

RG2="gappdtu0"
RG2APPLS="${RG2}_oradb01"
RG2VGS="${RG2}01vg"

RG3="mrkpdtu0"
RG3APPLS="${RG3}_oradb01"
RG3VGS="${RG3}01vg"

RG4="znmpdtu0"
RG4APPLS="${RG4}_oradb01"
RG4VGS="${RG4}01vg"

################################################################
# Remove Existing Cluster: mx1aporatu03
/usr/es/sbin/cluster/utilities/clrmclstr

# Create Cluster
/usr/es/sbin/cluster/utilities/clmodclstr -n ${NODE1%??}tu01 -p "${NODE1}-boot.tu.com ${NODE2}-boot.tu.com"

################################################################
# Create Service Addresses
/usr/es/sbin/cluster/utilities/clrmnode -a ${NODE1}.tu.com
/usr/es/sbin/cluster/utilities/clrmnode -a ${NODE2}.tu.com
/usr/es/sbin/cluster/utilities/claddnode -T service -B ${NODE1}.tu.com -w net_ether_01
/usr/es/sbin/cluster/utilities/claddnode -T service -B ${NODE2}.tu.com -w net_ether_01

################################################################
# Create Application Servers
for APPLS in ${RG1APPLS} ${RG2APPLS} ${RG3APPLS} ${RG4APPLS}
do
/usr/es/sbin/cluster/utilities/clrmserv ${APPLS}
/usr/es/sbin/cluster/utilities/claddserv -s ${APPLS} -b "/usr/local/hascripts/${APPLS}_start.sh" -e "/usr/local/hascripts/${APPLS}_stop.sh"
done

################################################################
# Create persistant addresses
/usr/es/sbin/cluster/utilities/clrmnode -a ${NODE1}-pers
/usr/es/sbin/cluster/utilities/clrmnode -a ${NODE2}-pers

/usr/es/sbin/cluster/utilities/claddnode -a ${NODE1}-pers.tu.com :ether :net_ether_01 : :persistent : : -n ${NODE1}

/usr/es/sbin/cluster/utilities/claddnode -a ${NODE2}-pers.tu.com :ether :net_ether_01 : :persistent : : -n ${NODE2}

################################################################
# Create resource groups
/usr/es/sbin/cluster/utilities/clrmgrp -g ${NODE1}
/usr/es/sbin/cluster/utilities/claddgrp -g ${NODE1} -n "${NODE1} ${NODE2}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${NODE2}
/usr/es/sbin/cluster/utilities/claddgrp -g ${NODE2} -n "${NODE2} ${NODE1}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${RG1}
/usr/es/sbin/cluster/utilities/claddgrp -g ${RG1} -n "${NODE1} ${NODE2}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${RG2}
/usr/es/sbin/cluster/utilities/claddgrp -g ${RG2} -n "${NODE2} ${NODE1}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${RG3}
/usr/es/sbin/cluster/utilities/claddgrp -g ${RG3} -n "${NODE2} ${NODE1}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${RG4}
/usr/es/sbin/cluster/utilities/claddgrp -g ${RG4} -n "${NODE2} ${NODE1}" -S OHN -O FNPN -B FBHPN

################################################################
# Add the Service address to the resource group
/usr/es/sbin/cluster/utilities/claddres -g "${NODE1}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE1}" \
  APPLICATIONS= \
  VOLUME_GROUP= \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

/usr/es/sbin/cluster/utilities/claddres -g "${NODE2}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE2}" \
  APPLICATIONS= \
  VOLUME_GROUP= \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

################################################################
# Add application servers and volumes to resource groups
/usr/es/sbin/cluster/utilities/claddres -g "${RG1}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE1}-rg01" \
  APPLICATIONS="${RG1APPLS}"  \
  VOLUME_GROUP="${RG1VGS}" \
  FORCED_VARYON="false"  \
  VG_AUTO_IMPORT="false"  \
  FILESYSTEM=  \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential"  \
  FS_BEFORE_IPADDR="false"  \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM=  \
  NFS_NETWORK=  \
  SHARED_TAPE_RESOURCES=  \
  DISK= \
  AIX_FAST_CONNECT_SERVICES=  \
  COMMUNICATION_LINKS=  \
  MISC_DATA=

/usr/es/sbin/cluster/utilities/claddres -g "${RG2}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE1}-rg02" \
  APPLICATIONS="${RG2APPLS}" \
  VOLUME_GROUP="${RG2VGS}" \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

/usr/es/sbin/cluster/utilities/claddres -g "${RG3}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE2}-rg03" \
  APPLICATIONS="${RG3APPLS}" \
  VOLUME_GROUP="${RG3VGS}" \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

/usr/es/sbin/cluster/utilities/claddres -g "${RG4}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE2}-rg04" \
  APPLICATIONS="${RG4APPLS}" \
  VOLUME_GROUP="${RG4VGS}" \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

The Automated HACMP configuration is almost complete, at this point manually configure the concurrent disk heartbeat.


Manual HACMP configuration

EXAMPLE: MX1 EDNA HACMP 590 Configuration

  1. Verify Appropriate Network Configuration :
  2. Remove Old HACMP configurations if they exist :

        smitty hacmp -->
            Extended Configuration -->
                Extended Topology Configuration -->
                     Configure an HACMP Cluster -->
                        Remove an HACMP Cluster
    

  3. Configure Various HACMP Required Files :

    /etc/hosts  :
    ## HACMP ip adress labels ##
    146.61.171.68   mx1apedn07.tu.com       mx1apedn07
    146.61.171.69   mx1apedn07-pers.tu.com  mx1apedn07-pers
    192.168.171.68  mx1apedn07-boot.tu.com  mx1apedn07-boot
    146.61.172.48   mx1apedn07-man.tu.com   mx1apedn07-man
    146.61.171.70   mx1apedn08.tu.com       mx1apedn08
    146.61.171.71   mx1apedn08-pers.tu.com  mx1apedn08-pers
    192.168.171.70  mx1apedn08-boot.tu.com  mx1apedn08-boot
    146.61.172.49   mx1apedn08-man.tu.com   mx1apedn08-man
    

    /.rhosts :
    mx0apnim01              root
    mx0apnim01.tu.com       root
    mx1apnim01.tu.com        root
    mx1apnim01              root
    46.61.171.68           root
    mx1apedn07.tu.com        root
    mx1apedn07        root
    146.61.171.69       root
    mx1apedn07-pers.tu.com  root
    mx1apedn07-pers        root
    192.168.171.68      root
    mx1apedn07-boot.tu.com  root
    mx1apedn07-boot        root
    146.61.172.48       root
    mx1apedn07-man.tu.com   root
    mx1apedn07-man        root
    146.61.171.70       root
    mx1apedn08.tu.com    root
    mx1apedn08        root
    146.61.171.71       root
    mx1apedn08-pers.tu.com  root
    mx1apedn08-pers        root
    192.168.171.70      root
    mx1apedn08-boot.tu.com    root
    mx1apedn08-boot        root
    146.61.172.49       root
    mx1apedn08-man.tu.com    root
    mx1apedn08-man        root
    

    /usr/es/sbin/cluster/etc/rhosts :
    mx1apedn07
    mx1apedn07-boot
    mx1apedn08
    mx1apedn08-boot
    

    • service address and boot address for participating nodes in cluster.

    /usr/es/sbin/cluster/etc/netmon.cf :
    mx1apedn07
    mx1apedn07-boot
    mx1apedn08
    mx1apedn08-boot
    

  4. Test rsh functionality between nodes.
    • Use boot address to rsh to and from clustered pairs.
  5. Add Nodes To Cluster Configuration :

        Initialization and Standard Configuration -->
            Add Nodes to an HACMP Cluster -->
                cluster name "mx1apedntu07"
                New Nodes "mx1apedn07 mx1apedn08"
                    ( Service Names )
    

  6. Add Service Lables :

        Initialization and Standard Configuration -->
            Configure Resources to Make Highly Available -->
                    Configure Service IP Labels/Addresses -->    
                        Add a Service IP Label/Address -->
                            IP Label Address "mx1apedn07.tu.com"
                                (Service hostname)
                            Network Name "net_ether_01"
                                (Boot Label 192.168.xx.xx)
            ***** Perform Twice, Next mx1apedn08.tu.com Service Label.
    

  7. Add Persistence Labels :

        Extended Configuration -->
            Extended Topology Configuration -->
                Configure HACMP Persistent Node IP Label/Addresses -->
                    Add a Persistent Node IP Label/Address -->
                        Select Node "mx1apedn07"
                        Network Name "net_ether_01"
                            ( Persistence Network Label )
                        Node IP Label/Address "mx1apedn07-pers.tu.com"
                            ( Persistence Node Label )
            ***** Perform Twice, Next mx1apedn08.tu.com Persistence Label.
    

  8. Synchronize Cluster configuration :

        Extended Configuration -->
            Extended Verification and Synchronization -->
                Automatically correct errors    found "yes"
                ( Correct Timestamp for HACMP file. )
        ***** Accept other defaults and synchronize cluster.
    

  9. Create Volume Groups and Filesystems On Systems:
  10. Create Resource Groups :

        Extended Configuration -->
            Extended Resource Configuration -->
                HACMP Extended Resource Group Configuration -->
                    Add a Resource Group -->
                    Enter RG Name for Service Labels "mx1apedn07"
                    Participating Nodes "mx1apedn07 mx1apedn08"
                    ( Priority Order of Nodes for the group )
                     *****  Accept defaults for additional fields.
        *****  Perform Twice if Mutual Failover else Once if 1-way.
                                    Enter RG Name for Volume groups "ednpdtu7"
                        Participating Nodes "mx1apedn07 mx1apedn08"
                    ( Priority Order of Nodes for the group )
        *****  Perform Twice if Mutual Failover else Once if 1-way.
    

  11. Add Application Server :

        ***** Perform these steps for application servers only.
          Initialization and Standard Configuration -->
            Configure Resources to Make Highly Available -->
                Configure Application Servers -->
                    Add an Application Server -->
                    Enter APP SERVER RG Name "ednpdtu7_appl01"
                    Enter Start Script "/usr/local/hascripts/ednpdtu7_appl01_start.sh"
                    Enter Stop Script  "/usr/local/hascripts/ednpdtu7_appl01_stop.sh"
                    ( Application Start and Stop Scripts )
    

  12. Configure Attributes for New Resource Groups :

        Extended Configuration -->
            Extended Resource Configuration -->
                Extended Resource Group Configuration -->
                    Change/Show Resources and Attributes for a Resource Group -->
                    Select RG "mx1apedn07"
                    Enter Service IP Labels "mx1apedn07"
                    ( Failover of Serve IP for RG )
        *****  Perform Twice if Mutual Failover else Once if 1-way.
                    Select RG "ednpdtu7"
                    Enter Application Server "ednpdtu7_appl01"
                    Enter Volume Groups "ednpdtu700vg"
                    ( Failover of VG's )
    

  13. Synchronize Cluster configuration :

        Extended Configuration -->
            Extended Verification and Synchronization -->
            ***** Accept defaults and synchronize cluster.
    

  14. Discover HACMP-related Information from Clustered Nodes :

        Discover HACMP-related Information from Configured Nodes
        ( Collect disk info for Disk Heartbeat Configuration etc ........ )
    

  15. Disk Heartbeat Configuration :

        Extended Configuration -->
            Extended Topology Configuration -->
                Configure HACMP Communication Interfaces/Devices -->
                    Add Communication Interfaces/Devices -->
                    Select "Add Discovered Communication Interface and Devices"
                    Select "Communication Devices"
                    Select 1 Disk from each Node /LPAR "mx1apedn07 mx1apedn08"
        *****  Configure Disk Heartbeat for all VG in cluster.
    

  16. Synchronize Cluster configuration :

       Extended Configuration -->
            Extended Verification and Synchronization -->
            ***** Accept defaults and synchronize cluster.