HACMP Procedures

The following provides an example script for configuring HACMP in a two-node mutual failover cluster with multiple resource groups and multiple volume groups.

This script assumes the appropriate entries and modifications have been made to the following files:

• /etc/hosts
• /.rhosts
• /usr/es/sbin/cluster/etc/rhosts
• /usr/es/sbin/cluster/netmon.cf

#!/usr/bin/ksh93
#################################################################
NODE1="ddcapora03"
NODE2="ddcapora04"

RG1="etapdtu0"
RG1APPLS="${RG1}_oradb01 ${RG1}_oradb02"
RG1VGS="${RG1}01vg ${RG1}02vg"

RG2="gappdtu0"
RG2APPLS="${RG2}_oradb01"
RG2VGS="${RG2}01vg"

RG3="mrkpdtu0"
RG3APPLS="${RG3}_oradb01"
RG3VGS="${RG3}01vg"

RG4="znmpdtu0"
RG4APPLS="${RG4}_oradb01"
RG4VGS="${RG4}01vg"

################################################################
# Remove Existing Cluster: ddcaporatu03
/usr/es/sbin/cluster/utilities/clrmclstr

# Create Cluster
/usr/es/sbin/cluster/utilities/clmodclstr -n ${NODE1%??}tu01 -p "${NODE1}-boot.tu.com ${NODE2}-boot.tu.com"

################################################################
# Create Service Addresses
/usr/es/sbin/cluster/utilities/clrmnode -a ${NODE1}.tu.com
/usr/es/sbin/cluster/utilities/clrmnode -a ${NODE2}.tu.com
/usr/es/sbin/cluster/utilities/claddnode -T service -B ${NODE1}.tu.com -w net_ether_01
/usr/es/sbin/cluster/utilities/claddnode -T service -B ${NODE2}.tu.com -w net_ether_01

################################################################
# Create Application Servers
for APPLS in ${RG1APPLS} ${RG2APPLS} ${RG3APPLS} ${RG4APPLS}
do
/usr/es/sbin/cluster/utilities/clrmserv ${APPLS}
/usr/es/sbin/cluster/utilities/claddserv -s ${APPLS} -b "/usr/local/hascripts/${APPLS}_start.sh" -e "/usr/local/hascripts/${APPLS}_stop.sh"
done

################################################################
# Create persistant addresses
/usr/es/sbin/cluster/utilities/clrmnode -a ${NODE1}-pers
/usr/es/sbin/cluster/utilities/clrmnode -a ${NODE2}-pers

/usr/es/sbin/cluster/utilities/claddnode -a ${NODE1}-pers.tu.com :ether :net_ether_01 : :persistent : : -n ${NODE1}

/usr/es/sbin/cluster/utilities/claddnode -a ${NODE2}-pers.tu.com :ether :net_ether_01 : :persistent : : -n ${NODE2}

################################################################
# Create resource groups
/usr/es/sbin/cluster/utilities/clrmgrp -g ${NODE1}
/usr/es/sbin/cluster/utilities/claddgrp -g ${NODE1} -n "${NODE1} ${NODE2}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${NODE2}
/usr/es/sbin/cluster/utilities/claddgrp -g ${NODE2} -n "${NODE2} ${NODE1}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${RG1}
/usr/es/sbin/cluster/utilities/claddgrp -g ${RG1} -n "${NODE1} ${NODE2}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${RG2}
/usr/es/sbin/cluster/utilities/claddgrp -g ${RG2} -n "${NODE2} ${NODE1}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${RG3}
/usr/es/sbin/cluster/utilities/claddgrp -g ${RG3} -n "${NODE2} ${NODE1}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${RG4}
/usr/es/sbin/cluster/utilities/claddgrp -g ${RG4} -n "${NODE2} ${NODE1}" -S OHN -O FNPN -B FBHPN

################################################################
# Add the Service address to the resource group
/usr/es/sbin/cluster/utilities/claddres -g "${NODE1}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE1}" \
  APPLICATIONS= \
  VOLUME_GROUP= \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

/usr/es/sbin/cluster/utilities/claddres -g "${NODE2}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE2}" \
  APPLICATIONS= \
  VOLUME_GROUP= \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

################################################################
# Add application servers and volumes to resource groups
/usr/es/sbin/cluster/utilities/claddres -g "${RG1}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE1}-rg01" \
  APPLICATIONS="${RG1APPLS}"  \
  VOLUME_GROUP="${RG1VGS}" \
  FORCED_VARYON="false"  \
  VG_AUTO_IMPORT="false"  \
  FILESYSTEM=  \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential"  \
  FS_BEFORE_IPADDR="false"  \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM=  \
  NFS_NETWORK=  \
  SHARED_TAPE_RESOURCES=  \
  DISK= \
  AIX_FAST_CONNECT_SERVICES=  \
  COMMUNICATION_LINKS=  \
  MISC_DATA=

/usr/es/sbin/cluster/utilities/claddres -g "${RG2}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE1}-rg02" \
  APPLICATIONS="${RG2APPLS}" \
  VOLUME_GROUP="${RG2VGS}" \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

/usr/es/sbin/cluster/utilities/claddres -g "${RG3}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE2}-rg03" \
  APPLICATIONS="${RG3APPLS}" \
  VOLUME_GROUP="${RG3VGS}" \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

/usr/es/sbin/cluster/utilities/claddres -g "${RG4}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE2}-rg04" \
  APPLICATIONS="${RG4APPLS}" \
  VOLUME_GROUP="${RG4VGS}" \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

The Automated HACMP configuration is almost complete, at this point manually configure the concurrent disk heartbeat.

Manual HACMP configuration

EXAMPLE: DDC EDNA HACMP 590 Configuration

1. Verify Appropriate Network Configuration :
   • To Add the service address to boot interface initially before HACMP
   • configuration is completed run the following command.

     ifconfig en4 146.61.171.60 netmask 255.255.255.0 alias
     

   • Add Default Route for system.

     mkdev -l inet0
     mkdev -l en4
     

   • After HACMP Configuration completed verify network configuration.

     netstat -in
     Name  Mtu   Network     Address
     en4   1500  link#2      12.f3.f0.0.61.f4
     en4   1500  192.168.171 192.168.171.60   ** Boot address
     en4   1500  146.61.171  146.61.171.60    ** Service Address
     en4   1500  146.61.171  146.61.171.61    ** Persistence Address
     en5   1500  link#3      12.f3.f0.0.63.52
     en5   1500  146.61.172  146.61.172.44    ** Man Address
     lo0   16896 link#1
     lo0   16896 127         127.0.0.1
     lo0   16896 ::1
           

2. Remove Old HACMP configurations if they exist :

       smitty hacmp -->
           Extended Configuration -->
               Extended Topology Configuration -->
                    Configure an HACMP Cluster -->
                       Remove an HACMP Cluster
   

3. Configure Various HACMP Required Files :

   /etc/hosts  :
   ## HACMP ip adress labels ##
   146.61.171.68   ddcapedn07.tu.com       ddcapedn07
   146.61.171.69   ddcapedn07-pers.tu.com  ddcapedn07-pers
   192.168.171.68  ddcapedn07-boot.tu.com  ddcapedn07-boot
   146.61.172.48   ddcapedn07-man.tu.com   ddcapedn07-man
   146.61.171.70   ddcapedn08.tu.com       ddcapedn08
   146.61.171.71   ddcapedn08-pers.tu.com  ddcapedn08-pers
   192.168.171.70  ddcapedn08-boot.tu.com  ddcapedn08-boot
   146.61.172.49   ddcapedn08-man.tu.com   ddcapedn08-man
   

   /.rhosts :
   mdcapnim01              root
   mdcapnim01.tu.com       root
   ddcapnim01.tu.com        root
   ddcapnim01              root
   46.61.171.68           root
   ddcapedn07.tu.com        root
   ddcapedn07        root
   146.61.171.69       root
   ddcapedn07-pers.tu.com  root
   ddcapedn07-pers        root
   192.168.171.68      root
   ddcapedn07-boot.tu.com  root
   ddcapedn07-boot        root
   146.61.172.48       root
   ddcapedn07-man.tu.com   root
   ddcapedn07-man        root
   146.61.171.70       root
   ddcapedn08.tu.com    root
   ddcapedn08        root
   146.61.171.71       root
   ddcapedn08-pers.tu.com  root
   ddcapedn08-pers        root
   192.168.171.70      root
   ddcapedn08-boot.tu.com    root
   ddcapedn08-boot        root
   146.61.172.49       root
   ddcapedn08-man.tu.com    root
   ddcapedn08-man        root
   

   /usr/es/sbin/cluster/etc/rhosts :
   ddcapedn07
   ddcapedn07-boot
   ddcapedn08
   ddcapedn08-boot
   

   • service address and boot address for participating nodes in cluster.

   /usr/es/sbin/cluster/etc/netmon.cf :
   ddcapedn07
   ddcapedn07-boot
   ddcapedn08
   ddcapedn08-boot
   

4. Test rsh functionality between nodes.
   • Use boot address to rsh to and from clustered pairs.
5. Add Nodes To Cluster Configuration :

       Initialization and Standard Configuration -->
           Add Nodes to an HACMP Cluster -->
               cluster name "ddcapedntu07"
               New Nodes "ddcapedn07 ddcapedn08"
                   ( Service Names )
   

6. Add Service Lables :

       Initialization and Standard Configuration -->
           Configure Resources to Make Highly Available -->
                   Configure Service IP Labels/Addresses -->    
                       Add a Service IP Label/Address -->
                           IP Label Address "ddcapedn07.tu.com"
                               (Service hostname)
                           Network Name "net_ether_01"
                               (Boot Label 192.168.xx.xx)
           ***** Perform Twice, Next ddcapedn08.tu.com Service Label.
   

7. Add Persistence Labels :

       Extended Configuration -->
           Extended Topology Configuration -->
               Configure HACMP Persistent Node IP Label/Addresses -->
                   Add a Persistent Node IP Label/Address -->
                       Select Node "ddcapedn07"
                       Network Name "net_ether_01"
                           ( Persistence Network Label )
                       Node IP Label/Address "ddcapedn07-pers.tu.com"
                           ( Persistence Node Label )
           ***** Perform Twice, Next ddcapedn08.tu.com Persistence Label.
   

8. Synchronize Cluster configuration :

       Extended Configuration -->
           Extended Verification and Synchronization -->
               Automatically correct errors    found "yes"
               ( Correct Timestamp for HACMP file. )
       ***** Accept other defaults and synchronize cluster.
   

9. Create Volume Groups and Filesystems On Systems:
10. Create Resource Groups :

        Extended Configuration -->
            Extended Resource Configuration -->
                HACMP Extended Resource Group Configuration -->
                    Add a Resource Group -->
                    Enter RG Name for Service Labels "ddcapedn07"
                    Participating Nodes "ddcapedn07 ddcapedn08"
                    ( Priority Order of Nodes for the group )
                     *****  Accept defaults for additional fields.
        *****  Perform Twice if Mutual Failover else Once if 1-way.
                                    Enter RG Name for Volume groups "ednpdtu7"
                        Participating Nodes "ddcapedn07 ddcapedn08"
                    ( Priority Order of Nodes for the group )
        *****  Perform Twice if Mutual Failover else Once if 1-way.
    

11. Add Application Server :

        ***** Perform these steps for application servers only.
          Initialization and Standard Configuration -->
            Configure Resources to Make Highly Available -->
                Configure Application Servers -->
                    Add an Application Server -->
                    Enter APP SERVER RG Name "ednpdtu7_appl01"
                    Enter Start Script "/usr/local/hascripts/ednpdtu7_appl01_start.sh"
                    Enter Stop Script  "/usr/local/hascripts/ednpdtu7_appl01_stop.sh"
                    ( Application Start and Stop Scripts )
    

12. Configure Attributes for New Resource Groups :

        Extended Configuration -->
            Extended Resource Configuration -->
                Extended Resource Group Configuration -->
                    Change/Show Resources and Attributes for a Resource Group -->
                    Select RG "ddcapedn07"
                    Enter Service IP Labels "ddcapedn07"
                    ( Failover of Serve IP for RG )
        *****  Perform Twice if Mutual Failover else Once if 1-way.
                    Select RG "ednpdtu7"
                    Enter Application Server "ednpdtu7_appl01"
                    Enter Volume Groups "ednpdtu700vg"
                    ( Failover of VG's )
    

13. Synchronize Cluster configuration :

        Extended Configuration -->
            Extended Verification and Synchronization -->
            ***** Accept defaults and synchronize cluster.
    

14. Discover HACMP-related Information from Clustered Nodes :

        Discover HACMP-related Information from Configured Nodes
        ( Collect disk info for Disk Heartbeat Configuration etc ........ )
    

15. Disk Heartbeat Configuration :

        Extended Configuration -->
            Extended Topology Configuration -->
                Configure HACMP Communication Interfaces/Devices -->
                    Add Communication Interfaces/Devices -->
                    Select "Add Discovered Communication Interface and Devices"
                    Select "Communication Devices"
                    Select 1 Disk from each Node /LPAR "ddcapedn07 ddcapedn08"
        *****  Configure Disk Heartbeat for all VG in cluster.
    

16. Synchronize Cluster configuration :

       Extended Configuration -->
            Extended Verification and Synchronization -->
            ***** Accept defaults and synchronize cluster.