/* IBM_PROLOG_BEGIN_TAG                                                   */
/* This is an automatically generated prolog.                             */
/*                                                                        */
/*                                                                        */
/*                                                                        */
/* Licensed Materials - Property of IBM                                   */
/*                                                                        */
/* (C) COPYRIGHT International Business Machines Corp. 1996,2021          */
/* All Rights Reserved                                                    */
/*                                                                        */
/* US Government Users Restricted Rights - Use, duplication or            */
/* disclosure restricted by GSA ADP Schedule Contract with IBM Corp.      */
/*                                                                        */
/* IBM_PROLOG_END_TAG                                                     */

static char *sccsid = "@(#)91   1.50   src/rsct/pgs/samples/sample_test.c, gssamples, rsct_rady, radys003a 6/6/21 12:23:55";

#if !defined(_HAGSD_COPYRIGHT_H)
#define _HAGSD_COPYRIGHT_H
static char copyright[] = "Licensed Materials - Property of IBM\n\
(C) COPYRIGHT International Business Machines Corp. 1996,2001.\n\
All Rights Reserved.\n\
US Government Users Restricted Rights - Use, duplication or \n\
disclosure restricted by GSA ADP Schedule Contract with IBM Corp.\n";
#endif

/*********************************************************************/
/*
 * Name:  sample_test.c
 *
 * This module controls an interactive program to use to interface with
 * the Group Services interfaces provided by IBM RSCT.
 *
 * This program allows you to join groups on one or more nodes, to
 * subscribe to groups, and to propose various protocols and respond
 * with votes.  It may be built as a threaded or non-threaded application.
 *
 * Components:
 *   sample_test.c - this module, supports interaction with the user, and
 *      most calls to the Group Services interfaces.
 *
 *   sample_callbacks.c - provides the definitions for the callback
 *      functions used for the groups created by this program.
 *
 *   sample_utility.c - provides the definitions for the utility functions
 *      used by the sample_test program.
 *
 *   sample_callbacks.h - declarations for the callback functions
 *      contained in sample_callbacks.c.
 *
 *   sample_utility.h - declarations for the utility functions contained
 *      in sample_utility.c
 *
 * The information here assumes that you are familiar with the information
 * presented in the IBM RSCT Group Services Programming Guide and Reference
 * manual.
 *
 * This program is provided for illustrative purposes only, and is not
 * intended to be an authoritative description of the "best" methods to
 * use when writing a Group Services application.  It is intended to
 * demonstrate the various interfaces in a relatively verbose manner,
 * and to allow you to relatively easily manipulate groups and their
 * members.
 *
 * To this end, various aspects of this program (in particular its
 * handling of screen input and output) are neither robust nor
 * foolproof.  Therefore, you should take care when giving input to
 * this program.
 */
/*********************************************************************/

/*********************************************************************/
/*
 * To build this program, use the Makefile in this directory:
 *
 *      make all (to build all sample programs)
 * or
 *      make sample_test (to build this program non-threaded)
 *      make sample_test_r (to build this program threaded)
 *
 * To execute this program, you need to have Group Services active on
 * your system.  Please refer to the manual if you have questions as to
 * how you may verify this.  Additionally, you need to execute as a
 * privileged (root) process.  Also, you should take care in the group
 * names you use, to avoid clashing with other groups.
 *
 * Prior to starting this program, you need to set the name of your SP
 * partition in the environment of the process running the program.
 * You should export the variable HA_SYSPAR_NAME to be the name of the
 * partition.  Contact your system administrator if you do not know how
 * to determine this.
 */
/*********************************************************************/

/*********************************************************************/
/*
 * This program supports two command-line arguments:
 *
 *  -v -- "verbose" mode.  If given, then the program will display
 *      full instructions with each prompt, as well as displaying
 *      detailed information for each responsiveness callback received,
 *      and a message will be displayed each time select() times out
 *      without receiving a message.
 *      If not given, then the instructions will be displayed only when
 *      requested (via an "h" command), 'Rcb' will be displayed for each
 *      responsiveness callback received, and a period ('.') will be
 *      displayed each time select() times out without receiving a
 *      message.
 *
 *  -i -- "interactive initialization" mode.  If given, then the user
 *      will be prompted for the arguments to be given on the
 *      ha_gs_init() call which initializes the process'es connection
 *      to Group Services.
 *      If not given, then the program will use predefined values for
 *      the call to ha_gs_init().
 *
 * In addition, via "make all" a number of linked executables will be
 * created (sample_test1-sample_test5 and sample_test_r1-sample_test_r5.)
 * The integer appended to the program name will be used as an "index"
 * for constructing the provider instance numbers when joining groups,
 * as well as modifying some of the group attributes in certain cases.
 * This allows you to easily execute different versions on a single
 * node, and be able to join multiple providers to a group from that
 * node.
 *
 * The easiest way to examine this program's usefulness is to log onto
 * a node and start the program.  Details about the commands supported
 * by sample_test, as well as descriptions of the outputs displayed,
 * follow.
 */
/*********************************************************************/

/*********************************************************************/
/*
 * The various commands supported in sample_test generally correspond to
 * the Group Services interfaces defined in the IBM RSCT Group Services
 * Programming Guide and Reference manual.  Please refer to the manual
 * for detailed descriptions of the Group Services interfaces.
 *
 * This description summarizes the sample_test commands (all commands
 * may be given in upper or lower case):
 *
 * Initialization (ha_gs_init()):
 *  'i' -- will call ha_gs_init() to initialize your process as a client
 *      of Group Services.  If you specified "-i" as a command-line
 *      argument, then you will be prompted for the parameters for the
 *      call to ha_gs_init().  If not, the following default parameters
 *      are used:
 *  ha_gs_socket_ctrl_t HA_GS_SOCKET_NO_SIGNAL;
 *  ha_gs_responsiveness_t {HA_GS_PING_RESPONSIVENESS,
 *                          3600,
 *                          10,
 *                          (char *)0,
 *                          0};
 *  char * default_deact=0;
 *  ha_gs_responsiveness_cb_t *responsive_cb,
 *  ha_gs_delayed_error_cb_t *delayed_error_cb,
 *  ha_gs_query_cb_t *query_cb);
 *
 *  '9' -- same as 'i' but initializes with a domain control callback.
 *
 * Joining groups (ha_gs_join()):
 *  Pre-defined groups:
 *   '1' -- attempt to join group "OnePhaseJoin".
 *   'j' -- attempt to join group "theSourceGroup".
 *   'x' -- attempt to join group "theTargetGroup".
 *   'm' -- attempt to join group "theLonelyGroup".
 *   'a' -- attempt to join group "ifFirstIWin".
 *   'g' -- attempt to join group "SourceOrNot".
 *   '3' -- attempt to join group "ChainGang".
 *  See below for details of the group attributes for each of these groups.
 *  "Roll-your-own" group:
 *   'b' -- interactively define a group to attempt to join.  For this
 *      group only, you will be prompted to enter the group attributes.
 *
 * Other protocol proposals (you need to have joined one or more groups
 * before using these).  When you issue one of these commands, you will
 * be prompted for the proposal data (number of phases, time limit, other
 * data for the proposal):
 *  Group state value changes (ha_gs_change_state_value()):
 *   't' -- propose a state value change protocol.
 *  Group broadcast message (ha_gs_send_message()):
 *   'p' -- propose a provider broadcast message protocol.
 *  Voluntarily leave a group (ha_gs_leave()):
 *   'l' -- propose to have this provider leave a group voluntarily.
 *  Expel one or more providers from a group (ha_gs_expel()):
 *   'e' -- propose to expel one or more providers.
 *  Change a group's attributes (ha_gs_change_attributes()):
 *   'n' -- propose to change the attributes of a group to which you
 *          must already be joined.
 *  To tell a group you've joined goodbye (ha_gs_goodbye()):
 *   'y' -- assuming it is valid to use, this will get the provider
 *          immediately out of a group, and it will no longer see
 *          any notifications for the group.
 *
 * Subscribing to groups (ha_gs_subscribe()):
 *  's' -- will prompt you for the name of the group to which it should
 *      subscribe.  This may be one of the groups listed above under
 *      "joining", it may be one of the system membership groups (host
 *      or adapter membership), or you may specify any group name.  If
 *      the group does not exist, then a delayed error will be returned.
 *      In addition to the group name, you will also be prompted for the
 *      subscription control information (e.g., membership only, state
 *      value only, membership and state value, etc.)
 *
 * Unsubscribing from a group (ha_gs_unsubscribe()):
 *  'u' -- unsubscribe from a group.  Similar to subscribe, you will be
 *      prompted for the group name to unsubscribe from.
 *
 * Dispatch (ha_gs_dispatch()):
 *  'd' -- call ha_gs_dispatch() to process messages received from Group
 *      Services.
 *      NOTE: this is normally done automatically when sample_test detects
 *      via the AIX "select()" library call that a message has arrived on
 *      the socket that connects the client to Group Services.  Issuing
 *      the 'd' command is normally harmless, in that it should just
 *      return without any errors.
 *
 * Quit (ha_gs_quit()):
 *  'q' -- close the socket connecting the client to Group Services.  If
 *      the process is still joined as a provider to any groups, Group
 *      Services will treat this as a failure of the provider.
 *
 *
 * Get Help:
 *  'h' -- will display a summary of information for these commands.
 */
/*********************************************************************/

/*********************************************************************/
/* Toggle "HA_GS_DEACTIVATE_ON_FAILURE" of batch control
 * 'z' -- will toggle HA_GS_DEACTIVATE_ON_FAILURE
 *        that was initialized by the environment variable 
 *        HA_GS_DEACTIVATE_ON_FAILURE
 */
/*********************************************************************/

/*********************************************************************/
/*
 * This section describes the attributes provided for the pre-defined
 * groups.  To change these, you need to modify this source code and
 * recompile.  Note that unless you ensure the changed executable is
 * available on all nodes where you want to run it (to ensure all
 * providers trying to join a group have matching attributes) some of
 * the join requests may be rejected due to mismatched attributes.
 *
 * Also, the 'b' command allows you to "B"uild a group by interactively
 * providing the attributes, without changing this program's source
 * code.
 * 
 * DeactivateOnFailure is controlled by the environment variable,
 * HA_GS_DEACTIVATE_ON_FAILURE.  For example, 
 * 'export HA_GS_DEACTIVATE_ON_FAILURE=1' enables the feature of 
 * DeactivateScriptOnFailure. It is also toggled thru 'z'.
 * In the followings, "deact_on_fail_ctrl" denotes the control
 *
 * For the predefined groups then:
 *  theSourceGroup:
 *      gs_client_version = 1;
 *      gs_batch_control = HA_GS_BATCH_BOTH | deact_on_fail_ctrl;
 *      gs_num_phases = HA_GS_N_PHASE;
 *      gs_source_reflection_num_phases = HA_GS_N_PHASE;
 *      gs_group_default_vote = HA_GS_VOTE_REJECT;
 *      gs_merge_control = HA_GS_DONTCARE_MERGE;
 *      gs_time_limit = 60;
 *      gs_source_reflection_time_limit = 30;
 *      gs_group_name = "theSourceGroup";
 *      gs_source_group_name = "";
 *      gs_provider_instance = 100 + sample_index;
 *      gs_provider_local_name = "SourceJoin";
 *      gs_n_phase_protocol_callback = n_phase_cb0;
 *      gs_protocol_approved_callback = approved_cb0;
 *      gs_protocol_rejected_callback = rejected_cb0;
 *      gs_announcement_callback = announce_cb0;
 *      gs_merge_callback = merge_cb0;
 *
 *   OnePhaseJoin:
 *      gs_client_version = 1;
 *      gs_batch_control = HA_GS_BATCH_BOTH | deact_on_fail_ctrl;
 *      gs_num_phases = HA_GS_1_PHASE;
 *      gs_source_reflection_num_phases = HA_GS_1_PHASE;
 *      gs_group_default_vote = HA_GS_VOTE_REJECT;
 *      gs_merge_control = HA_GS_DONTCARE_MERGE;
 *      gs_time_limit = 0;
 *      gs_source_reflection_time_limit = 0;
 *      gs_group_name = "OnePhaseJoin";
 *      gs_source_group_name = "";
 *      gs_provider_instance = 200 + sample_index;
 *      gs_provider_local_name = "SinglePhase";
 *      gs_n_phase_protocol_callback = n_phase_cb1;
 *      gs_protocol_approved_callback = approved_cb1;
 *      gs_protocol_rejected_callback = rejected_cb1;
 *      gs_announcement_callback = announce_cb1;
 *      gs_merge_callback = merge_cb1;
 *
 *   theTargetGroup (uses "theSourceGroup" as its source-group):
 *      gs_client_version = 1;
 *      gs_batch_control = HA_GS_BATCH_BOTH | deact_on_fail_ctrl;
 *      gs_num_phases = HA_GS_N_PHASE;
 *      gs_source_reflection_num_phases = HA_GS_N_PHASE;
 *      gs_group_default_vote = HA_GS_VOTE_REJECT;
 *      gs_merge_control = HA_GS_DONTCARE_MERGE;
 *      gs_time_limit = 60;
 *      gs_source_reflection_time_limit = 30;
 *      gs_group_name = "theTargetGroup";
 *      gs_source_group_name = "thsSourceGroup";
 *      gs_provider_instance = 300 + sample_index;
 *      gs_provider_local_name = "Existing";
 *      gs_n_phase_protocol_callback = n_phase_cb2;
 *      gs_protocol_approved_callback = approved_cb2;
 *      gs_protocol_rejected_callback = rejected_cb2;
 *      gs_announcement_callback = announce_cb2;
 *      gs_merge_callback = merge_cb2;
 *
 *   theLonelyGroup (uses "theMissingGroup" as its source-group):
 *      gs_client_version = 1;
 *      gs_batch_control = HA_GS_BATCH_BOTH | deact_on_fail_ctrl;
 *      gs_num_phases = HA_GS_N_PHASE;
 *      gs_source_reflection_num_phases = HA_GS_N_PHASE;
 *      gs_group_default_vote = HA_GS_VOTE_REJECT;
 *      gs_merge_control = HA_GS_DONTCARE_MERGE;
 *      gs_time_limit = 60;
 *      gs_source_reflection_time_limit = 30;
 *      gs_group_name = "theLonelyGroup";
 *      gs_source_group_name = "theMissingGroup";
 *      gs_provider_instance = 400 + sample_index;
 *      gs_provider_local_name = "Missing";
 *      gs_n_phase_protocol_callback = n_phase_cb3;
 *      gs_protocol_approved_callback = approved_cb3;
 *      gs_protocol_rejected_callback = rejected_cb3;
 *      gs_announcement_callback = announce_cb3;
 *      gs_merge_callback = merge_cb3;
 *
 *   ifFirstIWin (modifies attributes based on its index value, e.g.,
 *                if my index is even (i.e., sample_test2) then specify
 *                1-phase joins/failures; if odd, n-phase.):
 *      gs_client_version = 1;
 *      gs_batch_control = HA_GS_BATCH_BOTH | deact_on_fail_ctrl;
 *      if (0 == (sample_index % 2)) {
 *         gs_num_phases = HA_GS_1_PHASE;
 *      } else {
 *         gs_num_phases = HA_GS_N_PHASE;
 *      }
 *      gs_source_reflection_num_phases = HA_GS_N_PHASE;
 *      gs_group_default_vote = HA_GS_VOTE_REJECT;
 *      gs_merge_control = HA_GS_DONTCARE_MERGE;
 *      gs_time_limit = 60;
 *      gs_source_reflection_time_limit = 30;
 *      gs_group_name = "ifFirstIWin";
 *      gs_source_group_name = "";
 *      gs_provider_instance = 500 + sample_index;
 *      gs_provider_local_name = "AmIFirstOrALoser";
 *      gs_n_phase_protocol_callback = n_phase_cb4;
 *      gs_protocol_approved_callback = approved_cb4;
 *      gs_protocol_rejected_callback = rejected_cb4;
 *      gs_announcement_callback = announce_cb4;
 *      gs_merge_callback = merge_cb4;
 *
 *   SourceOrNot (modifies source-group names based on its index value,
 *                e.g., if my index is even (i.e., sample_test2) then
 *                specify "theSourceGroup" as this group's source-group,
 *                if odd, do not specify a source-group.):
 *      gs_client_version = 1;
 *      gs_batch_control = HA_GS_BATCH_BOTH | deact_on_fail_ctrl;
 *      gs_num_phases = HA_GS_N_PHASE;
 *      gs_source_reflection_num_phases = HA_GS_N_PHASE;
 *      gs_group_default_vote = HA_GS_VOTE_REJECT;
 *      gs_merge_control = HA_GS_DONTCARE_MERGE;
 *      gs_time_limit = 60;
 *      gs_source_reflection_time_limit = 30;
 *      gs_group_name = "SourceOrNot";
 *      if (0 == (sample_index % 2)) {
 *         gs_source_group_name = "theSourceGroup";
 *      } else {
 *         gs_source_group_name = "";
 *      }
 *      gs_provider_instance = 600 + sample_index;
 *      gs_provider_local_name = "MaybeSourceIt";
 *      gs_n_phase_protocol_callback = n_phase_cb5;
 *      gs_protocol_approved_callback = approved_cb5;
 *      gs_protocol_rejected_callback = rejected_cb5;
 *      gs_announcement_callback = announce_cb5;
 *      gs_merge_callback = merge_cb5;
 *
 *   ChainGang (uses theTargetGroup as its source-group, to create a
 *             3-level hierarchy):
 *      gs_client_version = 3;
 *      gs_batch_control = HA_GS_BATCH_JOINS | deact_on_fail_ctrl;
 *      gs_num_phases = HA_GS_N_PHASE;
 *      gs_source_reflection_num_phases = HA_GS_1_PHASE;
 *      gs_group_default_vote = HA_GS_VOTE_APPROVE;
 *      gs_merge_control = HA_GS_DONTCARE_MERGE;
 *      gs_time_limit = 60;
 *      gs_source_reflection_time_limit = 0;
 *      gs_group_name = "ChainGang";
 *      gs_source_group_name = "theTargetGroup";
 *      gs_provider_instance = 700 + sample_index;
 *      gs_provider_local_name = "DaisyChain";
 *      gs_n_phase_protocol_callback = n_phase_cb6;
 *      gs_protocol_approved_callback = approved_cb6;
 *      gs_protocol_rejected_callback = rejected_cb6;
 *      gs_announcement_callback = announce_cb6;
 *      gs_merge_callback = merge_cb6;
 *
 * Note that for the interactively-defined group ('b' command), it will
 * use the following set of callback functions, regardless of the
 * remaining attributes specified interactively:
 *
 *      gs_n_phase_protocol_callback = n_phase_cb7;
 *      gs_protocol_approved_callback = approved_cb7;
 *      gs_protocol_rejected_callback = rejected_cb7;
 *      gs_announcement_callback = announce_cb7;
 *      gs_merge_callback = merge_cb7;
 *
 * The callback functions are declared in sample_callbacks.h and are
 * defined in sample_callbacks.c.
 */
/*********************************************************************/

/*********************************************************************/
/*
 * Program structure:
 *
 * The main() function here is essentially a large while() loop, that
 * continues until an error condition is returned from ha_gs_dispatch(),
 * or until the 'q'uit command is specified.  A select() is used to
 * detect if the user has typed input, or if a message has arrived on
 * the socket connecting this program to Group Services.
 *
 * In the former case (a command is entered) the command is checked for
 * correctness, and the desired operation is performed.  E.g., if a 'j'
 * is entered, then the program will attempt to join the group named
 * "theSourceGroup".
 *
 * In the latter case (a message from Group Services) then the program
 * will automatically call ha_gs_dispatch() to process the messages.
 * In general, this will result in calls to the appropriate callback
 * functions.
 *
 * Note for the THREAD-SAFE version (sample_test_r): a separate thread
 * will be created (via pthread_create()) once this program has been
 * successfully initialized ('i' command) with Group Services.  This
 * thread will call ha_gs_dispatch(), and, whenever a message arrives
 * on the socket connecting this program to Group Services, that thread
 * will read the message and perform the appropriate actions.  The
 * primary thread will continue to monitor for user-inputted commands.
 *
 * The callback functions are defined in sample_test.c, and are linked
 * into this program when it is built.
 */
/*********************************************************************/

/*********************************************************************/
/*
 * Note on callback functions.
 *
 * The callback functions used by this program to handle asynchronous
 * messages received from Group Services are defined in the file
 * sample_callbacks.c.  In general, the functions will perform the
 * "appropriate" actions based on the callback being executed and the
 * contents of the message received.
 *
 * In almost all cases, the callback function will display the set of
 * information included in the message.  This information will normally
 * apply to a protocol being executed, or completing execution, in a
 * group to which this process is joined as a provider.  Or, the
 * callback may be delivering a "delayed error" applying to one of
 * this process'es proposals.  Finally, if this process is subscribing
 * to one or more groups, the callback will display information
 * pertaining to the subscription notification.
 *
 * If the callback being executed is the gs_n_phase_protocol_callback
 * for a group, then the user will be prompted for to "vote" (which
 * will result in a call to ha_gs_vote()) on the currently-executing
 * protocol.  If there is a time-limit associated with the protocol,
 * then the time limit will be displayed, and the vote response must
 * be made within that time limit, or else Group Services will apply
 * the group's default vote in lieu of your submitted vote value.
 *
 * When you submit a vote, this program will additionally submit a
 * rotating set (i.e., each subsequent phase of an n-phase protocol
 * will get the "next" entry in the array) of group state value changes
 * and/or provider-broadcast messages and/or default vote changes along
 * with the vote value.  (See the IBM RSCT Group Services Programming
 * Guide and Reference manual discussion of the ha_gs_vote() interface
 * for full information on what can be submitted during voting, and
 * voting on protocols in general.)
 *
 * You can modify the entries submitted by changing the entries in the
 * arrays (in sample_utility.c) and rebuilding this program:
 *      provider_msg_array[]
 *      state_value_array[]
 *      default_vote_array[]
 * If you specify {0,0} as an entry for provider_msg_array[] or
 * state_value_array[], then no provider-broadcast message or state value
 * will be submitted with the vote value for that phase.  Likewise,
 * specifying HA_GS_NULL_VOTE as the default_vote_array[] entry means
 * the group's current default vote value remains in effect.
 *
 * Additionally, on all submitted provider-broadcast messages and group
 * state values, this program will prepend a "tag" identifying the
 * name of the program submitting it.  The "tag" will be prepared using
 * the program name and the "index" described earlier.  For example:
 *      program executed: sample_test     tag: '0<sample_test>'
 *      program executed: sample_test4    tag: '4<sample_test4>'
 *      program executed: sample_test_r2  tag: '2<sample_test_r2>'
 */
/*********************************************************************/

/*********************************************************************/
/*
 * Output of the program.
 *
 * In the following, lines beginning with '%%%%' are annotations that
 * do not actually appear in the displayed output.
 *
 * up sample_test (details about the commands follow this example):

%%%% Log onto a node with Group Services active, then start this
%%%% program (refer to the IBM RSCT Group Services Programming Guide
%%%% and Reference manual for details about starting Group Services
%%%% itself):

k21n11.ppd.pok.ibm.com:[pgs/bin]$ sample_test1            
**********++++++++++==========**********++++++++++==========

My program index is 1 and the program name is: <sample_test1>

**********++++++++++==========**********++++++++++==========
Please enter a command('h' for help): 

%%%% You need to initialize before doing much:

Please enter a command('h' for help): i
Responsiveness parameters specified:  Type[HA_GS_PING_RESPONSIVENESS]
Interval[3600 seconds] Response time limit[10 seconds]

%%%% The program echos the arguments to be given to ha_gs_init().
%%%% This uses the default parameters for responsiveness settings.

ha_gs_init returned rc:[HA_GS_OK]

%%%% We received a successful return code (HA_GS_OK) from ha_gs_init().
%%%% Rememberin that ha_gs_init() is the only purely synchronous
%%%% interface to Group Services, this means we are connected and
%%%% ready to join or subscribe to groups.

%%%% Attempt to join the group named "OnePhaseJoin".  This means that
%%%% we will not have to vote on the join, as there will be only 1
%%%% phase, and the join will be automatically approved.

Please enter a command('h' for help): 1
Attempting JOIN for group[OnePhaseJoin]:
 My instance #[201]  local name[SinglePhase]
Group attributes: 
 version[1] size[40] supplicant version[1]
 Batching[HA_GS_BATCH_BOTH] Join/Fail phases[1_PHASE] Reflection phases[1_PHASE]

 Default vote[HA_GS_VOTE_REJECT] Merge[HA_GS_DONTCARE_MERGE]
 Time limits: join/fail[0] reflection[0]
 Group name[OnePhaseJoin] Source group name[]
**********++++++++++==========**********++++++++++==========

%%%% First, sample_test echoes the attributes to be specified on the
%%%% call to ha_gs_join().

ha_gs_join so far returned rc:[HA_GS_OK]

%%%% Synchronous return code says join request accepted, need to wait
%%%% for the asynchronous callbacks.

Please enter a command('h' for help): 

%%%% The gs_protocol_approved_callback displays the following message
%%%% when Group Services sends the join approved notification to this
%%%% program.  The select() call detects that the message has arrived
%%%% on the socket connecting this program to Group Services, and it
%%%% automatically calls ha_gs_dispatch().  Refer to the descriptions
%%%% of the various notification data structures in the IBM High 
%%%% Availability Services manual for full details of each of these 
%%%% fields.

[TOD(May 29 18:53:11)]Approved Callback One called
*********************  Wed May 29 18:53:11 1996  *********************
Type[HA_GS_JOIN] Token[0]
%%%% Displays kind of protocol, the assigned provider_token.
Summary_Code[HA_GS_EXPLICIT_APPROVE|]
%%%% Summary code indicates that no default votes were needed.
NumPhases[1_PHASE] ThisPhase[1] Proposer[201/11(13172747)]
%%%% NumPhases shows that this was a 1-phase protocol, therefore
%%%% ThisPhase can only be 1.  The proposer shows the ha_gs_provider_t
%%%% of THIS provider process.  The ha_gs_provider_t representing each
%%%% is also called its "provider_ID".  Each provider_ID is displayed
%%%% as three values: the provider's instance number, followed by the
%%%% slash ('/') and the provider's node number.  In parantheses, since
%%%% each ha_gs_provider_t is two shorts in a union with an int, is the
%%%% "int" value of the provider_ID.  Therefore, 201/11(13172747) shows
%%%% that this process is provider instance number 201 running on node
%%%% number 11.  13172747 is the int equivalent of the two shorts 
%%%% (201,11) in the union.
WhatsChanged[HA_GS_UPDATED_MEMBERSHIP]
%%%% WhatsChanged shows that this notification specifies that the
%%%% group's membership has changed.
CurrentProviders[count[1] Members[201/11(13172747) ]]
%%%% This lists the group's current providers.  This provider is the
%%%% only provider, so is the only one listed.
ChangingProviders[count[1] Members[201/11(13172747) ]]
%%%% This lists the providers joining the group during this protocol.
%%%% Again, this is the only provider.
>>>> Grabbed my provider id [201/11(13172747)]
CurrentState[Length[4] Value[0]]
%%%% This shows the current state value of the group.  Since we have 
%%%% not explicitly changed it, it is the initial state value of an 
%%%% integer zero (0).
**********************************************************************
ha_gs_dispatch returned rc:[HA_GS_OK]
Please enter a command('h' for help): 

 *
 * Assume now that we want to run an n-phase protocol, so, we will
 * propose an n-phase state value change protocol.
 *

Please enter a command('h' for help): t
Please enter a command('h' for help): **********++++++++++==========**********++
++++++++==========

So, you want to change the group's state value?
I would ask you to which group is this proposal to be targeted, but since
 you are only in the group named [OnePhaseJoin] you are going to use that one!

%%%% This is the only group we've joined, so the proposal will be targeted
%%%% to this group.

Now that you've chosen group [OnePhaseJoin], we need the protocol information:
Number of phases('1' or 'N'): n
%%%% Ask for n-phases.
Please specify time limit for for the state value change protocol.
 In seconds, range 0 (no time limit) to 65,535: 60
%%%% The provider(s) will have to vote within 60 seconds.
Please enter your proposed state value, anything up to 256 bytes:
This is the new state value.
%%%% Type in anything here.

%%%% The program echoes the choices you have made (remember, above we
%%%% described that the program will prepend a "tag" to each proposed
%%%% group state value or provider-broadcast message.  In this case,
%%%% the tag is "1<sample_test1>".
You have chosen [N_PHASE] with time limit of [60] and a proposed state of:
1<sample_test1>This is the new state value.
Thank you.  Will now submit your proposal.
**********++++++++++==========**********++++++++++==========
ha_gs_change_state_value so far returned rc:[HA_GS_OK]
Please enter a command('h' for help): 

%%%% The callback is executed when the notification arrives telling us
%%%% that we have to vote.
[TOD(May 31 10:09:32)]N Phase Callback One called
*********************  Fri May 31 10:09:32 1996  *********************
Type[HA_GS_STATE_VALUE_CHANGE] Token[0] TimeLimit[60]
%%%% Describes the type of protocol and the current voting time limit.
Summary_Code[]
%%%% No voting has yet taken place, so no summary code.
NumPhases[N_PHASE] ThisPhase[1] Proposer[201/10(13172746)]
%%%% n-phase protocol, this is phase 1, proposer was us.
WhatsChanged[HA_GS_PROPOSED_STATE_VALUE]
%%%% The only proposed change here is the group state value.
CurrentProviders[count[1] Members[201/10(13172746) ]]
CurrentState[Length[4] Value[0]]
%%%% This is the group's current state value.
ProposedState[Length[46] Value[1<sample_test1>This is the new state value.]]
%%%% This is the proposed state value.
**********************************************************************
Protocol is HA_GS_STATE_VALUE_CHANGE
SUMMARY_CODE=
**********++++++++++==========**********++++++++++==========
%%%% The program requests us to enter a vote value.  See the description
%%%% of ha_gs_vote() and related material in the IBM RSCT Group Services
%%%%  Programming Guide and Reference manual
**********++++++++++==========**********++++++++++==========
Enter a vote value: C [Continue]  A [Approve]  R [Reject]
 or, to force 'nested' proposals: T [State change] P [PBM]
%%%% We vote to approve this protocol.
a
You have chosen well, grasshopper. This protocol will be approved here and now!
**********++++++++++==========**********++++++++++==========
Phase[1]vote value[HA_GS_VOTE_APPROVE]default vote value[HA_GS_NULL_VOTE]
**********++++++++++==========**********++++++++++==========
ha_gs_dispatch returned rc:[HA_GS_OK]
Please enter a command('h' for help): 

%%%% A new notification arrives, and the appropriate callback function
%%%% is executed.
[TOD(May 31 10:09:54)]Approved Callback One called
*********************  Fri May 31 10:09:54 1996  *********************
Type[HA_GS_STATE_VALUE_CHANGE] Token[0]
Summary_Code[HA_GS_EXPLICIT_APPROVE|]
NumPhases[N_PHASE] ThisPhase[1] Proposer[201/10(13172746)]
WhatsChanged[HA_GS_UPDATED_PROVIDER_MESSAGE HA_GS_UPDATED_STATE_VALUE]
%%%% A provider-broadcast message is included, also, since we approved
%%%% the proposed state value change, the state value is updated.
CurrentProviders[count[1] Members[201/10(13172746) ]]
CurrentState[Length[46] Value[1<sample_test1>This is the new state value.]]
%%%% The proposed state value is now the committed state value.
ProposedState[Length[46] Value[1<sample_test1>This is the new state value.]]
ProviderMessage[Length[67] Value[1<sample_test1>Things do seem to be looking up 
now, do they not?]]
**********************************************************************
ha_gs_dispatch returned rc:[HA_GS_OK]
Please enter a command('h' for help): 

 *
 * You can run this program and try the other various commands.  Please
 * refer to the IBM RSCT Group Services Programming Guide and Reference
 * manual for details of the interfaces, data structures, notifications,
 * etc.
 */
/*********************************************************************/

/*********************************************************************/
/*
 * Include "standard" system files.  Note that pthread.h must be the
 * first file included, if it is to be included (see the standard AIX
 * documentation for more information about AIX thread support.)
 */
/*********************************************************************/

#if defined(__linux__) || defined(__INTERIX) || defined(__sun)
#define NFDS(a) a
#endif

#ifdef _THREAD_SAFE                     /* begin _THREAD_SAFE */
#include <pthread.h>
#endif                                  /* end _THREAD_SAFE */

#include <stdio.h>
#include <stdlib.h>
#include <sys/signal.h>
#include <sys/select.h>
#include <sys/time.h>
#define _XOPEN_EXTENDED_SOURCE          /* AIX 4.1 */
#include <arpa/inet.h>
#undef _XOPEN_EXTENDED_SOURCE           /* AIX 4.1 */
#include <errno.h>
#include <strings.h>
#include <memory.h>
#include <assert.h>
#include <signal.h>

/*********************************************************************/
/*
 * Include the Group Services declarations file.
 */
/*********************************************************************/

#include <ha_gs.h>

/*********************************************************************/
/*
 * Include the set of declarations for callbacks for this program.
 */
/*********************************************************************/

#include "sample_callbacks.h"

/*********************************************************************/
/*
 * Include the set of declarations for utility functions for this program.
 */
/*********************************************************************/

#include "sample_utility.h"

/*********************************************************************/
/*
 * For the thread-safe version, define a thread that will simply call
 * ha_gs_dispatch(), and remain there until the connection is closed.
 *
 * Note that an alternate mechanism would be to create a separate
 * thread whenever a message is detected on the socket connecting this
 * program to Group Services, and having that thread call ha_gs_dispatch()
 * (using the HA_GS_NON_BLOCKING flag) and then exiting once it returns.
 */
/*********************************************************************/

#ifdef _THREAD_SAFE                     /* begin _THREAD_SAFE */
#define MAX_THREADS 5
void *dispatch_thread(void *null) {
    int rc;
    rc = HA_GS_OK;
    fprintf(stderr, "dispatch_thread called\n");
    while (HA_GS_OK == rc) {
        rc = ha_gs_dispatch(HA_GS_BLOCKING);
             fprintf(stderr, "Returned from ha_gs_dispatch() with rc[%s]\n",
                     write_an_rc(rc));

	if(rc != HA_GS_OK) {
                fc_eid_t fid;
                memset(fid, 0, sizeof(fid));
		ha_gs_get_ffdc_id(fid);
                fprintf(stderr, " FFDCID=[%s] is received\n", fid);
                assert( rc == HA_GS_OK );
	}
    }
}
#endif                                  /* end _THREAD_SAFE */

/*************************************************************
 *************************************************************/

static  int     fulldump_signals[] = {
    SIGQUIT,
    SIGILL,
    SIGTRAP,
    SIGFPE,
#ifdef _AIX
    SIGKILL,
#endif
    SIGBUS,
    SIGSEGV,
#ifdef _AIX
    SIGSYS,
#endif
    SIGABRT
};

#define n_fulldumps (sizeof(fulldump_signals) / sizeof(int))

void set_fulldump_signals()
{
    int i, rc, sig;
    struct  sigaction       invec;

    for (i = 0; i < n_fulldumps; i++)
    {
        sig = fulldump_signals[i];
        sigaction(sig, (struct sigaction *) NULL, &invec);
        invec.sa_handler = SIG_DFL;
#ifdef _AIX
        invec.sa_flags |= SA_FULLDUMP;
#endif
        sigaction(sig, &invec, (struct sigaction *) NULL);
    }
}


/*************************************************************
 * Default DeactivateOnFailure batch_control value:
 * This will be supplied thru the environment variable 
 * "HA_GS_DEACTIVATE_ON_FAILURE".
 * i.e., to turn on this,
 * export HA_GS_DEACTIVATE_ON_FAILURE = 1 
 *************************************************************
 */
static ha_gs_batch_ctrl_t deact_on_fail_ctrl = 0;


/*********************************************************************/
void usage(const char *prog)
{
    fprintf(stderr, "%s [-i] [-v] [-m] [-c]\n", prog);
    fprintf(stderr, "      -i  interactive mode to specify init parameters \n");
    fprintf(stderr, "      -v   verbose mode\n");
    fprintf(stderr, "      -m   domain master mode\n");
    fprintf(stderr, "      -c   critical client mode\n");
}


/*********************************************************************/
/*
 * The main function.  Determine the name of the program, what, if any,
 * command line flags were given, and then set up the select() loop.
 */
/*********************************************************************/
int main(int argc,
          char ** argv)
{
#ifdef _THREAD_SAFE                     /* begin _THREAD_SAFE */
    pthread_t dispatch_id[MAX_THREADS + 1];
#endif                                  /* end _THREAD_SAFE */

    /*********************************************************************/
    /*
     * This is the file descriptor of the socket that will connect this
     * program to Group Services.  It will be initialized when we call
     * ha_gs_init().
     */
    ha_gs_descriptor_t socket_fd;

    /*********************************************************************/
    /*
     * This defines the "default" responsiveness settings used for the
     * ha_gs_init() call.  They can be overridden by specifying the
     * '-i' command-line flag.
     */
    ha_gs_responsiveness_t auto_responsiveness = {HA_GS_PING_RESPONSIVENESS,
                                                  3600,
                                                  10,
                                                  (char *)0,
                                                  0};
    ha_gs_responsiveness_t *responsiveness = 0;

    /*********************************************************************/
    /*
     * These variables represent various data types used to send and/or
     * receive data to/from Group Services.  Refer to the IBM High
     * Availability Services manual for full descriptions.
     */
    ha_gs_socket_ctrl_t     socket_ctrl;
    char *script;
    char *default_deact = "./sample_deactive_c_prog";
    ha_gs_rc_t            rc, ret_val, retVal = HA_GS_NO_INIT;
    static int init_ok = HA_GS_NO_INIT,adapter_info_enabled = HA_GS_NO_INIT;
    ha_gs_proposal_info_t info;
    char                  input = '\0'; /* BEAM FIX: uninitialized */
	char *function,  _newline;   
    ha_gs_token_t         group_token;
    ha_gs_vote_value_t    vote_value, def_vote;
    ha_gs_token_t         gToken;
    int                   gIndex;
    ha_gs_domain_control_response_t  quorum_response;
    ha_gs_time_limit_t  new_domaincb_ack_time_limit;
#ifdef _USE_DEDICATED_THREAD_MODEL  /* begin _USE_DEDICATED_THREAD_MODEL */
    ha_gs_tokenset_id_t  ts1, ts2, ts3, ts4, ts5, ts6, non_existant_tokenset;
    int fd1[2], fd2[2], fd3[2];
    ha_gs_tokenset_membership_t tokenset_member_tokens;
    ha_gs_token_t *temp;
    ha_gs_tokenset_id_t *tokensets_1, *tokensets_2;
    int index, num_tokensets_1 = 0, num_tokensets_2 = 0;
#endif /* end _USE_DEDICATED_THREAD_MODEL */
    

    /*********************************************************************/
    /*
     * Miscellaneous variables.
     */
    char  *endptr;
	char *pFilename;
    int    i, found, init_tries, expected_len, handledTimeOut;
    char  *supptest;
#ifdef _THREAD_SAFE                     /* begin _THREAD_SAFE */
    int    mkThreads, numThreads;
#endif                                  /* end _THREAD_SAFE */

    /*********************************************************************/
    /*
     * These variables are used to control the select() system call.  Please
     * refer to the AIX documentation for a full descriptions of select().
     */
    int         select_rc;
    int         highestDescriptor;
    int         howMany;
    int         commandFD;
    fd_set      socketsForSelect;       /* Maintain all registered sockets in mask. */
    fd_set      socketSelectMask;       /* Used for the actual select. */
    struct      timeval nextJob;        /* Wait time for select. */


    /*********************************************************************/
    /* check the environment variable for "DeactivateOnFailure" */
    char *deact_env = 0;
    deact_env = getenv("HA_GS_DEACTIVATE_ON_FAILURE");
    if(deact_env!=0) {
        deact_on_fail_ctrl = HA_GS_DEACTIVATE_ON_FAILURE;
    }

    set_fulldump_signals();

    /*********************************************************************/
    /*
     * Verify command-line arguments, if any.
     */
    int c;
    while( (c=getopt(argc,argv, "vimc")) != EOF)
    {
	switch(c) {
	case 'v':
		verbose = 1;
		break;

	case 'i':
		interactiveInit = 1;
		break;

	case 'm':
		act_as_domain_master = 1;
		break;

	case 'c':
		act_as_critical_client = 1;
		break;

	default:
            printf("Invalid flag [%c] given.\n", c);
	    usage(argv[0]);
            exit(3);
        }
    }

    /*********************************************************************/
    /*
     * Establish defaults for select().
     */
    nextJob.tv_sec = 60;              /* Wait one minute for select. */
    nextJob.tv_usec = 0;
    highestDescriptor = 0;
    commandFD = 0;                      /* stdin, by another name. */

    FD_ZERO(&socketsForSelect);         /* No sockets yet. */
    FD_SET(commandFD, &socketsForSelect); /* Deal with stdin. */

    socket_fd = -1;                     /* No socket connection to Group Services yet. */

    /*********************************************************************/
    /*
     * How many times have we called ha_gs_init().
     */
    init_tries = 0;

    /*********************************************************************/
    /*
     * We assume the name of the program is "<name><n>".  n is optional, or may
     * be any integral value.  If it isn't present, we assume sample_index 1, or we
     * use it as whatever is given.  The index is used to establish instance numbers
     * and flags and other things to distinguish each running sample_test instance
     * from any others.  Also allows you to easily make all sample_tests the same to
     * force errors if desired.
     */

	/* find the program name and index */

	if (NULL == (endptr = getNumericStringSuffix(argv[0], (uint32_t *)&sample_index)))
	{
		sample_index = 0;               /* no index, use 0. */
		endptr = "0";                   /* prefix value. */
	}

	pFilename = getFilename(argv[0]);

    /*
     * Set up prefix value using the index value for use on messages/etc.
     */
    sample_prefix_len = strlen(sample_pp) + strlen(sample_ee) + strlen(pFilename);
    sample_prefix = malloc(1 + sample_prefix_len);
                         
    strcpy(sample_prefix, sample_pp);
    strcat(sample_prefix, pFilename);
    strcat(sample_prefix, sample_ee);

    printf(starz);
    printf("\n");
    printf("My program index is %d and the program name is: %s\n",
           sample_index,
           sample_prefix);
    printf("\n");
    if(deact_on_fail_ctrl) {
	printf("'export HA_GS_DEACTIVATE_ON_FAILURE=1' is requested.\n");
    }
    printf(starz);

#ifdef _THREAD_SAFE                     /* begin _THREAD_SAFE */
    if (MAX_THREADS < sample_index) {
        numThreads = MAX_THREADS;
    } else if (0 == sample_index) {
        numThreads = 1;
    } else {
        numThreads = sample_index;
    }
#endif                                  /* end _THREAD_SAFE */

    init_membership_subscriptions();
    function = "";
    handledTimeOut = 0;

    /*********************************************************************/
    /*
     * Ready to go.  This while() loop is the rest of the program, and will
     * use select() to listen for commands and for messages from Group
     * Services.
     */
    /*********************************************************************/

    rc = HA_GS_OK;
    while (HA_GS_NOT_OK != rc) {
        /*********************************************************************/
        /*
         * Write instructions, or a simple prompt, based on if the '-v'
         * command-line flag was given.
         */
        if (verbose) {
            write_instructions(verbose);
            printf("Command: ");
        } else if ((!handledResponsiveness) &&
                   (!handledTimeOut)) {
            printf("Please enter a command('h' for help): ");
        }
        fflush(stdout);
        handledResponsiveness = 0;
        handledTimeOut = 0;
        /*********************************************************************/

        /*********************************************************************/
        /*
         * Set up and call select().
         */
        /*
         * Load all desired file descriptors into the select mask, then
         * call select.  We use "nextJob" as a timer, to pop out of select()
         * if no input arrives (default time 3600 seconds).
         */
#ifdef __linux__
    	nextJob.tv_sec = 60;              /* Wait one minute for select. */
   		nextJob.tv_usec = 0;
#endif
        memcpy(&socketSelectMask, &socketsForSelect, sizeof(socketsForSelect));
        select_rc = select(highestDescriptor + 1,
                           &socketSelectMask,
                           0,
                           0,
                           &nextJob);

        if (select_rc < 0) {
            /*
             * If rc < 0, then an error occured.  If select was interrupted,
             * then no worries, just start select() over.  Any other error,
             * give up.
             */
            if (errno == EINTR) {
                printf("Got EINTR during the select.\n");
                continue;
            } else {
                perror("Error on select");
                exit(errno);
            }
        } else if (0 < (howMany = NFDS(select_rc))) {
            /*
             * Input has arrived on one or more of our file descriptors.
             * We normally expect to have 2 -- stdin and the socket to
             * Group Services.
             */
            if (2 < howMany) {
                printf("Input on more than our two sockets??  Have [%d]!\n",
                       howMany);
                exit(howMany);
            }
            if (FD_ISSET(commandFD, &socketSelectMask)) {
                /* stdin */
                scanf("%c",&input);
            }
            if (-1 != socket_fd) {
                if (FD_ISSET(socket_fd, &socketSelectMask)) {
                    input = 'd';        /* do ha_gs_dispatch. */
                }
            }
        } else {
            /*
             * Select() simply timed out.  Display something to show
             * that the program is alive, then just restart select().
             */
            if (verbose) {
                printf("Select timed out, nothing arrived in [%d] seconds.  Try again.\n",
                       nextJob.tv_sec);
            } else {
                printf(".");
                handledTimeOut = 1;
            }
            
            fflush(stdout);
            continue;
        }
        /*
         * End select() handling.
         */
        /*********************************************************************/

        if (!verbose) {
            fflush(stdout);
        }

        /*********************************************************************/
        /*
         * Decipher the command, and take appropriate actions.
         */
        /*********************************************************************/

	switch(input) {

            /*********************************************************************/
            /*
             * Initialize our connection to Group Services, via ha_gs_init().
             */
          case 'i':		
	  case '9':
          case '2':
          case '0':
          /* case '$':*/
            if (interactiveInit) {
                socket_ctrl = build_socket_control();
                if( socket_ctrl & HA_GS_ENABLE_IPV6)
                {
                    ipv6_enabled = 1;
                }
                script = build_deact_script(default_deact);
            } else {
                /* default to enable IPV6 flag */
                socket_ctrl = HA_GS_SOCKET_NO_SIGNAL | HA_GS_ENABLE_ADAPTER_INFO| HA_GS_ENABLE_IPV6;
                ipv6_enabled = 1;
                script = default_deact;
            }
            responsiveness = construct_responsiveness(interactiveInit, &auto_responsiveness);
	    if(act_as_critical_client) {
		printf("Act as a critical client\n");
		responsiveness->gs_responsiveness_type |= HA_GS_CRITICAL_CLIENT;
	    }
            if ('i' == input) {
                rc = ha_gs_init(&socket_fd,
                                socket_ctrl,
                                responsiveness,
                                script,
                                responsive_cb,
                                delayed_error_cb,
                                query_cb);
            } else {
                if (!interactiveInit){
                    socket_ctrl = socket_ctrl | HA_GS_ENABLE_MIGRATION_CALLBACK | HA_GS_ENABLE_DOMAIN_EVENT;

                    if ('0' == input) {
                        socket_ctrl = socket_ctrl | HA_GS_ENABLE_OPQUORUM_CALLBACK;
                        printf("Client registered for asynchronous quorum notification.\n");
                    } else if ('2' == input) {
                        socket_ctrl = socket_ctrl | HA_GS_ENABLE_OPQUORUM_CALLBACK | HA_GS_ENABLE_OPQUORUM_ACK_REQUIRED;
                        printf("Client registered for synchronous quorum notification.\n");
#ifdef _USE_DEDICATED_THREAD_MODEL  /* begin _USE_DEDICATED_THREAD_MODEL */
                    } else if ('$' == input) {
                        socket_ctrl = socket_ctrl | HA_GS_ENABLE_OPQUORUM_CALLBACK | HA_GS_ENABLE_OPQUORUM_ACK_REQUIRED | HA_GS_ENABLE_TOKENSET_DISPATCH;
                        printf("Client enabling the dedicated dispatch thread support model.\n");
#endif /* end _USE_DEDICATED_THREAD_MODEL */
                    }
                }
                rc = ha_gs_setup(HA_GS_RELEASE,
                                8,
                                &socket_fd,
                                socket_ctrl,
                                responsiveness,
                                script,
                                responsive_cb,
                                delayed_error_cb,
                                query_cb,
                                domain_control_cb);
            }

	    function = "ha_gs_init";
            init_tries++;
#ifdef _THREAD_SAFE                     /* begin _THREAD_SAFE */
            /*
             * If desired, create our "dispatch" thread(s) at this time, and
             * simply launch into into ha_gs_dispatch(), there to await
             * messages.  The primary thread continues to monito the
             * user input.
             */
            if (rc == HA_GS_OK) {
#ifdef _USE_DEDICATED_THREAD_MODEL  /* begin _USE_DEDICATED_THREAD_MODEL */
                if(socket_ctrl & HA_GS_ENABLE_TOKENSET_DISPATCH) {

                   /* Creating a catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_create_catchall_tokenset(HA_GS_DISPATCH_TOKENSET, &ts1))){
                       printf("Created a catchall tokenset successfully, with ID: %llu.\n", ts1);
                   } else {
                       printf("Failed to create the catchall tokenset. rc = %d\n", ret_val);
                   }

                   /* Getting the pipe descriptors of the catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_get_dispatch_tokenset_descriptor(ts1, fd1))){
                       printf("Catchall tokenset descriptors: read fd: %d, write fd: %d.\n", fd1[0], fd1[1]);
                   } else {
                       printf("Failed to get the tokenset descriptors of the tokenset %llu. rc = %d.", ts1, ret_val);
                   }

                   /* Creating a non-catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_create_tokenset(HA_GS_DISPATCH_TOKENSET, &ts2))){
                       printf("Created a non-catchall tokenset successfully, with ID: %llu.\n", ts2);
                   } else {
                       printf("Failed to create the non-catchall tokenset. rc = %d\n", ret_val);
                   }

                   /* Getting the pipe descriptors of the non-catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_get_dispatch_tokenset_descriptor(ts2, fd2))){
                       printf("Non-catchall tokenset descriptors: read fd: %d, write fd: %d.\n", fd2[0], fd2[1]);
                   } else {
                       printf("Failed to get the tokenset descriptors of the tokenset %llu. rc = %d.", ts1, ret_val);
                   }

                   /* Getting the pipe descriptors of non-existant tokenset */
                   non_existant_tokenset = 100;
                   if(HA_GS_OK == (ret_val = ha_gs_get_dispatch_tokenset_descriptor(non_existant_tokenset, fd3))){
                       printf("Non-catchall tokenset descriptors: read fd: %d, write fd: %d.\n", fd3[0], fd3[1]);
                   } else {
                       printf("Failed to get the tokenset descriptors of the tokenset %llu. rc = %d.\n", non_existant_tokenset, ret_val);
                   }

                   /* Adding a token to a non-existant tokenset */
                   non_existant_tokenset = 100;
                   if(HA_GS_OK == (ret_val = ha_gs_add_token(non_existant_tokenset, HA_GS_RESPONSIVENESS_MSG_TOKEN))){
                       printf("Successfully added token to tokenset %llu\n", non_existant_tokenset);
                   } else {
                       printf("Failed to add token to tokenset %llu. rc = %d\n", non_existant_tokenset, ret_val);
                   }

                   /* Adding a token to the catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_add_token(ts1, HA_GS_RESPONSIVENESS_MSG_TOKEN))){
                       printf("Successfully added token to tokenset %llu\n", ts1);
                   } else {
                       printf("Failed to add token to tokenset %llu. rc = %d\n", ts1, ret_val);
                   }

                   /* Adding a token to the non-catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_add_token(ts2, HA_GS_RESPONSIVENESS_MSG_TOKEN))){
                       printf("Successfully added token to tokenset %llu\n", ts2);
                   } else {
                       printf("Failed to add token to tokenset %llu. rc = %d\n", ts2, ret_val);
                   }

                   /* Adding a token twice to the non-catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_add_token(ts2, HA_GS_RESPONSIVENESS_MSG_TOKEN))){
                       printf("Successfully added token to tokenset %llu\n", ts2);
                   } else {
                       printf("Failed to add token to tokenset %llu. rc = %d\n", ts2, ret_val);
                   }

                   /* Adding a token to the non-catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_add_token(ts2, HA_GS_QUERY_MSG_TOKEN))){
                       printf("Successfully added token to tokenset %llu\n", ts2);
                   } else {
                       printf("Failed to add token to tokenset %llu. rc = %d\n", ts2, ret_val);
                   }

                   /* Adding a token to the non-catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_add_token(ts2, HA_GS_DOMAINCONTROL_OPQUORUM_INFO_MSG_TOKEN))){
                       printf("Successfully added token to tokenset %llu\n", ts2);
                   } else {
                       printf("Failed to add token to tokenset %llu. rc = %d\n", ts2, ret_val);
                   }

                   /* Getting the tokenset members of a non-existant tokenset */
                   non_existant_tokenset = 100;
                   if(HA_GS_OK == (ret_val = ha_gs_get_tokenset_members(non_existant_tokenset, &tokenset_member_tokens))){
                       printf("Successfully got member tokens for tokenset %llu\n", non_existant_tokenset);
                       temp = tokenset_member_tokens.token_list;
                       for(index=0; index<tokenset_member_tokens.num_tokens; index++){
                           printf("Member token %d : %d\n", index, temp[index]);
                       }
                       free(temp);
                       temp = tokenset_member_tokens.token_list = NULL;
                   } else {
                       printf("Failed to get member tokens for tokenset %llu. rc = %d\n", non_existant_tokenset, ret_val);
                   }

                   /* Getting the tokenset members of a catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_get_tokenset_members(ts1, &tokenset_member_tokens))){
                       printf("Successfully got member tokens for tokenset %llu\n", ts1);
                       temp = tokenset_member_tokens.token_list;
                       for(index=0; index<tokenset_member_tokens.num_tokens; index++){
                           printf("Member token %d : %d\n", index, temp[index]);
                       }
                       free(temp);
                       temp = tokenset_member_tokens.token_list = NULL;
                   } else {
                       printf("Failed to get member tokens for tokenset %llu. rc = %d\n", ts1, ret_val);
                   }

                   /* Getting the tokenset members of a non-catchall tokenset */
                   if(HA_GS_OK == ha_gs_get_tokenset_members(ts2, &tokenset_member_tokens)){
                       printf("Successfully got member tokens for tokenset %llu\n", ts2);
                       temp = tokenset_member_tokens.token_list;
                       for(index=0; index<tokenset_member_tokens.num_tokens; index++){
                           printf("Member token %d : %d\n", index, temp[index]);
                       }
                       free(temp);
                       temp = tokenset_member_tokens.token_list = NULL;
                   } else {
                       printf("Failed to get member tokens for tokenset %llu. rc = %d\n", ts2, ret_val);
                   }

                   /* Getting a token's tokensets. (The token is part of a non-catchall tokenset) */
                   ret_val = ha_gs_get_token_tokensets(HA_GS_QUERY_MSG_TOKEN, &tokensets_1, &num_tokensets_1);
                   if((ret_val == HA_GS_MEMBER_OF_CATCHALL_TOKENSET) || (ret_val == HA_GS_OK)){
                       printf("Successfully got token's tokensets\n");
                       if(ret_val == HA_GS_MEMBER_OF_CATCHALL_TOKENSET){
                           printf("The token is part of catchall tokensets\n");
                       }
                       for(index=0; index<num_tokensets_1; index++){
                           printf("tokenset %d : %d\n", index, tokensets_1[index]);
                       }
                       free(tokensets_1);     
                       tokensets_1 = NULL;
                   } else {
                       printf("Failed to get the token's tokensets. rc = %d.\n", ret_val);
                   }

                   /* Creating a non-catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_create_tokenset(HA_GS_DISPATCH_TOKENSET, &ts3))){
                       printf("Created a non-catchall tokenset successfully, with ID: %llu.\n", ts3);
                   } else {
                       printf("Failed to create the non-catchall tokenset. rc = %d\n", ret_val);
                   }

                   /* Adding existing token to the non-catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_add_token(ts3, HA_GS_RESPONSIVENESS_MSG_TOKEN))){
                       printf("Successfully added token to tokenset %llu\n", ts3);
                   } else {
                       printf("Failed to add token to tokenset %llu. rc = %d\n", ts3, ret_val);
                   }

                   /* Removing a token from a non-existant tokenset */
                   non_existant_tokenset = 100;
                   if(HA_GS_OK == (ret_val = ha_gs_remove_token(non_existant_tokenset, HA_GS_DOMAINCONTROL_OPQUORUM_INFO_MSG_TOKEN))){
                       printf("Successfully removed token from tokenset %llu\n", non_existant_tokenset);
                   } else {
                       printf("Failed to remove token from tokenset %llu. rc = %d\n", non_existant_tokenset, ret_val);
                   }

                   /* Removing a token from a catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_remove_token(ts1, HA_GS_DOMAINCONTROL_OPQUORUM_INFO_MSG_TOKEN))){
                       printf("Successfully removed token from tokenset %llu\n", ts1);
                   } else {
                       printf("Failed to remove token from tokenset %llu. rc = %d\n", ts1, ret_val);
                   }

                   /* Removing a token from a non-catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_remove_token(ts2, HA_GS_DOMAINCONTROL_OPQUORUM_INFO_MSG_TOKEN))){
                       printf("Successfully removed token from tokenset %llu\n", ts2);
                   } else {
                       printf("Failed to remove token from tokenset %llu. rc = %d\n", ts2, ret_val);
                   }

                   /* Removing a token twice from a non-catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_remove_token(ts2, HA_GS_DOMAINCONTROL_OPQUORUM_INFO_MSG_TOKEN))){
                       printf("Successfully removed token from tokenset %llu\n", ts2);
                   } else {
                       printf("Failed to remove token from tokenset %llu. rc = %d\n", ts2, ret_val);
                   }

                   /* Getting the member tokens of a non-catchall tokenset, after token removal */
                   if(HA_GS_OK == (ret_val = ha_gs_get_tokenset_members(ts2, &tokenset_member_tokens))){
                       printf("Successfully got member tokens for tokenset %llu\n", ts2);
                       temp = tokenset_member_tokens.token_list;
                       for(index=0; index<tokenset_member_tokens.num_tokens; index++){
                           printf("Member token %d : %d\n", index, temp[index]);
                       }
                       free(temp);
                       temp = tokenset_member_tokens.token_list = NULL;
                   } else {
                       printf("Failed to get member tokens for tokenset %llu. rc = %d\n", ts2, ret_val);
                   }

                   /* Creating a catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_create_catchall_tokenset(HA_GS_DISPATCH_TOKENSET, &ts4))){
                       printf("Created a catchall tokenset successfully, with ID: %llu.\n", ts4);
                   } else {
                       printf("Failed to create the catchall tokenset. rc = %d\n", ret_val);
                   }

                   /* Getting a token's tokensets. (The token is not part of any non-catchall tokenset) */
                   ret_val = ha_gs_get_token_tokensets(HA_GS_DOMAINCONTROL_DOMAIN_MERGE_MSG_TOKEN, &tokensets_2, &num_tokensets_2);
                   if((ret_val == HA_GS_MEMBER_OF_CATCHALL_TOKENSET) || (ret_val == HA_GS_OK)){
                       printf("Successfully got token's tokensets\n");
                       if(ret_val == HA_GS_MEMBER_OF_CATCHALL_TOKENSET){
                           printf("The token is part of catchall tokensets\n");
                       }
                       for(index=0; index<num_tokensets_2; index++){
                           printf("tokenset %d : %d\n", index, tokensets_2[index]);
                       }
                       free(tokensets_2);
                       tokensets_2 = NULL;
                   } else {
                       printf("Failed to get the token's tokensets. rc = %d.\n", ret_val);
                   }

                   /* Creating a catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_create_catchall_tokenset(HA_GS_DISPATCH_TOKENSET, &ts5))){
                       printf("Created a catchall tokenset successfully, with ID: %llu.\n", ts5);
                   } else {
                       printf("Failed to create the catchall tokenset. rc = %d\n", ret_val);
                   }

                  /* Creating a non-catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_create_tokenset(HA_GS_DISPATCH_TOKENSET, &ts6))){
                       printf("Created a non-catchall tokenset successfully, with ID: %llu.\n", ts6);
                   } else {
                       printf("Failed to create the non-catchall tokenset. rc = %d\n", ret_val);
                   }

                   /* Deleting a non-existant tokenset */
                   non_existant_tokenset = 100;
                   if(HA_GS_OK == (ret_val = ha_gs_delete_tokenset(non_existant_tokenset))){
                      printf("Successfully deleted tokenset, %llu.\n", non_existant_tokenset);
                   } else {
                      printf("Failed to delete tokenset, %llu. rc = %d\n", non_existant_tokenset, ret_val);
                   }

                   /* Deleting a catch-all tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_delete_tokenset(ts1))){
                      printf("Successfully deleted tokenset, %llu.\n", ts1);
                   } else {
                      printf("Failed to delete tokenset, %llu. rc = %d\n", ts1, ret_val);
                   }

                   /* Deleting a non-empty, non-catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_delete_tokenset(ts2))){
                      printf("Successfully deleted tokenset, %llu.\n", ts2);
                   } else {
                      printf("Failed to delete tokenset, %llu. rc = %d\n", ts2, ret_val);
                   }

                   /* Deleting an empty, non-catchall tokenset */
                   if(HA_GS_OK == (ret_val = ha_gs_delete_tokenset(ts3))){
                      printf("Successfully deleted tokenset, %llu.\n", ts3);
                   } else {
                      printf("Failed to delete tokenset, %llu. rc = %d\n", ts3, ret_val);
                   }

                } else {
#endif /* end _USE_DEDICATED_THREAD_MODEL */
                   if( socket_ctrl == HA_GS_SOCKET_NO_SIGNAL | HA_GS_ENABLE_ADAPTER_INFO ){
                       adapter_info_enabled = HA_GS_OK;
                   }
                   /* feature 150043: IPv6 support */
                   if( socket_ctrl & HA_GS_ENABLE_IPV6 ){
                       ipv6_enabled = 1;
                   }
                   init_ok = HA_GS_OK;
                   fprintf(stderr, "Attempt to start %d dispatch thread%s\n",
                           numThreads,
                          ((1 == numThreads) ? "." : "s."));
                   for (mkThreads = 0;
                        mkThreads < numThreads;
                        mkThreads++) {
                        rc = pthread_create(&dispatch_id[mkThreads],
                                            NULL,
                                            dispatch_thread,
                                            NULL);
                        if (rc) {
                            fprintf(stderr, "pthread_create for thread %d returned %d\n",
                                    mkThreads,
                                    rc);
                            break;
                        }
                   }
#ifdef _USE_DEDICATED_THREAD_MODEL  /* begin _USE_DEDICATED_THREAD_MODEL */
                }
#endif /* end _USE_DEDICATED_THREAD_MODEL */
            } else if (HA_GS_EXISTS == rc) {
                fprintf(stderr,
                        "You have already initialized! Why do it again?\n");
            } else {
                fprintf(stderr,
                        "Bad news - ha_gs_init() returned rc:[%s]",
                        write_an_rc(rc));
                if (init_tries == 5) {
                    fprintf(stderr," - Giving up!\n");
                    assert( rc == HA_GS_OK );
                    exit(rc);
                }
                fprintf(stderr, "\n");
                continue;
            }
            fflush(stdout);
#else                                   /* else _THREAD_SAFE */
            /*
             * If not thread-safe, then add the socket file descriptor
             * returned from ha_gs_init() into our select() mask.
             */
            if (HA_GS_OK == rc) {
#ifdef _USE_DEDICATED_THREAD_MODEL  /* begin _USE_DEDICATED_THREAD_MODEL */
                if(socket_ctrl & HA_GS_ENABLE_TOKENSET_DISPATCH) {
                   if(HA_GS_OK == ha_gs_create_catchall_tokenset(HA_GS_DISPATCH_TOKENSET, &ts1)){
                      printf("Created a catchall tokenset successfully, with ID: %llu.\n", ts1);
                   }
                } else {
#endif /* end _USE_DEDICATED_THREAD_MODEL */
                  FD_SET(socket_fd, &socketsForSelect);
                  if (socket_fd > highestDescriptor) {
                      highestDescriptor = socket_fd;
                  }
#ifdef _USE_DEDICATED_THREAD_MODEL  /* begin _USE_DEDICATED_THREAD_MODEL */
                }
#endif /* end _USE_DEDICATED_THREAD_MODEL */
            } else if (HA_GS_EXISTS == rc) {
                printf("You have already initialized!  Why do it again?\n");
            } else {
                printf("Bad news - ha_gs_init() returned rc:[%s]",
                       write_an_rc(rc));
                if (5 == init_tries) {
                    printf(" - Giving up!\n");
                    exit(rc);
                }
                printf("\n");
                continue;
            }
            fflush(stdout);
#endif                                  /* end _THREAD_SAFE */
	    break;
            /*
             * Finished with initialization ('i') handling.
             */

            /*********************************************************************/
            /*
             * Display help data.
             */
          case 'h':
          case 'H':
            write_instructions(verbose);
            function = "Write instructions";
            rc = HA_GS_OK;
            break;


            /*********************************************************************/
            /*
             * toggle HA_GS_DEACTIVATE_ON_FAILURE
             */
          case 'z':
          case 'Z':
		if(deact_on_fail_ctrl) {
			deact_on_fail_ctrl = 0;
			printf("HA_GS_DEACTIVATE_ON_FAILURE=0\n"); 
			fflush(stdout);
		} else {
			deact_on_fail_ctrl = HA_GS_DEACTIVATE_ON_FAILURE;
			printf("HA_GS_DEACTIVATE_ON_FAILURE=1\n"); 
			fflush(stdout);
		}
            rc = HA_GS_OK;
            break;

	    case '4':
		/* change the responsiveness */
		responsiveness = construct_responsiveness(1, &auto_responsiveness);
		rc = ha_gs_change_responsiveness(responsiveness);
		fprintf(stderr,
                        "ha_gs_change_responsiveness returned rc:[%s]",
                        write_an_rc(rc));
		rc = HA_GS_OK;
		function = "ha_gs_change_responsiveness";
		break;

	    case '5':
		/* domain dissolve control */
		if(!act_as_domain_master) {
			printf("Unrecognized command: %c\n", input);
			fflush(stdout);
		} else {
			int		    node_number = -1;
			ha_gs_domain_spec_t domain;
			ha_gs_get_node_number(&node_number);
			domain.node_number = node_number;
			rc = ha_gs_dissolve_domain(&domain);
			fprintf(stderr, "ha_gs_dissolve_domain returned rc:[%s]",
				write_an_rc(rc));
			rc = HA_GS_OK;
			
		}
		function = "ha_gs_dissolve_domain";
		break;

           case '@':
                /* site dissolve control */
                if(!act_as_domain_master) {
                        printf("Unrecognized command: %c\n", input);
                        fflush(stdout);
                } else {
                        ha_gs_site_spec_t site;
                        ha_gs_site_dissolve_action_t action = HA_GS_SITE_DISSOLVE_AND_DEFAULT;
                        site.site_number = -1;
			
                        rc = ha_gs_dissolve_site(&site, action);
                        fprintf(stderr, "ha_gs_dissolve_sitereturned rc:[%s]",
                                write_an_rc(rc));
                        rc = HA_GS_OK;

                }
                function = "ha_gs_dissolve_site";


		break;


            /*********************************************************************/
            /*
             * Attempt to join "theSourceGroup".  Fill in the group attributes,
             * then call ha_gs_join().  For this, and all of the other joins, the
             * synchronous return code will tell us simply that Group Services has
             * accepted the join request, we will have to wait for the callback
             * functions to execute to determine the ultimate success of our join
             * request.
             *
             * Note that here we encode the group attributes within the program
             * itself.  Another method would be to define the attributes in a
             * separate "configuration" file (or other external repository) and
             * have the program read them in.  This would allow the attributes
             * to be modified without recompiling the program.
             */
          case 'j':
          case 'J':
            voting_phase = 1;
	    function = "ha_gs_join so far";
            gattr[0] = malloc(sizeof(ha_gs_group_attributes_t));
	    gattr[0]->gs_version = 1;
	    gattr[0]->gs_sizeof_group_attributes = sizeof(ha_gs_group_attributes_t);
	    gattr[0]->gs_client_version = 1;
	    gattr[0]->gs_batch_control = HA_GS_BATCH_BOTH | deact_on_fail_ctrl;
	    gattr[0]->gs_num_phases = HA_GS_N_PHASE;
	    gattr[0]->gs_source_reflection_num_phases = HA_GS_N_PHASE;
	    gattr[0]->gs_group_default_vote = HA_GS_VOTE_REJECT;
	    gattr[0]->gs_merge_control = HA_GS_DONTCARE_MERGE;
	    gattr[0]->gs_time_limit = 60;
	    gattr[0]->gs_source_reflection_time_limit = 30;
	    gattr[0]->gs_group_name = group_names[0];
	    gattr[0]->gs_source_group_name = source_group_names[0];
            instance_numbers[0] += sample_index;

            /*
             * Display the attributes on stdout.
             */
            write_join_information(0);
	    
	    info.gs_join_request.gs_group_attributes = gattr[0];
            info.gs_join_request.gs_provider_instance = instance_numbers[0];
	    info.gs_join_request.gs_provider_local_name = prov_local_names[0];
	    info.gs_join_request.gs_n_phase_protocol_callback = 
		n_phase_cb0;
	    info.gs_join_request.gs_protocol_approved_callback = 
		approved_cb0;
	    info.gs_join_request.gs_protocol_rejected_callback = 
		rejected_cb0;
	    info.gs_join_request.gs_announcement_callback = 
		announce_cb0;
	    info.gs_join_request.gs_merge_callback =
		merge_cb0;
	    rc = ha_gs_join(&gid[0], &info);
	    break;
            /*
             * Finished with the 'j' command.
             */

            /*********************************************************************/
            /*
             * Attempt to join the group "OnePhaseJoin".
             */
          case '1':
	    function = "ha_gs_join so far";
            gattr[1] = malloc(sizeof(ha_gs_group_attributes_t));
	    gattr[1]->gs_version = 1;
	    gattr[1]->gs_sizeof_group_attributes = sizeof(ha_gs_group_attributes_t);
	    gattr[1]->gs_client_version = 1;
	    gattr[1]->gs_batch_control = HA_GS_BATCH_BOTH | deact_on_fail_ctrl;
	    gattr[1]->gs_num_phases = HA_GS_1_PHASE;
	    gattr[1]->gs_source_reflection_num_phases = HA_GS_1_PHASE;
	    gattr[1]->gs_group_default_vote = HA_GS_VOTE_REJECT;
	    gattr[1]->gs_merge_control = HA_GS_DONTCARE_MERGE;
	    gattr[1]->gs_time_limit = 0;
	    gattr[1]->gs_source_reflection_time_limit = 0;
	    gattr[1]->gs_group_name = group_names[1];
	    gattr[1]->gs_source_group_name = source_group_names[1];
            instance_numbers[1] += sample_index;

            write_join_information(1);
	    
	    info.gs_join_request.gs_group_attributes = gattr[1];
            info.gs_join_request.gs_provider_instance = instance_numbers[1];
	    info.gs_join_request.gs_provider_local_name = prov_local_names[1];
	    info.gs_join_request.gs_n_phase_protocol_callback = 
		n_phase_cb1;
	    info.gs_join_request.gs_protocol_approved_callback = 
		approved_cb1;
	    info.gs_join_request.gs_protocol_rejected_callback = 
		rejected_cb1;
	    info.gs_join_request.gs_announcement_callback = 
		announce_cb1;
	    info.gs_join_request.gs_merge_callback =
		merge_cb1;
	    rc = ha_gs_join(&gid[1], &info);
	    break;
            /*
             * Finished with the '1' command.
             */

            /*********************************************************************/
            /*
             * Attempt to join the group "theTargetGroup".
             */
          case 'X':
          case 'x':
            voting_phase = 1;
	    function = "ha_gs_join so far";
            gattr[2] = malloc(sizeof(ha_gs_group_attributes_t));
	    gattr[2]->gs_version = 1;
	    gattr[2]->gs_sizeof_group_attributes = sizeof(ha_gs_group_attributes_t);
	    gattr[2]->gs_client_version = 1;
	    gattr[2]->gs_batch_control = HA_GS_BATCH_BOTH | deact_on_fail_ctrl;
	    gattr[2]->gs_num_phases = HA_GS_N_PHASE;
	    gattr[2]->gs_source_reflection_num_phases = HA_GS_N_PHASE;
	    gattr[2]->gs_group_default_vote = HA_GS_VOTE_REJECT;
	    gattr[2]->gs_merge_control = HA_GS_DONTCARE_MERGE;
	    gattr[2]->gs_time_limit = 60;
	    gattr[2]->gs_source_reflection_time_limit = 30;
	    gattr[2]->gs_group_name = group_names[2];
	    gattr[2]->gs_source_group_name = source_group_names[2];
            instance_numbers[2] += sample_index;

            write_join_information(2);
	    
	    info.gs_join_request.gs_group_attributes = gattr[2];
            info.gs_join_request.gs_provider_instance = instance_numbers[2];
	    info.gs_join_request.gs_provider_local_name = prov_local_names[2];
	    info.gs_join_request.gs_n_phase_protocol_callback = 
		n_phase_cb2;
	    info.gs_join_request.gs_protocol_approved_callback = 
		approved_cb2;
	    info.gs_join_request.gs_protocol_rejected_callback = 
		rejected_cb2;
	    info.gs_join_request.gs_announcement_callback = 
		announce_cb2;
	    info.gs_join_request.gs_merge_callback =
		merge_cb2;
	    rc = ha_gs_join(&gid[2], &info);
	    break;
            /*
             * Finished with the 'x' command.
             */

            /*********************************************************************/
            /*
             * Attempt to join the group "theLonelyGroup".
             */
          case 'm':
            voting_phase = 1;
	    function = "ha_gs_join so far";
            gattr[3] = malloc(sizeof(ha_gs_group_attributes_t));
	    gattr[3]->gs_version = 1;
	    gattr[3]->gs_sizeof_group_attributes = sizeof(ha_gs_group_attributes_t);
	    gattr[3]->gs_client_version = 1;
	    gattr[3]->gs_batch_control = HA_GS_BATCH_BOTH | deact_on_fail_ctrl;
	    gattr[3]->gs_num_phases = HA_GS_N_PHASE;
	    gattr[3]->gs_source_reflection_num_phases = HA_GS_N_PHASE;
	    gattr[3]->gs_group_default_vote = HA_GS_VOTE_REJECT;
	    gattr[3]->gs_merge_control = HA_GS_DONTCARE_MERGE;
	    gattr[3]->gs_time_limit = 60;
	    gattr[3]->gs_source_reflection_time_limit = 30;
	    gattr[3]->gs_group_name = group_names[3];
	    gattr[3]->gs_source_group_name = source_group_names[3];
            instance_numbers[3] += sample_index;

            write_join_information(3);
	    
	    info.gs_join_request.gs_group_attributes = gattr[3];
            info.gs_join_request.gs_provider_instance = instance_numbers[3];
	    info.gs_join_request.gs_provider_local_name = prov_local_names[3];
	    info.gs_join_request.gs_n_phase_protocol_callback = 
		n_phase_cb3;
	    info.gs_join_request.gs_protocol_approved_callback = 
		approved_cb3;
	    info.gs_join_request.gs_protocol_rejected_callback = 
		rejected_cb3;
	    info.gs_join_request.gs_announcement_callback = 
		announce_cb3;
	    info.gs_join_request.gs_merge_callback =
		merge_cb3;
	    rc = ha_gs_join(&gid[3], &info);
	    break;
            /*
             * Finished with the 'm' command.
             */

            /*********************************************************************/
            /*
             * Attempt to join the group "ifFirstIWin".
             */
          case 'A':
          case 'a':
            voting_phase = 1;
	    function = "ha_gs_join so far";
            gattr[4] = malloc(sizeof(ha_gs_group_attributes_t));
	    gattr[4]->gs_version = 1;
	    gattr[4]->gs_sizeof_group_attributes = sizeof(ha_gs_group_attributes_t);
	    gattr[4]->gs_client_version = 1;
	    gattr[4]->gs_batch_control = HA_GS_BATCH_BOTH | deact_on_fail_ctrl;
            if (0 == (sample_index % 2)) {
                gattr[4]->gs_num_phases = HA_GS_1_PHASE;
            } else {
                gattr[4]->gs_num_phases = HA_GS_N_PHASE;
            }
	    gattr[4]->gs_source_reflection_num_phases = HA_GS_N_PHASE;
	    gattr[4]->gs_group_default_vote = HA_GS_VOTE_REJECT;
	    gattr[4]->gs_merge_control = HA_GS_DONTCARE_MERGE;
	    gattr[4]->gs_time_limit = 60;
	    gattr[4]->gs_source_reflection_time_limit = 30;
	    gattr[4]->gs_group_name = group_names[4];
	    gattr[4]->gs_source_group_name = source_group_names[4];
            instance_numbers[4] += sample_index;

            write_join_information(4);
	    
	    info.gs_join_request.gs_group_attributes = gattr[4];
            info.gs_join_request.gs_provider_instance = instance_numbers[4];
	    info.gs_join_request.gs_provider_local_name = prov_local_names[4];
	    info.gs_join_request.gs_n_phase_protocol_callback = 
		n_phase_cb4;
	    info.gs_join_request.gs_protocol_approved_callback = 
		approved_cb4;
	    info.gs_join_request.gs_protocol_rejected_callback = 
		rejected_cb4;
	    info.gs_join_request.gs_announcement_callback = 
		announce_cb4;
	    info.gs_join_request.gs_merge_callback =
		merge_cb4;
	    rc = ha_gs_join(&gid[4], &info);
	    break;
            /*
             * Finished with the 'a' command.
             */

            /*********************************************************************/
            /*
             * Attempt to join the group "SourceOrNot".
             */
          case 'G':
          case 'g':
            voting_phase = 1;
	    function = "ha_gs_join so far";
            gattr[5] = malloc(sizeof(ha_gs_group_attributes_t));
	    gattr[5]->gs_version = 1;
	    gattr[5]->gs_sizeof_group_attributes = sizeof(ha_gs_group_attributes_t);
	    gattr[5]->gs_client_version = 1;
	    gattr[5]->gs_batch_control = HA_GS_BATCH_BOTH | deact_on_fail_ctrl;
            gattr[5]->gs_num_phases = HA_GS_N_PHASE;
	    gattr[5]->gs_source_reflection_num_phases = HA_GS_N_PHASE;
	    gattr[5]->gs_group_default_vote = HA_GS_VOTE_REJECT;
	    gattr[5]->gs_merge_control = HA_GS_DONTCARE_MERGE;
	    gattr[5]->gs_time_limit = 60;
	    gattr[5]->gs_source_reflection_time_limit = 30;
	    gattr[5]->gs_group_name = group_names[5];
            if (0 == (sample_index % 2)) {
                gattr[5]->gs_source_group_name = source_group_names[5];
            } else {
                gattr[5]->gs_source_group_name = "";
            }
            instance_numbers[5] += sample_index;

            write_join_information(5);
	    
	    info.gs_join_request.gs_group_attributes = gattr[5];
            info.gs_join_request.gs_provider_instance = instance_numbers[5];
	    info.gs_join_request.gs_provider_local_name = prov_local_names[5];
	    info.gs_join_request.gs_n_phase_protocol_callback = 
		n_phase_cb5;
	    info.gs_join_request.gs_protocol_approved_callback = 
		approved_cb5;
	    info.gs_join_request.gs_protocol_rejected_callback = 
		rejected_cb5;
	    info.gs_join_request.gs_announcement_callback = 
		announce_cb5;
	    info.gs_join_request.gs_merge_callback =
		merge_cb5;
	    rc = ha_gs_join(&gid[5], &info);
	    break;
            /*
             * Finished with the 'g' command.
             */

            /*********************************************************************/
            /*
             * Attempt to join the group "ChainGang".
             */
          case '3':
            voting_phase = 1;
	    function = "ha_gs_join so far";
            gattr[6] = malloc(sizeof(ha_gs_group_attributes_t));
	    gattr[6]->gs_version = 1;
	    gattr[6]->gs_sizeof_group_attributes = sizeof(ha_gs_group_attributes_t);
	    gattr[6]->gs_client_version = 3;
	    gattr[6]->gs_batch_control = HA_GS_BATCH_JOINS | deact_on_fail_ctrl;
            gattr[6]->gs_num_phases = HA_GS_N_PHASE;
	    gattr[6]->gs_source_reflection_num_phases = HA_GS_1_PHASE;
	    gattr[6]->gs_group_default_vote = HA_GS_VOTE_APPROVE;
	    gattr[6]->gs_merge_control = HA_GS_DONTCARE_MERGE;
	    gattr[6]->gs_time_limit = 60;
	    gattr[6]->gs_source_reflection_time_limit = 0;
	    gattr[6]->gs_group_name = group_names[6];
            gattr[6]->gs_source_group_name = source_group_names[6];
            instance_numbers[6] += sample_index;

            write_join_information(6);
	    
	    info.gs_join_request.gs_group_attributes = gattr[6];
            info.gs_join_request.gs_provider_instance = instance_numbers[6];
	    info.gs_join_request.gs_provider_local_name = prov_local_names[6];
	    info.gs_join_request.gs_n_phase_protocol_callback = 
		n_phase_cb6;
	    info.gs_join_request.gs_protocol_approved_callback = 
		approved_cb6;
	    info.gs_join_request.gs_protocol_rejected_callback = 
		rejected_cb6;
	    info.gs_join_request.gs_announcement_callback = 
		announce_cb6;
	    info.gs_join_request.gs_merge_callback =
		merge_cb6;
	    rc = ha_gs_join(&gid[6], &info);
	    break;
            /*
             * Finished with the '3' command.
             */

            /*********************************************************************/
            /*
             * Have the user specify the attributes to use for this group.  If
             * any problems occur, then we do not call ha_gs_join().
             */
          case 'b':
          case 'B':
            voting_phase = 1;
	    function = "ha_gs_join so far";

            instance_numbers[7] += sample_index;
            if (NULL == (gattr[7] = build_group_attributes(7, 0))) {
                printf("Not joining, could not build the group's attributes!\n");
                fflush(stdout);
                break;
            }

            write_join_information(7);
	    
	    info.gs_join_request.gs_group_attributes = gattr[7];
            info.gs_join_request.gs_provider_instance = instance_numbers[7];
	    info.gs_join_request.gs_provider_local_name = prov_local_names[7];
	    info.gs_join_request.gs_n_phase_protocol_callback = 
		n_phase_cb7;
	    info.gs_join_request.gs_protocol_approved_callback = 
		approved_cb7;
	    info.gs_join_request.gs_protocol_rejected_callback = 
		rejected_cb7;
	    info.gs_join_request.gs_announcement_callback = 
		announce_cb7;
	    info.gs_join_request.gs_merge_callback =
		merge_cb7;
	    rc = ha_gs_join(&gid[7], &info);

            break;
            /*
             * Finished with the 'b' command.
             */

            /*********************************************************************/
            /*
             * These proposal requests require that we have already joined one
             * or more groups.  In the case where we have joined only one group,
             * then the program will assume that the protocol request is to be
             * made in that group.  Where we have joined multiple groups, the
             * program will prompt the user to specify to which group the request
             * should be made.
             *
             * In all cases, the user will be prompted for the information to
             * submit for the proposal (number of phases, time limit, data).
             *
             * Note that the synchronous return code of HA_GS_OK merely indicates
             * that Group Services has accepted the proposal, but, does *NOT*
             * guarantee that our proposal will actually execute as a protocol
             * for the group.  It is possible that our proposal may be returned
             * to us via the "delayed_error" callback, if there is a problem with
             * it, or, if it has 'collided' with another proposal.
             */
            /*********************************************************************/

            /*********************************************************************/
            /*
             * Request a group state value change.
             */
          case 't':
          case 'T':
            if (build_state_change(&group_token, &info)) {
                function="ha_gs_change_state_value so far";
                rc = ha_gs_change_state_value(group_token, &info);
                break;
            } else {
                continue;
            }
            /*
             * Finished with the 't' command.
             */

            /*********************************************************************/
            /*
             * Request to send a provider-broadcast message to the group.
             */
          case 'p':
          case 'P':
            if (build_pbm(&group_token, &info)) {
                function="ha_gs_send_message so far";
                rc = ha_gs_send_message(group_token, &info);
                break;
            } else {
                continue;
            }
            /*
             * Finished with the 'p' command.
             */

            /*********************************************************************/
            /*
             * Request to voluntarily leave the group.
             */
          case 'l':                     /* voluntary leave */
          case 'L':
            if (build_leave_request(&group_token, &info)) {
                function="ha_gs_leave so far";
                rc = ha_gs_leave(group_token, &info);
                break;
            } else {
                continue;
            }
            /*
             * Finished with the 'l' command.
             */

            /*********************************************************************/
            /*
             * Request to expel some poor provider.
             */
          case 'e':                     /* expel */
          case 'E':
            if (build_expel_request(&group_token, &info)) {
                function="ha_gs_expel so far";
                rc = ha_gs_expel(group_token, &info);
                break;
            } else {
                continue;
            }
            /*
             * Finished with the 'e' command.
             */

            /*********************************************************************/
            /*
             * Request a group attributes change.
             */
          case 'n':
          case 'N':
            if (build_attributes_change(&group_token, &info)) {
                function="ha_gs_change_attributes so far";
                rc = ha_gs_change_attributes(group_token, &info);
                break;
            } else {
                continue;
            }
            /*
             * Finished with the 'n' command.
             */

            /*********************************************************************/
            /*
             * Tell a group goodbye.
             */
          case 'y':
          case 'Y':
            if (build_goodbye_request(&group_token)) {
                function="ha_gs_goodbye so far";
                if (HA_GS_OK == (rc = ha_gs_goodbye(group_token))) {
                    told_group_goodbye(group_token);
                }
                break;
            } else {
                continue;
            }
            /*
             * Finished with the 'y' command.
             */

            /*********************************************************************/
            /*
             * Subscription.
             *
             * To subscribe to a group, the group must "exist" somewhere in the
             * partition.  This means that at least one provider must have already
             * joined the group, or the subscription will be returned with a
             * delayed_error callback and an error code of HA_GS_UNKNOWN_GROUP.
             * The user will be prompted for the name of the group.
             *
             * To unsubscribe, if only one group has been subscribed to, then
             * the unsubscribe request will be applied to that group.  If multiple
             * groups are subscribed to, then the user will be prompted for which
             * group to unsubscribe.
             */
            /*********************************************************************/

            /*
             * Subscribe to a group.
             */
          case 's':
          case 'S':
            rc = subscribe_to_a_group();
            function = "ha_gs_subscribe so far";
            break;

            /*
             * Unsubscribe from a group.
             */
          case 'u':
          case 'U':
            rc = unsubscribe_from_a_group();
            function = "ha_gs_unsubscribe so far";
            break;
            /*
             * Finished with the subscription ('s','u') commands.
             */

            /*********************************************************************/
            /*
             * Close the socket connecting us to Group Services, and exit the
             * program.
             *
             * If thread-safe, first cancel the thread we had created at
             * initialization.
             */
	  case 'q':
          case 'Q':
	  case 'w':
	  case 'W':

	    /* quit first */
	    (void) ha_gs_quit();
            highestDescriptor = 0;
            socket_fd = -1;

#ifdef _THREAD_SAFE                     /* begin _THREAD_SAFE */
            for (mkThreads = 0;
                 mkThreads < numThreads;
                 mkThreads++) {
                pthread_cancel(dispatch_id[mkThreads]);
            }

	    /* wait the threads */
            for (mkThreads = 0;
                 mkThreads < numThreads;
                 mkThreads++) {
                pthread_join(dispatch_id[mkThreads], NULL);
            }

#endif                                  /* end _THREAD_SAFE */

	    if(input == 'q' || input == 'Q') {
		/* exit the process */
	    	exit(0);
	    }

            function = "ha_gs_quit() so far";
	    break;
            /*
             * Finished with the 'q' command.
             */

            /*********************************************************************/
            /*
             * Call ha_gs_dispatch().  If this is the thread-safe version, then
             * this command is a nop.
             */
	  case 'd':
#ifndef _THREAD_SAFE                    /* begin !_THREAD_SAFE */
#ifdef _USE_DEDICATED_THREAD_MODEL  /* begin _USE_DEDICATED_THREAD_MODEL */
            if (socket_ctrl & HA_GS_ENABLE_TOKENSET_DISPATCH) {
                rc = ha_gs_dispatch_tokenset(HA_GS_NON_BLOCKING, 0);
                function="ha_gs_dispatch_tokenset";
            } else {
#endif /* end _USE_DEDICATED_THREAD_MODEL */
	        rc = ha_gs_dispatch(HA_GS_NON_BLOCKING);
	        function="ha_gs_dispatch";
#ifdef _USE_DEDICATED_THREAD_MODEL  /* begin _USE_DEDICATED_THREAD_MODEL */
            }
#endif /* end _USE_DEDICATED_THREAD_MODEL */
            if (HA_GS_OK != rc) {
                fc_eid_t fid;
                printf("\nBad news, bad return code from dispatch[%s]!  "
			"Exiting.\n",
                       write_an_rc(rc));
                memset(fid, 0, sizeof(fid));
                ha_gs_get_ffdc_id(fid);
                fprintf(stderr, " FFDCID=[%s] is received\n", fid);
                if (HA_GS_NOT_OK == rc) {
                    exit(0);
                }
            }
	    break;
#else                                   /* else !_THREAD_SAFE */
            continue;
#endif                                  /* end !_THREAD_SAFE */
            /*
             * Finished with the 'd' command.
             */

            /*********************************************************************/
            /*
             * Just gobble up the newline, or if an unrecognized command, then
             * write an error.
             */
	  case '\n':                    /* Gobble up the newline */
	    continue;

          case 'o':
          case 'O':
            /* get node number */
            retVal = get_local_node_number();
            function="ha_gs_get_node_number()";
            if( (retVal == HA_GS_NOT_OK || retVal == HA_GS_NO_INIT ) && init_ok != HA_GS_OK ){
                rc = HA_GS_NO_INIT;
            } else {
                rc = retVal; 
            }
            break;
 
          case 'v':
          case 'V':
            /* adapter info */ 
            retVal = get_adapter_info();
            function="ha_gs_get_adapter_info()";
            if( (retVal == HA_GS_NOT_OK || retVal == HA_GS_NO_INIT ) && adapter_info_enabled != HA_GS_OK ){
                printf(" adapter info not enabled yet. quit and recall ha_gs_init().\n");
                rc = HA_GS_NO_INIT;
            } else {
                rc = retVal; 
            }
            break;

          case 'c':
          case 'C':
            /* get adapter info by ip */
            retVal = get_adapter_info_by_addr();
            function="ha_gs_get_adapter_info_by_addr()";
            if( retVal == HA_GS_NOT_OK  && adapter_info_enabled != HA_GS_OK ){
                printf(" adapter info not enabled yet. quit and recall ha_gs_init().\n");
                rc = retVal;
            } else if( retVal == HA_GS_NO_INIT ){
                printf(" call ha_gs_init().\n");
                rc = HA_GS_NO_INIT;
            } else if( retVal == HA_GS_NOT_OK ){
                printf("return code: HA_GS_NOT_OK, check your input.\n");
                rc = HA_GS_OK;
            } else {
                rc = retVal;
            }
            break;

          case 'k':
          case 'K':
            /*  get adapter info by group & id */
            retVal = get_adapter_info_by_id();
            function="ha_gs_get_adapter_info_by_id()";
            if( retVal == HA_GS_NO_INIT  ){
                printf("  HA_GS_NO_INIT,recall ha_gs_init().\n");
                rc = HA_GS_NO_INIT;
            } else if( retVal == HA_GS_NOT_OK ){
                printf("return code: HA_GS_NOT_OK, check your input.\n");
                rc = HA_GS_OK;
            } else {
                rc = retVal;
            }
            break;

          case 'r':
          case 'R':
            /* get adapter ip addr by group & id */
            retVal = get_ipaddr_by_id();
            function="ha_gs_get_ipaddr_by_id()";
            if(  retVal == HA_GS_NO_INIT ){
                printf(" recall ha_gs_init().\n");
                rc = HA_GS_NO_INIT;
            } else if( retVal == HA_GS_NOT_OK ){
                printf("return code: HA_GS_NOT_OK, check your input.\n");
                rc = HA_GS_OK;
            } else {
                rc = retVal;
            }
            break;

          case '6':
            /* get RSCT active version  */
            retVal = get_rsct_active_version();
            function="ha_gs_get_rsct_active_version()";
            if(  retVal == HA_GS_NO_INIT ){
                printf(" recall ha_gs_init().\n");
                rc = HA_GS_NO_INIT;
            } else if( retVal == HA_GS_NOT_OK ){
                printf("return code: HA_GS_NOT_OK, check your input.\n");
                rc = HA_GS_OK;
            } else {
                rc = retVal;
            }
            break;

          case '7':
            /* get RSCT installed version  */
            retVal = get_rsct_installed_version();
            function="ha_gs_get_rsct_installed_version()";
            if(  retVal == HA_GS_NO_INIT ){
                printf(" recall ha_gs_init().\n");
                rc = HA_GS_NO_INIT;
            } else if( retVal == HA_GS_NOT_OK ){
                printf("return code: HA_GS_NOT_OK, check your input.\n");
                rc = HA_GS_OK;
            } else {
                rc = retVal;
            }
            break;

          /* send a big message */
          /* To use this option:
             1. type "i"  initialize with hags
             2. type "j"  join a group
             3. follow the instruction on the screen to approve
                the join protocol so that it create a group
             4. type "8", then following the instruction on the screen
                to send a big message to the joined group. One can enter
                a message length (>=2000 and <=8M bytes) and then the code will
                build a given length message to send. When notification
                is received it will print first 10 characters and the last
                10 characters of the message. The last 3 characters are
                "END". (The code will add a checksum at the end of the
                message and so the actual message length = given length + 2.)
          */
          case '8':
            printf("\nTo use this option one should first use \'j\' option to\njoin a group and then type this option to send a big message to the group.\nOne can enter the message length.\n");
            if (build_big_pbm(&group_token, &info)) {
                function="ha_gs_send_message so far";
                rc = ha_gs_send_message(group_token, &info);
                break;
            } else {
                continue;
            }
            break;

          case 'M':
            /* call ha_gs_migrate_to_caa_prep() */
            printf("\n To use this option one should first initialize with hags library by using '9' option\n ");   
            function ="ha_gs_migrate_to_caa_to_prep so far";
            rc = ha_gs_migrate_to_caa_prep();
            break;

          case '#':
            /* call ha_gs_respond_domain_control() */
            quorum_response.notification_type = HA_GS_DOMAIN_NOTIFICATION;
            quorum_response.domain_event_type = HA_GS_OPQUORUM_INFO;
            quorum_response.info.quorum_response.notification_sequence = recvd_quorum_info_seqnum;
            printf( "Call ha_gs_respond_domain_control()\n" );
            rc = ha_gs_respond_domain_control(&quorum_response);
            fprintf(stderr, "Returned from ha_gs_respond_domain_control() with rc[%s]\n", \
                    write_an_rc(rc));
            rc = HA_GS_OK;
            break;
 
          case '*':
            /* call ha_gs_change_domaincb_ack_timeout() */
            fflush(stdout);
            fflush(stdin);

            printf("\nEnter the new timeout value :");
            scanf("%hu", &new_domaincb_ack_time_limit);

            printf("new_domaincb_ack_time_limit = %d\n", new_domaincb_ack_time_limit);
            printf( "Call ha_gs_change_domaincb_ack_timeout()\n");
            rc = ha_gs_change_domaincb_ack_timeout(&new_domaincb_ack_time_limit);
            fprintf(stderr, "Returned from ha_gs_change_domaincb_ack_timeout() with rc[%s]\n", \
                    write_an_rc(rc));
            rc = HA_GS_OK;
            break;
 
	  default:
	    printf("Unrecognized command: %c\n", input);
            fflush(stdout);
	    continue;
	}                               /* end switch() */

        /*
         * Write out the synchronous return code from the Group Services
         * interface call.  With the exception of the initialization call,
         * if we have HA_GS_OK here, we need await the asynchronous callbacks
         * to determine if our request was accepted and is executing.
         */
        if (verbose || (!handledResponsiveness)) {
            printf("%s returned rc:[%s]\n",
                   function,
                   write_an_rc(rc));
            fflush(stdout);
        }
    }                                   /* end while() loop */
}                                       /* end main() function */
