package org.jgroups.protocols.pbcast;
import org.jgroups.*;
import org.jgroups.stack.Protocol;
import org.jgroups.util.*;
import org.jgroups.util.Queue;
import org.apache.commons.logging.Log;
import java.io.*;
import java.util.*;
import java.util.List;
/**
* Group membership protocol. Handles joins/leaves/crashes (suspicions) and emits new views
* accordingly. Use VIEW_ENFORCER on top of this layer to make sure new members don't receive
* any messages until they are members
* @author Bela Ban
* @version $Id: GMS.java,v 1.68.2.5 2007/04/27 08:03:55 belaban Exp $
*/
public class GMS extends Protocol {
private GmsImpl impl=null;
Address local_addr=null;
final Membership members=new Membership(); // real membership
private final Membership tmp_members=new Membership(); // base for computing next view
/** Members joined but for which no view has been received yet */
private final Vector joining=new Vector(7);
/** Members excluded from group, but for which no view has been received yet */
private final Vector leaving=new Vector(7);
View view=null;
ViewId view_id=null;
private long ltime=0;
long join_timeout=5000;
long join_retry_timeout=2000;
long flush_timeout=4000;
long leave_timeout=5000;
private long digest_timeout=0; // time to wait for a digest (from PBCAST). should be fast
long merge_timeout=10000; // time to wait for all MERGE_RSPS
private final Object impl_mutex=new Object(); // synchronizes event entry into impl
private final Promise digest_promise=new Promise(); // holds result of GET_DIGEST event
private final Promise flush_promise = new Promise();
boolean use_flush=false;
private final Hashtable impls=new Hashtable(3);
private boolean shun=false;
boolean merge_leader=false; // can I initiate a merge ?
private boolean print_local_addr=true;
boolean disable_initial_coord=false; // can the member become a coord on startup or not ?
/** Setting this to false disables concurrent startups. This is only used by unit testing code
* for testing merging. To everybody else: don't change it to false ! */
boolean handle_concurrent_startup=true;
/** Whether view bundling (http://jira.jboss.com/jira/browse/JGRP-144) should be enabled or not. Setting this to
* false forces each JOIN/LEAVE/SUPSECT request to be handled separately. By default these requests are processed
* together if they are queued at approximately the same time */
private boolean view_bundling=true;
private long max_bundling_time=50; // 50ms max to wait for other JOIN, LEAVE or SUSPECT requests
static final String CLIENT="Client";
static final String COORD="Coordinator";
static final String PART="Participant";
TimeScheduler timer=null;
/** Max number of old members to keep in history */
protected int num_prev_mbrs=50;
/** Keeps track of old members (up to num_prev_mbrs) */
BoundedList prev_members=null;
int num_views=0;
/** Stores the last 20 views */
BoundedList prev_views=new BoundedList(20);
/** Class to process JOIN, LEAVE and MERGE requests */
private final ViewHandler view_handler=new ViewHandler();
/** To collect VIEW_ACKs from all members */
final AckCollector ack_collector=new AckCollector();
/** Time in ms to wait for all VIEW acks (0 == wait forever) */
long view_ack_collection_timeout=2000;
/** How long should a Resumer wait until resuming the ViewHandler */
long resume_task_timeout=20000;
boolean flushProtocolInStack=false;
public static final String name="GMS";
public GMS() {
initState();
}
public String getName() {
return name;
}
public String getView() {return view_id != null? view_id.toString() : "null";}
public int getNumberOfViews() {return num_views;}
public String getLocalAddress() {return local_addr != null? local_addr.toString() : "null";}
public String getMembers() {return members != null? members.toString() : "[]";}
public int getNumMembers() {return members != null? members.size() : 0;}
public long getJoinTimeout() {return join_timeout;}
public void setJoinTimeout(long t) {join_timeout=t;}
public long getJoinRetryTimeout() {return join_retry_timeout;}
public void setJoinRetryTimeout(long t) {join_retry_timeout=t;}
public boolean isShun() {return shun;}
public void setShun(boolean s) {shun=s;}
public String printPreviousMembers() {
StringBuffer sb=new StringBuffer();
if(prev_members != null) {
for(Enumeration en=prev_members.elements(); en.hasMoreElements();) {
sb.append(en.nextElement()).append("\n");
}
}
return sb.toString();
}
public int viewHandlerSize() {return view_handler.size();}
public boolean isViewHandlerSuspended() {return view_handler.suspended();}
public String dumpViewHandlerQueue() {
return view_handler.dumpQueue();
}
public String dumpViewHandlerHistory() {
return view_handler.dumpHistory();
}
public void suspendViewHandler() {
view_handler.suspend(null);
}
public void resumeViewHandler() {
view_handler.resumeForce();
}
Log getLog() {return log;}
ViewHandler getViewHandler() {return view_handler;}
public String printPreviousViews() {
StringBuffer sb=new StringBuffer();
for(Enumeration en=prev_views.elements(); en.hasMoreElements();) {
sb.append(en.nextElement()).append("\n");
}
return sb.toString();
}
public boolean isCoordinator() {
Address coord=determineCoordinator();
return coord != null && local_addr != null && local_addr.equals(coord);
}
public void resetStats() {
super.resetStats();
num_views=0;
prev_views.removeAll();
}
public Vector requiredDownServices() {
Vector retval=new Vector(3);
retval.addElement(new Integer(Event.GET_DIGEST));
retval.addElement(new Integer(Event.SET_DIGEST));
retval.addElement(new Integer(Event.FIND_INITIAL_MBRS));
return retval;
}
public void setImpl(GmsImpl new_impl) {
synchronized(impl_mutex) {
if(impl == new_impl) // superfluous
return;
impl=new_impl;
if(log.isDebugEnabled()) {
String msg=(local_addr != null? local_addr.toString()+" " : "") + "changed role to " + new_impl.getClass().getName();
log.debug(msg);
}
}
}
public GmsImpl getImpl() {
return impl;
}
public void init() throws Exception {
prev_members=new BoundedList(num_prev_mbrs);
timer=stack != null? stack.timer : null;
if(timer == null)
throw new Exception("GMS.init(): timer is null");
if(impl != null)
impl.init();
}
public void start() throws Exception {
if(impl != null) impl.start();
if(!flushProtocolInStack && use_flush){
log.warn("use_flush is true, however, FLUSH protocol not found in stack.");
use_flush = false;
}
}
public void stop() {
view_handler.stop(true);
if(impl != null) impl.stop();
if(prev_members != null)
prev_members.removeAll();
}
public void becomeCoordinator() {
CoordGmsImpl tmp=(CoordGmsImpl)impls.get(COORD);
if(tmp == null) {
tmp=new CoordGmsImpl(this);
impls.put(COORD, tmp);
}
try {
tmp.init();
}
catch(Exception e) {
log.error("exception switching to coordinator role", e);
}
setImpl(tmp);
}
public void becomeParticipant() {
ParticipantGmsImpl tmp=(ParticipantGmsImpl)impls.get(PART);
if(tmp == null) {
tmp=new ParticipantGmsImpl(this);
impls.put(PART, tmp);
}
try {
tmp.init();
}
catch(Exception e) {
log.error("exception switching to participant", e);
}
setImpl(tmp);
}
public void becomeClient() {
ClientGmsImpl tmp=(ClientGmsImpl)impls.get(CLIENT);
if(tmp == null) {
tmp=new ClientGmsImpl(this);
impls.put(CLIENT, tmp);
}
try {
tmp.init();
}
catch(Exception e) {
log.error("exception switching to client role", e);
}
setImpl(tmp);
}
boolean haveCoordinatorRole() {
return impl != null && impl instanceof CoordGmsImpl;
}
/**
* Computes the next view. Returns a copy that has <code>old_mbrs</code> and
* <code>suspected_mbrs</code> removed and <code>new_mbrs</code> added.
*/
public View getNextView(Collection new_mbrs, Collection old_mbrs, Collection suspected_mbrs) {
Vector mbrs;
long vid;
View v;
Membership tmp_mbrs;
Address tmp_mbr;
synchronized(members) {
if(view_id == null) {
log.error("view_id is null");
return null; // this should *never* happen !
}
vid=Math.max(view_id.getId(), ltime) + 1;
ltime=vid;
tmp_mbrs=tmp_members.copy(); // always operate on the temporary membership
tmp_mbrs.remove(suspected_mbrs);
tmp_mbrs.remove(old_mbrs);
tmp_mbrs.add(new_mbrs);
mbrs=tmp_mbrs.getMembers();
v=new View(local_addr, vid, mbrs);
// Update membership (see DESIGN for explanation):
tmp_members.set(mbrs);
// Update joining list (see DESIGN for explanation)
if(new_mbrs != null) {
for(Iterator it=new_mbrs.iterator(); it.hasNext();) {
tmp_mbr=(Address)it.next();
if(!joining.contains(tmp_mbr))
joining.addElement(tmp_mbr);
}
}
// Update leaving list (see DESIGN for explanations)
if(old_mbrs != null) {
for(Iterator it=old_mbrs.iterator(); it.hasNext();) {
Address addr=(Address)it.next();
if(!leaving.contains(addr))
leaving.add(addr);
}
}
if(suspected_mbrs != null) {
for(Iterator it=suspected_mbrs.iterator(); it.hasNext();) {
Address addr=(Address)it.next();
if(!leaving.contains(addr))
leaving.add(addr);
}
}
return v;
}
}
/**
Compute a new view, given the current view, the new members and the suspected/left
members. Then simply mcast the view to all members. This is different to the VS GMS protocol,
in which we run a FLUSH protocol which tries to achive consensus on the set of messages mcast in
the current view before proceeding to install the next view.
The members for the new view are computed as follows:
<pre>
existing leaving suspected joining
1. new_view y n n y
2. tmp_view y y n y
(view_dest)
</pre>
<ol>
<li>
The new view to be installed includes the existing members plus the joining ones and
excludes the leaving and suspected members.
<li>
A temporary view is sent down the stack as an <em>event</em>. This allows the bottom layer
(e.g. UDP or TCP) to determine the members to which to send a multicast message. Compared
to the new view, leaving members are <em>included</em> since they have are waiting for a
view in which they are not members any longer before they leave. So, if we did not set a
temporary view, joining members would not receive the view (signalling that they have been
joined successfully). The temporary view is essentially the current view plus the joining
members (old members are still part of the current view).
</ol>
*/
public void castViewChange(Vector new_mbrs, Vector old_mbrs, Vector suspected_mbrs) {
View new_view;
// next view: current mbrs + new_mbrs - old_mbrs - suspected_mbrs
new_view=getNextView(new_mbrs, old_mbrs, suspected_mbrs);
castViewChange(new_view, null);
}
public void castViewChange(View new_view, Digest digest) {
castViewChangeWithDest(new_view, digest, null);
}
/**
* Broadcasts the new view and digest, and waits for acks from all members in the list given as argument.
* If the list is null, we take the members who are part of new_view
* @param new_view
* @param digest
* @param members
*/
public void castViewChangeWithDest(View new_view, Digest digest, java.util.List members) {
Message view_change_msg;
GmsHeader hdr;
long start, stop;
ViewId vid=new_view.getVid();
int size=-1;
if(members == null || members.size() == 0)
members=new_view.getMembers();
if(log.isTraceEnabled())
log.trace("mcasting view {" + new_view + "} (" + new_view.size() + " mbrs)\n");
start=System.currentTimeMillis();
view_change_msg=new Message(); // bcast to all members
hdr=new GmsHeader(GmsHeader.VIEW, new_view);
hdr.my_digest=digest;
view_change_msg.putHeader(name, hdr);
ack_collector.reset(vid, members);
size=ack_collector.size();
passDown(new Event(Event.MSG, view_change_msg));
try {
ack_collector.waitForAllAcks(view_ack_collection_timeout);
stop=System.currentTimeMillis();
if(log.isTraceEnabled())
log.trace("received all ACKs (" + size + ") for " + vid + " in " + (stop-start) + "ms");
}
catch(TimeoutException e) {
log.warn("failed to collect all ACKs (" + size + ") for view " + new_view + " after " + view_ack_collection_timeout +
"ms, missing ACKs from " + ack_collector.printMissing() + " (received=" + ack_collector.printReceived() +
"), local_addr=" + local_addr);
}
}
/**
* Sets the new view and sends a VIEW_CHANGE event up and down the stack. If the view is a MergeView (subclass
* of View), then digest will be non-null and has to be set before installing the view.
*/
public void installView(View new_view, Digest digest) {
if(digest != null)
mergeDigest(digest);
installView(new_view);
}
/**
* Sets the new view and sends a VIEW_CHANGE event up and down the stack.
*/
public void installView(View new_view) {
Address coord;
int rc;
ViewId vid=new_view.getVid();
Vector mbrs=new_view.getMembers();
if(log.isDebugEnabled()) log.debug("[local_addr=" + local_addr + "] view is " + new_view);
if(stats) {
num_views++;
prev_views.add(new_view);
}
ack_collector.handleView(new_view);
// Discards view with id lower than our own. Will be installed without check if first view
if(view_id != null) {
rc=vid.compareTo(view_id);
if(rc <= 0) {
if(log.isTraceEnabled() && rc < 0) // only scream if view is smaller, silently discard same views
log.trace("[" + local_addr + "] received view < current view;" +
" discarding it (current vid: " + view_id + ", new vid: " + vid + ')');
return;
}
}
ltime=Math.max(vid.getId(), ltime); // compute Lamport logical time
/* Check for self-inclusion: if I'm not part of the new membership, I just discard it.
This ensures that messages sent in view V1 are only received by members of V1 */
if(checkSelfInclusion(mbrs) == false) {
// only shun if this member was previously part of the group. avoids problem where multiple
// members (e.g. X,Y,Z) join {A,B} concurrently, X is joined first, and Y and Z get view
// {A,B,X}, which would cause Y and Z to be shunned as they are not part of the membership
// bela Nov 20 2003
if(shun && local_addr != null && prev_members.contains(local_addr)) {
if(log.isWarnEnabled())
log.warn("I (" + local_addr + ") am not a member of view " + new_view +
", shunning myself and leaving the group (prev_members are " + prev_members +
", current view is " + view + ")");
if(impl != null)
impl.handleExit();
passUp(new Event(Event.EXIT));
}
else {
if(log.isWarnEnabled()) log.warn("I (" + local_addr + ") am not a member of view " + new_view + "; discarding view");
}
return;
}
synchronized(members) { // serialize access to views
// assign new_view to view_id
if(new_view instanceof MergeView)
view=new View(new_view.getVid(), new_view.getMembers());
else
view=new_view;
view_id=vid.copy();
// Set the membership. Take into account joining members
if(mbrs != null && mbrs.size() > 0) {
members.set(mbrs);
tmp_members.set(members);
joining.removeAll(mbrs); // remove all members in mbrs from joining
// remove all elements from 'leaving' that are not in 'mbrs'
leaving.retainAll(mbrs);
tmp_members.add(joining); // add members that haven't yet shown up in the membership
tmp_members.remove(leaving); // remove members that haven't yet been removed from the membership
// add to prev_members
for(Iterator it=mbrs.iterator(); it.hasNext();) {
Address addr=(Address)it.next();
if(!prev_members.contains(addr))
prev_members.add(addr);
}
}
// Send VIEW_CHANGE event up and down the stack:
Event view_event=new Event(Event.VIEW_CHANGE, new_view.clone());
// changed order of passing view up and down (http://jira.jboss.com/jira/browse/JGRP-347)
passUp(view_event);
passDown(view_event); // needed e.g. by failure detector or UDP
coord=determineCoordinator();
// if(coord != null && coord.equals(local_addr) && !(coord.equals(vid.getCoordAddress()))) {
// changed on suggestion by yaronr and Nicolas Piedeloupe
if(coord != null && coord.equals(local_addr) && !haveCoordinatorRole()) {
becomeCoordinator();
}
else {
if(haveCoordinatorRole() && !local_addr.equals(coord))
becomeParticipant();
}
}
}
protected Address determineCoordinator() {
synchronized(members) {
return members != null && members.size() > 0? (Address)members.elementAt(0) : null;
}
}
/** Checks whether the potential_new_coord would be the new coordinator (2nd in line) */
protected boolean wouldBeNewCoordinator(Address potential_new_coord) {
Address new_coord;
if(potential_new_coord == null) return false;
synchronized(members) {
if(members.size() < 2) return false;
new_coord=(Address)members.elementAt(1); // member at 2nd place
return new_coord != null && new_coord.equals(potential_new_coord);
}
}
/** Returns true if local_addr is member of mbrs, else false */
protected boolean checkSelfInclusion(Vector mbrs) {
Object mbr;
if(mbrs == null)
return false;
for(int i=0; i < mbrs.size(); i++) {
mbr=mbrs.elementAt(i);
if(mbr != null && local_addr.equals(mbr))
return true;
}
return false;
}
public View makeView(Vector mbrs) {
Address coord=null;
long id=0;
if(view_id != null) {
coord=view_id.getCoordAddress();
id=view_id.getId();
}
return new View(coord, id, mbrs);
}
public View makeView(Vector mbrs, ViewId vid) {
Address coord=null;
long id=0;
if(vid != null) {
coord=vid.getCoordAddress();
id=vid.getId();
}
return new View(coord, id, mbrs);
}
/** Send down a SET_DIGEST event */
public void setDigest(Digest d) {
passDown(new Event(Event.SET_DIGEST, d));
}
/** Send down a MERGE_DIGEST event */
public void mergeDigest(Digest d) {
passDown(new Event(Event.MERGE_DIGEST, d));
}
/** Sends down a GET_DIGEST event and waits for the GET_DIGEST_OK response, or
timeout, whichever occurs first */
public Digest getDigest() {
Digest ret=null;
digest_promise.reset();
passDown(Event.GET_DIGEST_EVT);
try {
ret=(Digest)digest_promise.getResultWithTimeout(digest_timeout);
}
catch(TimeoutException e) {
if(log.isErrorEnabled()) log.error("digest could not be fetched from below");
}
return ret;
}
boolean startFlush(View new_view,int numberOfAttempts){
boolean successfulFlush = false;
Vector membersInNewView = new_view.getMembers();
if(membersInNewView == null || membersInNewView.isEmpty()){
//there are no members to FLUSH
successfulFlush = true;
}
else{
flush_promise.reset();
passUp(new Event(Event.SUSPEND, new_view));
try{
Boolean r = (Boolean) flush_promise.getResultWithTimeout(flush_timeout);
successfulFlush = r.booleanValue();
}
catch (TimeoutException e){
log.warn("GMS coordinator " + local_addr + " timed out waiting for flush responses after " + flush_timeout
+ " msec");
}
if (!successfulFlush && numberOfAttempts > 0){
long backOffSleepTime = Util.random(5000);
if(log.isInfoEnabled())
log.info("Flush in progress detected at GMS coordinator " + local_addr + ". Backing off for "
+ backOffSleepTime + " ms. Attempts left " + numberOfAttempts);
Util.sleepRandom(backOffSleepTime);
successfulFlush = startFlush(new_view, --numberOfAttempts);
}
}
return successfulFlush;
}
void stopFlush(View view) {
//since we did not call startFlush on
//empty view do not call RESUME either
if(view != null && view.getMembers().isEmpty())
return;
if (log.isDebugEnabled()) {
log.debug("sending RESUME event");
}
passUp(new Event(Event.RESUME));
}
public void up(Event evt) {
Object obj;
Message msg;
GmsHeader hdr;
MergeData merge_data;
switch(evt.getType()) {
case Event.MSG:
msg=(Message)evt.getArg();
obj=msg.getHeader(name);
if(obj == null || !(obj instanceof GmsHeader))
break;
hdr=(GmsHeader)msg.removeHeader(name);
switch(hdr.type) {
case GmsHeader.JOIN_REQ:
view_handler.add(new Request(Request.JOIN, hdr.mbr, false, null));
break;
case GmsHeader.JOIN_RSP:
impl.handleJoinResponse(hdr.join_rsp);
break;
case GmsHeader.LEAVE_REQ:
if(log.isDebugEnabled())
log.debug("received LEAVE_REQ for " + hdr.mbr + " from " + msg.getSrc());
if(hdr.mbr == null) {
if(log.isErrorEnabled()) log.error("LEAVE_REQ's mbr field is null");
return;
}
view_handler.add(new Request(Request.LEAVE, hdr.mbr, false, null));
break;
case GmsHeader.LEAVE_RSP:
impl.handleLeaveResponse();
break;
case GmsHeader.VIEW:
if(hdr.view == null) {
if(log.isErrorEnabled()) log.error("[VIEW]: view == null");
return;
}
// send VIEW_ACK to sender of view
Address coord=msg.getSrc();
Message view_ack=new Message(coord, null, null);
GmsHeader tmphdr=new GmsHeader(GmsHeader.VIEW_ACK, hdr.view);
view_ack.putHeader(name, tmphdr);
if(log.isTraceEnabled())
log.trace("sending VIEW_ACK to " + coord);
passDown(new Event(Event.MSG, view_ack));
impl.handleViewChange(hdr.view, hdr.my_digest);
break;
case GmsHeader.VIEW_ACK:
Object sender=msg.getSrc();
ack_collector.ack(sender);
return; // don't pass further up
case GmsHeader.MERGE_REQ:
impl.handleMergeRequest(msg.getSrc(), hdr.merge_id);
break;
case GmsHeader.MERGE_RSP:
merge_data=new MergeData(msg.getSrc(), hdr.view, hdr.my_digest);
merge_data.merge_rejected=hdr.merge_rejected;
impl.handleMergeResponse(merge_data, hdr.merge_id);
break;
case GmsHeader.INSTALL_MERGE_VIEW:
impl.handleMergeView(new MergeData(msg.getSrc(), hdr.view, hdr.my_digest), hdr.merge_id);
break;
case GmsHeader.CANCEL_MERGE:
impl.handleMergeCancelled(hdr.merge_id);
break;
default:
if(log.isErrorEnabled()) log.error("GmsHeader with type=" + hdr.type + " not known");
}
return; // don't pass up
case Event.CONNECT_OK: // sent by someone else, but WE are responsible for sending this !
case Event.DISCONNECT_OK: // dito (e.g. sent by TP layer). Don't send up the stack
return;
case Event.SET_LOCAL_ADDRESS:
local_addr=(Address)evt.getArg();
if(print_local_addr) {
System.out.println("\n-------------------------------------------------------\n" +
"GMS: address is " + local_addr +
"\n-------------------------------------------------------");
}
break; // pass up
case Event.SUSPECT:
Address suspected=(Address)evt.getArg();
view_handler.add(new Request(Request.SUSPECT, suspected, true, null));
ack_collector.suspect(suspected);
break; // pass up
case Event.UNSUSPECT:
impl.unsuspect((Address)evt.getArg());
return; // discard
case Event.MERGE:
view_handler.add(new Request(Request.MERGE, null, false, (Vector)evt.getArg()));
return; // don't pass up
}
if(impl.handleUpEvent(evt))
passUp(evt);
}
/**
This method is overridden to avoid hanging on getDigest(): when a JOIN is received, the coordinator needs
to retrieve the digest from the NAKACK layer. It therefore sends down a GET_DIGEST event, to which the NAKACK layer
responds with a GET_DIGEST_OK event.<p>
However, the GET_DIGEST_OK event will not be processed because the thread handling the JOIN request won't process
the GET_DIGEST_OK event until the JOIN event returns. The receiveUpEvent() method is executed by the up-handler
thread of the lower protocol and therefore can handle the event. All we do here is unblock the mutex on which
JOIN is waiting, allowing JOIN to return with a valid digest. The GET_DIGEST_OK event is then discarded, because
it won't be processed twice.
*/
public void receiveUpEvent(Event evt) {
switch(evt.getType()) {
case Event.GET_DIGEST_OK:
digest_promise.setResult(evt.getArg());
return; // don't pass further up
}
super.receiveUpEvent(evt);
}
public void down(Event evt) {
switch(evt.getType()) {
case Event.CONNECT:
Object arg=null;
passDown(evt);
if(local_addr == null)
if(log.isFatalEnabled()) log.fatal("[CONNECT] local_addr is null");
try {
impl.join(local_addr);
}
catch(SecurityException e) {
arg=e;
}
passUp(new Event(Event.CONNECT_OK, arg));
return; // don't pass down: was already passed down
case Event.DISCONNECT:
impl.leave((Address)evt.getArg());
if(!(impl instanceof CoordGmsImpl)) {
passUp(new Event(Event.DISCONNECT_OK));
initState(); // in case connect() is called again
}
break; // pass down
case Event.SUSPEND_OK:
flush_promise.setResult(Boolean.TRUE);
break;
case Event.SUSPEND_FAILED :
flush_promise.setResult(Boolean.FALSE);
break;
case Event.CONFIG :
Map config = (Map) evt.getArg();
if(config != null && config.containsKey("flush_timeout")){
Long ftimeout = (Long) config.get("flush_timeout");
use_flush = true;
flush_timeout = ftimeout.longValue();
}
if((config != null && !config.containsKey("flush_suported"))){
flushProtocolInStack = true;
}
break;
}
passDown(evt);
}
/** Setup the Protocol instance according to the configuration string */
public boolean setProperties(Properties props) {
String str;
super.setProperties(props);
str=props.getProperty("shun");
if(str != null) {
shun=Boolean.valueOf(str).booleanValue();
props.remove("shun");
}
str=props.getProperty("merge_leader");
if(str != null) {
merge_leader=Boolean.valueOf(str).booleanValue();
props.remove("merge_leader");
}
str=props.getProperty("print_local_addr");
if(str != null) {
print_local_addr=Boolean.valueOf(str).booleanValue();
props.remove("print_local_addr");
}
str=props.getProperty("join_timeout"); // time to wait for JOIN
if(str != null) {
join_timeout=Long.parseLong(str);
props.remove("join_timeout");
}
str=props.getProperty("join_retry_timeout"); // time to wait between JOINs
if(str != null) {
join_retry_timeout=Long.parseLong(str);
props.remove("join_retry_timeout");
}
str=props.getProperty("leave_timeout"); // time to wait until coord responds to LEAVE req.
if(str != null) {
leave_timeout=Long.parseLong(str);
props.remove("leave_timeout");
}
str=props.getProperty("merge_timeout"); // time to wait for MERGE_RSPS from subgroup coordinators
if(str != null) {
merge_timeout=Long.parseLong(str);
props.remove("merge_timeout");
}
str=props.getProperty("digest_timeout"); // time to wait for GET_DIGEST_OK from PBCAST
if(str != null) {
digest_timeout=Long.parseLong(str);
props.remove("digest_timeout");
}
str=props.getProperty("view_ack_collection_timeout");
if(str != null) {
view_ack_collection_timeout=Long.parseLong(str);
props.remove("view_ack_collection_timeout");
}
str=props.getProperty("resume_task_timeout");
if(str != null) {
resume_task_timeout=Long.parseLong(str);
props.remove("resume_task_timeout");
}
str=props.getProperty("disable_initial_coord");
if(str != null) {
disable_initial_coord=Boolean.valueOf(str).booleanValue();
props.remove("disable_initial_coord");
}
str=props.getProperty("handle_concurrent_startup");
if(str != null) {
handle_concurrent_startup=Boolean.valueOf(str).booleanValue();
props.remove("handle_concurrent_startup");
}
str=props.getProperty("num_prev_mbrs");
if(str != null) {
num_prev_mbrs=Integer.parseInt(str);
props.remove("num_prev_mbrs");
}
str=props.getProperty("use_flush");
if(str != null) {
use_flush=Boolean.valueOf(str).booleanValue();
props.remove("use_flush");
}
str=props.getProperty("flush_timeout");
if(str != null) {
flush_timeout=Long.parseLong(str);
props.remove("flush_timeout");
}
str=props.getProperty("view_bundling");
if(str != null) {
view_bundling=Boolean.valueOf(str).booleanValue();
props.remove("view_bundling");
}
str=props.getProperty("max_bundling_time");
if(str != null) {
max_bundling_time=Long.parseLong(str);
props.remove("max_bundling_time");
}
if(props.size() > 0) {
log.error("the following properties are not recognized: " + props);
return false;
}
return true;
}
/* ------------------------------- Private Methods --------------------------------- */
final void initState() {
becomeClient();
view_id=null;
view=null;
}
/* --------------------------- End of Private Methods ------------------------------- */
public static class GmsHeader extends Header implements Streamable {
public static final byte JOIN_REQ=1;
public static final byte JOIN_RSP=2;
public static final byte LEAVE_REQ=3;
public static final byte LEAVE_RSP=4;
public static final byte VIEW=5;
public static final byte MERGE_REQ=6;
public static final byte MERGE_RSP=7;
public static final byte INSTALL_MERGE_VIEW=8;
public static final byte CANCEL_MERGE=9;
public static final byte VIEW_ACK=10;
byte type=0;
View view=null; // used when type=VIEW or MERGE_RSP or INSTALL_MERGE_VIEW
Address mbr=null; // used when type=JOIN_REQ or LEAVE_REQ
JoinRsp join_rsp=null; // used when type=JOIN_RSP
Digest my_digest=null; // used when type=MERGE_RSP or INSTALL_MERGE_VIEW
ViewId merge_id=null; // used when type=MERGE_REQ or MERGE_RSP or INSTALL_MERGE_VIEW or CANCEL_MERGE
boolean merge_rejected=false; // used when type=MERGE_RSP
public GmsHeader() {
} // used for Externalization
public GmsHeader(byte type) {
this.type=type;
}
/** Used for VIEW header */
public GmsHeader(byte type, View view) {
this.type=type;
this.view=view;
}
/** Used for JOIN_REQ or LEAVE_REQ header */
public GmsHeader(byte type, Address mbr) {
this.type=type;
this.mbr=mbr;
}
/** Used for JOIN_RSP header */
public GmsHeader(byte type, JoinRsp join_rsp) {
this.type=type;
this.join_rsp=join_rsp;
}
public byte getType() {
return type;
}
public Address getMemeber() {
return mbr;
}
public String toString() {
StringBuffer sb=new StringBuffer("GmsHeader");
sb.append('[' + type2String(type) + ']');
switch(type) {
case JOIN_REQ:
sb.append(": mbr=" + mbr);
break;
case JOIN_RSP:
sb.append(": join_rsp=" + join_rsp);
break;
case LEAVE_REQ:
sb.append(": mbr=" + mbr);
break;
case LEAVE_RSP:
break;
case VIEW:
case VIEW_ACK:
sb.append(": view=" + view);
break;
case MERGE_REQ:
sb.append(": merge_id=" + merge_id);
break;
case MERGE_RSP:
sb.append(": view=" + view + ", digest=" + my_digest + ", merge_rejected=" + merge_rejected +
", merge_id=" + merge_id);
break;
case INSTALL_MERGE_VIEW:
sb.append(": view=" + view + ", digest=" + my_digest);
break;
case CANCEL_MERGE:
sb.append(", <merge cancelled>, merge_id=" + merge_id);
break;
}
return sb.toString();
}
public static String type2String(int type) {
switch(type) {
case JOIN_REQ: return "JOIN_REQ";
case JOIN_RSP: return "JOIN_RSP";
case LEAVE_REQ: return "LEAVE_REQ";
case LEAVE_RSP: return "LEAVE_RSP";
case VIEW: return "VIEW";
case MERGE_REQ: return "MERGE_REQ";
case MERGE_RSP: return "MERGE_RSP";
case INSTALL_MERGE_VIEW: return "INSTALL_MERGE_VIEW";
case CANCEL_MERGE: return "CANCEL_MERGE";
case VIEW_ACK: return "VIEW_ACK";
default: return "<unknown>";
}
}
public void writeExternal(ObjectOutput out) throws IOException {
out.writeByte(type);
out.writeObject(view);
out.writeObject(mbr);
out.writeObject(join_rsp);
out.writeObject(my_digest);
out.writeObject(merge_id);
out.writeBoolean(merge_rejected);
}
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
type=in.readByte();
view=(View)in.readObject();
mbr=(Address)in.readObject();
join_rsp=(JoinRsp)in.readObject();
my_digest=(Digest)in.readObject();
merge_id=(ViewId)in.readObject();
merge_rejected=in.readBoolean();
}
public void writeTo(DataOutputStream out) throws IOException {
out.writeByte(type);
boolean isMergeView=view != null && view instanceof MergeView;
out.writeBoolean(isMergeView);
Util.writeStreamable(view, out);
Util.writeAddress(mbr, out);
Util.writeStreamable(join_rsp, out);
Util.writeStreamable(my_digest, out);
Util.writeStreamable(merge_id, out); // kludge: we know merge_id is a ViewId
out.writeBoolean(merge_rejected);
}
public void readFrom(DataInputStream in) throws IOException, IllegalAccessException, InstantiationException {
type=in.readByte();
boolean isMergeView=in.readBoolean();
if(isMergeView)
view=(View)Util.readStreamable(MergeView.class, in);
else
view=(View)Util.readStreamable(View.class, in);
mbr=Util.readAddress(in);
join_rsp=(JoinRsp)Util.readStreamable(JoinRsp.class, in);
my_digest=(Digest)Util.readStreamable(Digest.class, in);
merge_id=(ViewId)Util.readStreamable(ViewId.class, in);
merge_rejected=in.readBoolean();
}
public long size() {
long retval=Global.BYTE_SIZE *2; // type + merge_rejected
retval+=Global.BYTE_SIZE; // presence view
retval+=Global.BYTE_SIZE; // MergeView or View
if(view != null)
retval+=view.serializedSize();
retval+=Util.size(mbr);
retval+=Global.BYTE_SIZE; // presence of join_rsp
if(join_rsp != null)
retval+=join_rsp.serializedSize();
retval+=Global.BYTE_SIZE; // presence for my_digest
if(my_digest != null)
retval+=my_digest.serializedSize();
retval+=Global.BYTE_SIZE; // presence for merge_id
if(merge_id != null)
retval+=merge_id.serializedSize();
return retval;
}
}
public static class Request {
static final int JOIN = 1;
static final int LEAVE = 2;
static final int SUSPECT = 3;
static final int MERGE = 4;
static final int VIEW = 5;
int type=-1;
Address mbr;
boolean suspected;
Vector coordinators;
View view;
Digest digest;
List target_members;
Request(int type) {
this.type=type;
}
Request(int type, Address mbr, boolean suspected, Vector coordinators) {
this.type=type;
this.mbr=mbr;
this.suspected=suspected;
this.coordinators=coordinators;
}
public int getType() {
return type;
}
public String toString() {
switch(type) {
case JOIN: return "JOIN(" + mbr + ")";
case LEAVE: return "LEAVE(" + mbr + ", " + suspected + ")";
case SUSPECT: return "SUSPECT(" + mbr + ")";
case MERGE: return "MERGE(" + coordinators + ")";
case VIEW: return "VIEW (" + view.getVid() + ")";
}
return "<invalid (type=" + type + ")";
}
/**
* Specifies whether this request can be processed with other request simultaneously
*/
public boolean canBeProcessedTogether(Request other) {
if(other == null)
return false;
int other_type=other.getType();
return (type == JOIN || type == LEAVE || type == SUSPECT) &&
(other_type == JOIN || other_type == LEAVE || other_type == SUSPECT);
}
}
/**
* Class which processes JOIN, LEAVE and MERGE requests. Requests are queued and processed in FIFO order
* @author Bela Ban
* @version $Id: GMS.java,v 1.68.2.5 2007/04/27 08:03:55 belaban Exp $
*/
class ViewHandler implements Runnable {
volatile Thread thread;
Queue q=new Queue(); // Queue<Request>
boolean suspended=false;
final static long INTERVAL=5000;
private static final long MAX_COMPLETION_TIME=10000;
/** Maintains a list of the last 20 requests */
private final BoundedList history=new BoundedList(20);
/** Map<Object,TimeScheduler.CancellableTask>. Keeps track of Resumer tasks which have not fired yet */
private final Map resume_tasks=new HashMap();
private Object merge_id=null;
void add(Request req) {
add(req, false, false);
}
synchronized void add(Request req, boolean at_head, boolean unsuspend) {
if(suspended && !unsuspend) {
log.warn("queue is suspended; request " + req + " is discarded");
return;
}
start(unsuspend);
try {
if(at_head)
q.addAtHead(req);
else
q.add(req);
history.add(new Date() + ": " + req.toString());
}
catch(QueueClosedException e) {
if(log.isTraceEnabled())
log.trace("queue is closed; request " + req + " is discarded");
}
}
void waitUntilCompleted(long timeout) {
waitUntilCompleted(timeout, false);
}
synchronized void waitUntilCompleted(long timeout, boolean resume) {
if(thread != null) {
try {
thread.join(timeout);
}
catch(InterruptedException e) {
}
}
if(resume)
resumeForce();
}
/**
* Waits until the current request has been processes, then clears the queue and discards new
* requests from now on
*/
public synchronized void suspend(Object merge_id) {
if(suspended)
return;
suspended=true;
this.merge_id=merge_id;
q.clear();
waitUntilCompleted(MAX_COMPLETION_TIME);
q.close(true);
if(log.isTraceEnabled())
log.trace("suspended ViewHandler");
Resumer r=new Resumer(resume_task_timeout, merge_id, resume_tasks, this);
resume_tasks.put(merge_id, r);
timer.add(r);
}
public synchronized void resume(Object merge_id) {
if(!suspended)
return;
boolean same_merge_id=this.merge_id != null && merge_id != null && this.merge_id.equals(merge_id);
same_merge_id=same_merge_id || (this.merge_id == null && merge_id == null);
if(!same_merge_id) {
if(log.isWarnEnabled())
log.warn("resume(" +merge_id+ ") does not match " + this.merge_id + ", ignoring resume()");
return;
}
synchronized(resume_tasks) {
TimeScheduler.CancellableTask task=(TimeScheduler.CancellableTask)resume_tasks.get(merge_id);
if(task != null) {
task.cancel();
resume_tasks.remove(merge_id);
}
}
resumeForce();
}
public synchronized void resumeForce() {
if(q.closed())
q.reset();
suspended=false;
if(log.isTraceEnabled())
log.trace("resumed ViewHandler");
}
public void run() {
long start, stop, wait_time;
List requests=new LinkedList();
while(Thread.currentThread().equals(thread)) {
requests.clear();
try {
boolean keepGoing=false;
start=System.currentTimeMillis();
do {
Request firstRequest=(Request)q.remove(INTERVAL); // throws a TimeoutException if it runs into timeout
requests.add(firstRequest);
if(q.size() > 0) {
Request nextReq=(Request)q.peek();
keepGoing=view_bundling && firstRequest.canBeProcessedTogether(nextReq);
}
else {
stop=System.currentTimeMillis();
wait_time=max_bundling_time - (stop-start);
if(wait_time > 0)
Util.sleep(wait_time);
keepGoing=q.size() > 0;
}
}
while(keepGoing);
process(requests);
}
catch(QueueClosedException e) {
break;
}
catch(TimeoutException e) {
break;
}
catch(Throwable catchall) {
Util.sleep(50);
}
}
}
public int size() {return q.size();}
public boolean suspended() {return suspended;}
public String dumpQueue() {
StringBuffer sb=new StringBuffer();
List v=q.values();
for(Iterator it=v.iterator(); it.hasNext();) {
sb.append(it.next() + "\n");
}
return sb.toString();
}
public String dumpHistory() {
StringBuffer sb=new StringBuffer();
for(Enumeration en=history.elements(); en.hasMoreElements();) {
sb.append(en.nextElement() + "\n");
}
return sb.toString();
}
private void process(List requests) {
if(requests.isEmpty())
return;
if(log.isTraceEnabled())
log.trace("processing " + requests);
Request firstReq=(Request)requests.get(0);
switch(firstReq.type) {
case Request.JOIN:
case Request.LEAVE:
case Request.SUSPECT:
Collection newMembers=new LinkedHashSet(requests.size());
Collection suspectedMembers=new LinkedHashSet(requests.size());
Collection oldMembers=new LinkedHashSet(requests.size());
for(Iterator i=requests.iterator(); i.hasNext();) {
Request req=(Request)i.next();
switch(req.type) {
case Request.JOIN:
newMembers.add(req.mbr);
break;
case Request.LEAVE:
if(req.suspected)
suspectedMembers.add(req.mbr);
else
oldMembers.add(req.mbr);
break;
case Request.SUSPECT:
suspectedMembers.add(req.mbr);
break;
}
}
impl.handleMembershipChange(newMembers, oldMembers, suspectedMembers);
break;
case Request.MERGE:
if(requests.size() > 1)
log.error("more than one MERGE request to process, ignoring the others");
impl.merge(firstReq.coordinators);
break;
case Request.VIEW:
if(requests.size() > 1)
log.error("more than one VIEW request to process, ignoring the others");
try {
if (use_flush){
boolean successfulFlush = startFlush(firstReq.view, 3);
if (successfulFlush){
log.info("Successful GMS flush by coordinator at " + getLocalAddress());
}
}
castViewChangeWithDest(firstReq.view, firstReq.digest, firstReq.target_members);
}
finally {
if(use_flush)
stopFlush(firstReq.view);
}
break;
default:
log.error("request " + firstReq.type + " is unknown; discarded");
}
}
synchronized void start(boolean unsuspend) {
if(q.closed())
q.reset();
if(unsuspend) {
suspended=false;
synchronized(resume_tasks) {
TimeScheduler.CancellableTask task=(TimeScheduler.CancellableTask)resume_tasks.get(merge_id);
if(task != null) {
task.cancel();
resume_tasks.remove(merge_id);
}
}
}
merge_id=null;
if(thread == null || !thread.isAlive()) {
thread=new Thread(Util.getGlobalThreadGroup(), this, "ViewHandler");
thread.setDaemon(false); // thread cannot terminate if we have tasks left, e.g. when we as coord leave
thread.start();
if(log.isTraceEnabled())
log.trace("ViewHandler started");
}
}
synchronized void stop(boolean flush) {
q.close(flush);
TimeScheduler.CancellableTask task;
synchronized(resume_tasks) {
for(Iterator it=resume_tasks.values().iterator(); it.hasNext();) {
task=(TimeScheduler.CancellableTask)it.next();
task.cancel();
}
resume_tasks.clear();
}
merge_id=null;
// resumeForce();
}
}
/**
* Resumer is a second line of defense: when the ViewHandler is suspended, it will be resumed when the current
* merge is cancelled, or when the merge completes. However, in a case where this never happens (this
* shouldn't be the case !), the Resumer will nevertheless resume the ViewHandler.
* We chose this strategy because ViewHandler is critical: if it is suspended indefinitely, we would
* not be able to process new JOIN requests ! So, this is for peace of mind, although it most likely
* will never be used...
*/
static class Resumer implements TimeScheduler.CancellableTask {
boolean cancelled=false;
long interval;
final Object token;
final Map tasks;
final ViewHandler handler;
public Resumer(long interval, final Object token, final Map t, final ViewHandler handler) {
this.interval=interval;
this.token=token;
this.tasks=t;
this.handler=handler;
}
public void cancel() {
cancelled=true;
}
public boolean cancelled() {
return cancelled;
}
public long nextInterval() {
return interval;
}
public void run() {
TimeScheduler.CancellableTask t;
boolean execute=true;
synchronized(tasks) {
t=(TimeScheduler.CancellableTask)tasks.get(token);
if(t != null) {
t.cancel();
execute=true;
}
else {
execute=false;
}
tasks.remove(token);
}
if(execute) {
handler.resume(token);
}
}
}
}
|